A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat(enrichers): add decorator to all enrichers
dextmorgne5eb5b622876
diff --git a/flowsint-enrichers/src/flowsint_enrichers/__init__.py b/flowsint-enrichers/src/flowsint_enrichers/__init__.pyindex 78aeb170..2e365ba6 100644--- a/flowsint-enrichers/src/flowsint_enrichers/__init__.py+++ b/flowsint-enrichers/src/flowsint_enrichers/__init__.py@@ -2,7 +2,14 @@Flowsint Enrichers - Enricher modules for flowsint"""+# Import registry utilities+from .registry import ENRICHER_REGISTRY, flowsint_enricher, load_all_enrichers+__version__ = "0.1.0"__author__ = "dextmorgn <[email protected]>"-__all__ = []\ No newline at end of file+__all__ = [+ "ENRICHER_REGISTRY",+ "flowsint_enricher",+ "load_all_enrichers",+]\ No newline at end of filediff --git a/flowsint-enrichers/src/flowsint_enrichers/asn/to_cidrs.py b/flowsint-enrichers/src/flowsint_enrichers/asn/to_cidrs.pyindex e759a9e3..48912388 100644--- a/flowsint-enrichers/src/flowsint_enrichers/asn/to_cidrs.py+++ b/flowsint-enrichers/src/flowsint_enrichers/asn/to_cidrs.py@@ -2,6 +2,7 @@ import jsonimport osfrom typing import List, Dict, Any, Union, Optionalfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_core.core.graph_db import Neo4jConnectionfrom flowsint_types.cidr import CIDRfrom flowsint_types.asn import ASN@@ -10,6 +11,7 @@ from flowsint_core.core.logger import Loggerfrom tools.network.asnmap import AsnmapTool+@flowsint_enricherclass AsnToCidrsEnricher(Enricher):"""[ASNMAP] Takes an ASN and returns its corresponding CIDRs."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/cidr/to_ips.py b/flowsint-enrichers/src/flowsint_enrichers/cidr/to_ips.pyindex 19de4ba2..5fe28444 100644--- a/flowsint-enrichers/src/flowsint_enrichers/cidr/to_ips.py+++ b/flowsint-enrichers/src/flowsint_enrichers/cidr/to_ips.py@@ -1,6 +1,7 @@import osfrom typing import Any, Dict, List, Optional, Unionfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_core.core.graph_db import Neo4jConnectionfrom flowsint_types.cidr import CIDRfrom flowsint_types.ip import Ip@@ -8,6 +9,7 @@ from flowsint_core.core.logger import Loggerfrom tools.network.mapcidr import MapcidrTool+@flowsint_enricherclass CidrToIpsEnricher(Enricher):"""[MAPCIDR] Takes a CIDR and returns its corresponding IP addresses."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/crypto/to_nfts.py b/flowsint-enrichers/src/flowsint_enrichers/crypto/to_nfts.pyindex 5b89e710..6ae8d386 100644--- a/flowsint-enrichers/src/flowsint_enrichers/crypto/to_nfts.py+++ b/flowsint-enrichers/src/flowsint_enrichers/crypto/to_nfts.py@@ -2,6 +2,7 @@ import osfrom typing import List, Dict, Any, Optional, Unionimport requestsfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.wallet import CryptoWallet, CryptoNFTfrom flowsint_core.core.logger import Loggerfrom flowsint_core.core.graph_db import Neo4jConnection@@ -10,6 +11,7 @@ from dotenv import load_dotenvload_dotenv()+@flowsint_enricherclass CryptoWalletAddressToNFTs(Enricher):"""[ETHERSCAN] Resolve NFTs for a wallet address (ETH)."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/crypto/to_transactions.py b/flowsint-enrichers/src/flowsint_enrichers/crypto/to_transactions.pyindex 51b72340..b98573d2 100644--- a/flowsint-enrichers/src/flowsint_enrichers/crypto/to_transactions.py+++ b/flowsint-enrichers/src/flowsint_enrichers/crypto/to_transactions.py@@ -5,6 +5,7 @@ import requests.exceptionsfrom datetime import datetimefrom dotenv import load_dotenvfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.wallet import CryptoWallet, CryptoWalletTransactionfrom flowsint_core.core.graph_db import Neo4jConnectionfrom flowsint_core.core.logger import Logger@@ -16,6 +17,7 @@ def wei_to_eth(wei_str):return int(wei_str) / 10**18+@flowsint_enricherclass CryptoWalletAddressToTransactions(Enricher):"""[ETHERSCAN] Resolve transactions for a wallet address (ETH)."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_asn.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_asn.pyindex c86e32e4..7bb8b50d 100644--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_asn.py+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_asn.py@@ -3,6 +3,7 @@ import osimport socketfrom typing import Any, Dict, List, Optional, Unionfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_core.core.graph_db import Neo4jConnectionfrom flowsint_types.domain import Domainfrom flowsint_types.asn import ASN@@ -11,6 +12,7 @@ from flowsint_core.core.logger import Loggerfrom tools.network.asnmap import AsnmapTool+@flowsint_enricherclass DomainToAsnEnricher(Enricher):"""[ASNMAP] Takes a domain and returns its corresponding ASN."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_dummy_ip.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_dummy_ip.pynew file mode 100644index 00000000..82deaa88--- /dev/null+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_dummy_ip.py@@ -0,0 +1,84 @@+from typing import List+from flowsint_core.core.logger import Logger+from flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricher+from flowsint_types.ip import Ip+from flowsint_types.domain import Domain+import random+++@flowsint_enricher+class DomainToDummyIpEnricher(Enricher):+ """+ TEST TRANSFORM: Generate dummy ips for testing SSE incremental updates.+ """++ InputType = Domain+ OutputType = Ip++ @classmethod+ def name(cls) -> str:+ return "domain_to_dummy_ip"++ @classmethod+ def category(cls) -> str:+ return "Domain"++ @classmethod+ def key(cls) -> str:+ return "domain"++ @classmethod+ def documentation(cls) -> str:+ """Return formatted markdown documentation for the test enricher."""+ return """+ """++ async def scan(self, data: List[InputType]) -> List[OutputType]:+ """ """+ results: List[OutputType] = []+ # Dummy ips+ ips = [+ "192.168.12.34",+ "10.42.77.19",+ "172.16.5.201",+ "192.168.0.242",+ "10.0.23.88",+ "172.20.14.9",+ "192.168.56.101",+ "10.11.99.3",+ "172.31.222.17",+ "192.168.100.45",+ "10.25.60.7",+ "172.18.44.91",+ "192.168.77.12",+ "10.90.4.156",+ "172.26.200.66",+ ]++ for domain in data:+ # select a random ip+ address = random.choice(ips)+ ip = Ip(address=address)+ results.append(ip)+ # Log domain creation+ Logger.info(+ self.sketch_id,+ {"message": f"Found dummy IP {ip.address} for domain {domain.domain}."},+ )+ return results++ def postprocess(+ self, results: List[OutputType], original_input: List[InputType]+ ) -> List[OutputType]:+ """ """+ for domain, ip in zip(original_input, results):+ self.create_node(domain)+ self.create_node(ip)+ self.create_relationship(domain, ip, "RESOLVES_TO")++ return results+++InputType = DomainToDummyIpEnricher.InputType+OutputType = DomainToDummyIpEnricher.OutputTypediff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_dummy_ip_2.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_dummy_ip_2.pynew file mode 100644index 00000000..aa37cc1d--- /dev/null+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_dummy_ip_2.py@@ -0,0 +1,84 @@+from typing import List+from flowsint_core.core.logger import Logger+from flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricher+from flowsint_types.ip import Ip+from flowsint_types.domain import Domain+import random+++@flowsint_enricher+class DomainToDummyIp2Enricher(Enricher):+ """+ TEST TRANSFORM: Generate dummy ips for testing SSE incremental updates.+ """++ InputType = Domain+ OutputType = Ip++ @classmethod+ def name(cls) -> str:+ return "domain_to_dummy_ip_2"++ @classmethod+ def category(cls) -> str:+ return "Domain"++ @classmethod+ def key(cls) -> str:+ return "domain"++ @classmethod+ def documentation(cls) -> str:+ """Return formatted markdown documentation for the test enricher."""+ return """+ """++ async def scan(self, data: List[InputType]) -> List[OutputType]:+ """ """+ results: List[OutputType] = []+ # Dummy ips+ ips = [+ "192.168.12.34",+ "10.42.77.19",+ "172.16.5.201",+ "192.168.0.242",+ "10.0.23.88",+ "172.20.14.9",+ "192.168.56.101",+ "10.11.99.3",+ "172.31.222.17",+ "192.168.100.45",+ "10.25.60.7",+ "172.18.44.91",+ "192.168.77.12",+ "10.90.4.156",+ "172.26.200.66",+ ]++ for domain in data:+ # select a random ip+ address = random.choice(ips)+ ip = Ip(address=address)+ results.append(ip)+ # Log domain creation+ Logger.info(+ self.sketch_id,+ {"message": f"Found dummy IP {ip.address} for domain {domain.domain}."},+ )+ return results++ def postprocess(+ self, results: List[OutputType], original_input: List[InputType]+ ) -> List[OutputType]:+ """ """+ for domain, ip in zip(original_input, results):+ self.create_node(domain)+ self.create_node(ip)+ self.create_relationship(domain, ip, "RESOLVES_TO")++ return results+++InputType = DomainToDummyIp2Enricher.InputType+OutputType = DomainToDummyIp2Enricher.OutputTypediff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_history.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_history.pyindex 2744215c..837c92d9 100644--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_history.py+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_history.py@@ -3,6 +3,7 @@ import reimport jsonfrom typing import Any, List, Union, Dict, Set, Optionalfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.domain import Domainfrom flowsint_types.individual import Individualfrom flowsint_types.organization import Organization@@ -17,6 +18,7 @@ from dotenv import load_dotenvload_dotenv()+@flowsint_enricherclass DomainToHistoryEnricher(Enricher):"""[WHOXY] Takes a domain and returns history infos about it (history, organization, owners, emails, etc.)."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_ip.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_ip.pyindex 0ff5d87a..d583333e 100644--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_ip.py+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_ip.py@@ -2,10 +2,12 @@ import socketfrom typing import Listfrom flowsint_core.core.logger import Loggerfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.domain import Domainfrom flowsint_types.ip import Ip+@flowsint_enricherclass ResolveEnricher(Enricher):"""Resolve domain names to IP addresses."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_root_domain.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_root_domain.pyindex 360635ec..889c6436 100644--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_root_domain.py+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_root_domain.py@@ -1,10 +1,12 @@from typing import List, Unionfrom flowsint_enrichers.utils import is_valid_domain, get_root_domainfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.domain import Domainfrom flowsint_core.core.logger import Logger+@flowsint_enricherclass DomainToRootDomain(Enricher):"""Subdomain to root domain."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_subdomains.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_subdomains.pyindex 08d0495e..e730f3b9 100644--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_subdomains.py+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_subdomains.py@@ -1,12 +1,14 @@import requestsfrom typing import List, Unionfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.domain import Domainfrom flowsint_core.utils import is_valid_domainfrom flowsint_core.core.logger import Loggerfrom tools.network.subfinder import SubfinderTool+@flowsint_enricherclass SubdomainEnricher(Enricher):"""Enricher to find subdomains associated with a domain."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_website.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_website.pyindex 9dab4d68..73dffb9f 100644--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_website.py+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_website.py@@ -2,11 +2,13 @@ from typing import List, Unionimport requestsfrom flowsint_core.utils import is_valid_domainfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.domain import Domainfrom flowsint_types.website import Websitefrom flowsint_core.core.logger import Logger+@flowsint_enricherclass DomainToWebsiteEnricher(Enricher):"""From domain to website."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_whois.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_whois.pyindex 59dcdb1b..4ab71001 100644--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_whois.py+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_whois.py@@ -1,6 +1,7 @@from typing import Listimport whoisfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.domain import Domainfrom flowsint_types.whois import Whoisfrom flowsint_types.email import Email@@ -8,6 +9,7 @@ from flowsint_types.organization import Organizationfrom flowsint_core.core.logger import Logger+@flowsint_enricherclass WhoisEnricher(Enricher):"""Scan for WHOIS information of a domain."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/email/to_domains.py b/flowsint-enrichers/src/flowsint_enrichers/email/to_domains.pyindex 5d4e2af6..12f4cb6d 100644--- a/flowsint-enrichers/src/flowsint_enrichers/email/to_domains.py+++ b/flowsint-enrichers/src/flowsint_enrichers/email/to_domains.py@@ -2,6 +2,7 @@ import osimport refrom typing import Any, List, Dict, Set, Optionalfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.domain import Domainfrom flowsint_types.individual import Individualfrom flowsint_types.email import Email@@ -17,6 +18,7 @@ load_dotenv()WHOXY_API_KEY = os.getenv("WHOXY_API_KEY")+@flowsint_enricherclass EmailToDomainsEnricher(Enricher):"""[WHOXY] Takes an email and returns the domains it registered."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/email/to_gravatar.py b/flowsint-enrichers/src/flowsint_enrichers/email/to_gravatar.pyindex 205090f2..4ddfb96d 100644--- a/flowsint-enrichers/src/flowsint_enrichers/email/to_gravatar.py+++ b/flowsint-enrichers/src/flowsint_enrichers/email/to_gravatar.py@@ -2,11 +2,13 @@ import hashlibfrom typing import List, Unionimport requestsfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_core.core.logger import Loggerfrom flowsint_types.email import Emailfrom flowsint_types.gravatar import Gravatar+@flowsint_enricherclass EmailToGravatarEnricher(Enricher):"""From md5 hash of email to gravatar."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/email/to_leaks.py b/flowsint-enrichers/src/flowsint_enrichers/email/to_leaks.pyindex 175f019b..2e8d504f 100644--- a/flowsint-enrichers/src/flowsint_enrichers/email/to_leaks.py+++ b/flowsint-enrichers/src/flowsint_enrichers/email/to_leaks.py@@ -3,6 +3,7 @@ from typing import Any, Dict, List, Optional, Unionimport requestsfrom urllib.parse import urljoinfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_core.core.logger import Loggerfrom flowsint_types.email import Emailfrom flowsint_types.breach import Breach@@ -15,6 +16,7 @@ load_dotenv()HIBP_API_KEY = os.getenv("HIBP_API_KEY")+@flowsint_enricherclass EmailToBreachesEnricher(Enricher):"""[HIBPWNED] Get the breaches the email might be invovled in."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/individual/to_domains.py b/flowsint-enrichers/src/flowsint_enrichers/individual/to_domains.pyindex 7c3c0a07..70840c91 100644--- a/flowsint-enrichers/src/flowsint_enrichers/individual/to_domains.py+++ b/flowsint-enrichers/src/flowsint_enrichers/individual/to_domains.py@@ -2,6 +2,7 @@ import osimport refrom typing import Any, List, Union, Dict, Set, Optionalfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.domain import Domainfrom flowsint_types.organization import Organizationfrom flowsint_types.individual import Individual@@ -16,6 +17,7 @@ load_dotenv()WHOXY_API_KEY = os.getenv("WHOXY_API_KEY")+@flowsint_enricherclass IndividualToDomainsEnricher(Enricher):"""[WHOXY] Takes an individual and returns the domains it registered."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/individual/to_org.py b/flowsint-enrichers/src/flowsint_enrichers/individual/to_org.pyindex 033c1f7a..199a2a58 100644--- a/flowsint-enrichers/src/flowsint_enrichers/individual/to_org.py+++ b/flowsint-enrichers/src/flowsint_enrichers/individual/to_org.py@@ -1,12 +1,14 @@import requestsfrom typing import List, Dict, Union, Setfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.organization import Organizationfrom flowsint_types.individual import Individualfrom flowsint_core.core.logger import Loggerfrom tools.organizations.sirene import SireneTool+@flowsint_enricherclass IndividualToOrgEnricher(Enricher):"""[SIRENE] Find organization from a person with data from SIRENE (France only)."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/ip/to_asn.py b/flowsint-enrichers/src/flowsint_enrichers/ip/to_asn.pyindex 612eed82..21b3b2aa 100644--- a/flowsint-enrichers/src/flowsint_enrichers/ip/to_asn.py+++ b/flowsint-enrichers/src/flowsint_enrichers/ip/to_asn.py@@ -2,6 +2,7 @@ import jsonimport osfrom typing import Any, Dict, List, Optional, Unionfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_core.core.graph_db import Neo4jConnectionfrom flowsint_types.ip import Ipfrom flowsint_types.asn import ASN@@ -10,6 +11,7 @@ from flowsint_core.core.logger import Loggerfrom tools.network.asnmap import AsnmapTool+@flowsint_enricherclass IpToAsnEnricher(Enricher):"""[ASNMAP] Takes an IP address and returns its corresponding ASN."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/ip/to_domain.py b/flowsint-enrichers/src/flowsint_enrichers/ip/to_domain.pyindex 11dd3631..dd2d63c7 100644--- a/flowsint-enrichers/src/flowsint_enrichers/ip/to_domain.py+++ b/flowsint-enrichers/src/flowsint_enrichers/ip/to_domain.py@@ -3,10 +3,12 @@ import requestsfrom typing import Listfrom flowsint_core.core.logger import Loggerfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.domain import Domainfrom flowsint_types.ip import Ip+@flowsint_enricherclass ReverseResolveEnricher(Enricher):"""Resolve IP addresses to domain names using PTR, Certificate Transparency and optional API calls."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/ip/to_dummy_domains.py b/flowsint-enrichers/src/flowsint_enrichers/ip/to_dummy_domains.pyindex ed049154..46f47f27 100644--- a/flowsint-enrichers/src/flowsint_enrichers/ip/to_dummy_domains.py+++ b/flowsint-enrichers/src/flowsint_enrichers/ip/to_dummy_domains.py@@ -2,10 +2,12 @@ import asynciofrom typing import Listfrom flowsint_core.core.logger import Loggerfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.ip import Ipfrom flowsint_types.domain import Domain+@flowsint_enricherclass IpToDummyDomainsEnricher(Enricher):"""TEST TRANSFORM: Generate dummy domains for testing SSE incremental updates.@@ -84,12 +86,12 @@ class IpToDummyDomainsEnricher(Enricher):ip_normalized = ip_obj.address.replace(".", "-")# Generate 20 dummy domains per IP- for i in range(20):+ for i in range(530):service = services[i % len(services)]tld = tlds[i % len(tlds)]# Create dummy domain with pattern: {service}-{ip}.{tld}- dummy_domain = f"{service}-{ip_normalized}.{tld}"+ dummy_domain = f"{service}-{i}-{ip_normalized}.{tld}"domain_obj = Domain(domain=dummy_domain)results.append(domain_obj)diff --git a/flowsint-enrichers/src/flowsint_enrichers/ip/to_infos.py b/flowsint-enrichers/src/flowsint_enrichers/ip/to_infos.pyindex 9a1cb574..9edded47 100644--- a/flowsint-enrichers/src/flowsint_enrichers/ip/to_infos.py+++ b/flowsint-enrichers/src/flowsint_enrichers/ip/to_infos.py@@ -1,9 +1,11 @@import requestsfrom typing import List, Dict, Anyfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.ip import Ip+@flowsint_enricherclass IpToInfosEnricher(Enricher):"""[ip-api.com] Get information data for IP addresses."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/ip/to_ports.py b/flowsint-enrichers/src/flowsint_enrichers/ip/to_ports.pyindex a657fd1a..9a8caaa8 100644--- a/flowsint-enrichers/src/flowsint_enrichers/ip/to_ports.py+++ b/flowsint-enrichers/src/flowsint_enrichers/ip/to_ports.py@@ -1,5 +1,6 @@from typing import Any, Dict, List, Optional, Unionfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_core.core.graph_db import Neo4jConnectionfrom flowsint_types.ip import Ipfrom flowsint_types.port import Port@@ -8,6 +9,7 @@ from flowsint_core.core.logger import Loggerfrom tools.network.naabu import NaabuTool+@flowsint_enricherclass IpToPortsEnricher(Enricher):"""[NAABU] Performs port scanning on IP addresses to discover open ports and services."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/n8n/connector.py b/flowsint-enrichers/src/flowsint_enrichers/n8n/connector.pyindex 5879f472..7fbaf740 100644--- a/flowsint-enrichers/src/flowsint_enrichers/n8n/connector.py+++ b/flowsint-enrichers/src/flowsint_enrichers/n8n/connector.py@@ -2,10 +2,12 @@ import jsonimport aiohttpfrom typing import List, Dict, Any, Optionalfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_core.core.logger import Loggerfrom flowsint_core.core.graph_db import Neo4jConnection+@flowsint_enricherclass N8nConnector(Enricher):"""Connect to your custom n8n workflows to process data through webhooks.diff --git a/flowsint-enrichers/src/flowsint_enrichers/organization/to_asn.py b/flowsint-enrichers/src/flowsint_enrichers/organization/to_asn.pyindex 94f50c27..b49cd8ab 100644--- a/flowsint-enrichers/src/flowsint_enrichers/organization/to_asn.py+++ b/flowsint-enrichers/src/flowsint_enrichers/organization/to_asn.py@@ -1,6 +1,7 @@import osfrom typing import List, Dict, Any, Union, Optionalfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_core.core.graph_db import Neo4jConnectionfrom flowsint_types.organization import Organizationfrom flowsint_types.asn import ASN@@ -8,6 +9,7 @@ from flowsint_core.core.logger import Loggerfrom tools.network.asnmap import AsnmapTool+@flowsint_enricherclass OrgToAsnEnricher(Enricher):"""Takes an organization and returns its corresponding ASN."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/organization/to_domains.py b/flowsint-enrichers/src/flowsint_enrichers/organization/to_domains.pyindex 4bb7246b..bc6e8bad 100644--- a/flowsint-enrichers/src/flowsint_enrichers/organization/to_domains.py+++ b/flowsint-enrichers/src/flowsint_enrichers/organization/to_domains.py@@ -2,6 +2,7 @@ import osimport refrom typing import Any, List, Dict, Set, Optionalfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types import Email, Phonefrom flowsint_types.domain import Domainfrom flowsint_types.organization import Organization@@ -16,6 +17,7 @@ from dotenv import load_dotenvload_dotenv()+@flowsint_enricherclass OrgToDomainsEnricher(Enricher):"""[WHOXY] Takes an organization and returns the domains it registered."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/organization/to_infos.py b/flowsint-enrichers/src/flowsint_enrichers/organization/to_infos.pyindex 1c64cfc7..62bf7012 100644--- a/flowsint-enrichers/src/flowsint_enrichers/organization/to_infos.py+++ b/flowsint-enrichers/src/flowsint_enrichers/organization/to_infos.py@@ -1,5 +1,6 @@from typing import List, Dict, Any, Unionfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.organization import Organizationfrom flowsint_types.address import Locationfrom flowsint_types.individual import Individual@@ -7,6 +8,7 @@ from flowsint_core.core.logger import Loggerfrom tools.organizations.sirene import SireneTool+@flowsint_enricherclass OrgToInfosEnricher(Enricher):"""Enrich Organization with data from SIRENE (France only)."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/phone/to_infos.py b/flowsint-enrichers/src/flowsint_enrichers/phone/to_infos.pyindex 76f228b3..d7202947 100644--- a/flowsint-enrichers/src/flowsint_enrichers/phone/to_infos.py+++ b/flowsint-enrichers/src/flowsint_enrichers/phone/to_infos.py@@ -1,12 +1,14 @@import asynciofrom typing import Dict, Any, List, Unionfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_core.utils import is_valid_numberfrom flowsint_core.core.logger import Loggerfrom flowsint_types.phone import Phoneimport httpx+@flowsint_enricherclass IgnorantEnricher(Enricher):# Define types as class attributes - base class handles schema generation automaticallydiff --git a/flowsint-enrichers/src/flowsint_enrichers/registry.py b/flowsint-enrichers/src/flowsint_enrichers/registry.pynew file mode 100644index 00000000..702cbad0--- /dev/null+++ b/flowsint-enrichers/src/flowsint_enrichers/registry.py@@ -0,0 +1,217 @@+"""+This module provides an automatic enricher registration system using decorators.+All Enricher subclasses can be decorated with @flowsint_enricher to automatically+register themselves in the global ENRICHER_REGISTRY.++Auto-discovery is performed by calling load_all_enrichers() which imports all modules+in the flowsint_enrichers package, triggering the @flowsint_enricher decorators.+"""++import inspect+import importlib+import os+import sys+from typing import Dict, Optional, Type, List, Any, TypeVar+from flowsint_core.core.enricher_base import Enricher+++E = TypeVar("E", bound=Enricher)+++class EnricherRegistry:+ """+ Global registry for Flowsint enrichers.+ Stores mappings: enricher name -> enricher class+ """++ def __init__(self):+ self._enrichers: Dict[str, Type[Enricher]] = {}++ def register(self, enricher_class: Type[E]) -> Type[E]:+ """+ Register an enricher in the registry.+ Args:+ enricher_class: The enricher class to register+ Returns:+ The same class (for use as a decorator)+ """+ self._enrichers[enricher_class.name()] = enricher_class+ return enricher_class++ def enricher_exists(self, name: str) -> bool:+ return name in self._enrichers++ def get_enricher(+ self, name: str, sketch_id: str, scan_id: str, **kwargs+ ) -> Enricher:+ if name not in self._enrichers:+ raise Exception(f"Enricher '{name}' not found")+ return self._enrichers[name](sketch_id=sketch_id, scan_id=scan_id, **kwargs)++ def _create_enricher_metadata(self, enricher: Type[Enricher]) -> Dict[str, str]:+ """Helper method to create enricher metadata dictionary."""+ return {+ "class_name": enricher.__name__,+ "name": enricher.name(),+ "module": enricher.__module__,+ "description": enricher.__doc__,+ "documentation": inspect.cleandoc(enricher.documentation()),+ "category": enricher.category(),+ "inputs": enricher.input_schema(),+ "outputs": enricher.output_schema(),+ "params": {},+ "params_schema": enricher.get_params_schema(),+ "required_params": enricher.required_params(),+ "icon": enricher.icon(),+ }++ def list(+ self, exclude: Optional[List[str]] = None, wobbly_type: Optional[bool] = False+ ) -> List[Dict[str, Any]]:+ if exclude is None:+ exclude = []+ return [+ {+ **self._create_enricher_metadata(enricher),+ "wobblyType": wobbly_type,+ }+ for enricher in self._enrichers.values()+ if enricher.name() not in exclude+ ]++ def list_by_categories(self) -> Dict[str, List[Dict[str, str]]]:+ enrichers_by_category = {}+ for _, enricher in self._enrichers.items():+ category = enricher.category()+ if category not in enrichers_by_category:+ enrichers_by_category[category] = []+ enrichers_by_category[category].append(+ self._create_enricher_metadata(enricher)+ )+ return enrichers_by_category++ def list_by_input_type(+ self, input_type: str, exclude: Optional[List[str]] = []+ ) -> List[Dict[str, str]]:+ input_type_lower = input_type.lower()++ if input_type_lower == "any":+ return [+ self._create_enricher_metadata(enricher)+ for enricher in self._enrichers.values()+ if enricher.name() not in exclude+ ]++ return [+ self._create_enricher_metadata(enricher)+ for enricher in self._enrichers.values()+ if enricher.input_schema()["type"].lower() in ["any", input_type_lower]+ and enricher.name() not in exclude+ ]+++# Global enricher registry instance+ENRICHER_REGISTRY = EnricherRegistry()+++def flowsint_enricher(cls: Type[E]) -> Type[E]:+ """+ Decorator to automatically register an Enricher subclass.++ Usage:+ @flowsint_enricher+ class SubdomainEnricher(Enricher):+ InputType = Domain+ OutputType = Domain+ ...++ This will automatically register the enricher in ENRICHER_REGISTRY with+ the name returned by enricher_class.name()++ Args:+ cls: The enricher class to register++ Returns:+ The same class (unmodified)+ """+ return ENRICHER_REGISTRY.register(cls)+++# Auto-discovery cache+_enrichers_loaded = False+++def load_all_enrichers() -> None:+ """+ Automatically discover and import all enricher modules in the flowsint_enrichers package.++ This function uses importlib to dynamically import all Python modules in the+ flowsint_enrichers package (including subdirectories), which triggers the+ @flowsint_enricher decorators and registers all enrichers in ENRICHER_REGISTRY.++ Features:+ - Only imports modules once (cached via _enrichers_loaded flag)+ - Ignores private modules (starting with _)+ - Only imports .py files+ - Recursively scans subdirectories (domain/, ip/, etc.) even without __init__.py+ - Uses os.walk for comprehensive discovery++ This function is idempotent - calling it multiple times is safe and efficient.+ """+ global _enrichers_loaded++ # Early return if already loaded+ if _enrichers_loaded:+ return++ try:+ # Get the flowsint_enrichers package+ import flowsint_enrichers+ package = flowsint_enrichers+ except ImportError:+ # Package not available - skip auto-discovery+ print("Warning: flowsint_enrichers package not found", file=sys.stderr)+ _enrichers_loaded = True+ return++ # Get the root path of the enrichers package+ package_path = package.__path__[0]+ package_name = package.__name__++ # Walk through all directories and files+ for root, dirs, files in os.walk(package_path):+ # Skip __pycache__ directories+ dirs[:] = [d for d in dirs if not d.startswith('__pycache__')]++ # Calculate the module prefix for this directory+ rel_path = os.path.relpath(root, package_path)+ if rel_path == '.':+ module_prefix = package_name+ else:+ # Convert path separators to dots for module names+ module_parts = rel_path.replace(os.sep, '.')+ module_prefix = f"{package_name}.{module_parts}"++ # Import all Python files in this directory+ for filename in files:+ # Skip non-Python files and private files+ if not filename.endswith('.py') or filename.startswith('_'):+ continue++ # Build the full module name+ module_basename = filename[:-3] # Remove .py extension+ module_name = f"{module_prefix}.{module_basename}"++ # Skip if already imported+ if module_name in sys.modules:+ continue++ # Import the module to trigger @flowsint_enricher decorators+ try:+ importlib.import_module(module_name)+ except Exception as e:+ # Log but don't fail - some modules might have optional dependencies+ print(f"Warning: Failed to import {module_name}: {e}", file=sys.stderr)++ # Mark as loaded+ _enrichers_loaded = Truediff --git a/flowsint-enrichers/src/flowsint_enrichers/social/to_maigret.py b/flowsint-enrichers/src/flowsint_enrichers/social/to_maigret.pyindex fa7b8f31..04a7a4de 100644--- a/flowsint-enrichers/src/flowsint_enrichers/social/to_maigret.py+++ b/flowsint-enrichers/src/flowsint_enrichers/social/to_maigret.py@@ -3,6 +3,7 @@ import subprocessfrom pathlib import Pathfrom typing import Listfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types import Usernamefrom flowsint_types.social_account import SocialAccountfrom flowsint_core.core.logger import Logger@@ -10,6 +11,7 @@ from flowsint_core.core.logger import Loggerfalse_positives = ["LeagueOfLegends"]+@flowsint_enricherclass MaigretEnricher(Enricher):"""[MAIGRET] Scans usernames for associated social accounts using Maigret."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/social/to_sherlock.py b/flowsint-enrichers/src/flowsint_enrichers/social/to_sherlock.pyindex c6223261..00adda0d 100644--- a/flowsint-enrichers/src/flowsint_enrichers/social/to_sherlock.py+++ b/flowsint-enrichers/src/flowsint_enrichers/social/to_sherlock.py@@ -4,9 +4,11 @@ from typing import List, Unionfrom flowsint_core.utils import is_valid_usernamefrom flowsint_types import SocialAccount, Usernamefrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_core.core.logger import Logger+@flowsint_enricherclass SherlockEnricher(Enricher):"""[SHERLOCK] Scans the usernames for associated social accounts using Sherlock."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/website/to_crawler.py b/flowsint-enrichers/src/flowsint_enrichers/website/to_crawler.pyindex f9df507c..763efc6c 100644--- a/flowsint-enrichers/src/flowsint_enrichers/website/to_crawler.py+++ b/flowsint-enrichers/src/flowsint_enrichers/website/to_crawler.py@@ -1,6 +1,7 @@from typing import List, Optional, Unionfrom urllib.parse import urlparsefrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.website import Websitefrom flowsint_types.phone import Phonefrom flowsint_types.email import Email@@ -15,6 +16,7 @@ class ReturnType(BaseModel):phones: Optional[Phone]+@flowsint_enricherclass WebsiteToCrawler(Enricher):"""From website to crawler."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/website/to_domain.py b/flowsint-enrichers/src/flowsint_enrichers/website/to_domain.pyindex 0abe1e7b..6944ad7d 100644--- a/flowsint-enrichers/src/flowsint_enrichers/website/to_domain.py+++ b/flowsint-enrichers/src/flowsint_enrichers/website/to_domain.py@@ -1,11 +1,13 @@from typing import List, Unionfrom urllib.parse import urlparsefrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.website import Websitefrom flowsint_types.domain import Domainfrom flowsint_core.core.logger import Logger+@flowsint_enricherclass WebsiteToDomainEnricher(Enricher):"""From website to domain."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/website/to_links.py b/flowsint-enrichers/src/flowsint_enrichers/website/to_links.pyindex 6edca6b9..4a4822d5 100644--- a/flowsint-enrichers/src/flowsint_enrichers/website/to_links.py+++ b/flowsint-enrichers/src/flowsint_enrichers/website/to_links.py@@ -1,12 +1,14 @@from typing import List, Unionfrom urllib.parse import urlparsefrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.website import Websitefrom flowsint_types.domain import Domainfrom flowsint_core.core.logger import Loggerfrom reconspread import Crawler+@flowsint_enricherclass WebsiteToLinks(Enricher):"""From website to spread crawler that extracts domains and internal/external links."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/website/to_text.py b/flowsint-enrichers/src/flowsint_enrichers/website/to_text.pyindex 5aded9f9..9f8d8f7e 100644--- a/flowsint-enrichers/src/flowsint_enrichers/website/to_text.py+++ b/flowsint-enrichers/src/flowsint_enrichers/website/to_text.py@@ -1,11 +1,13 @@from typing import List, Unionfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.phrase import Phrasefrom flowsint_types.website import Websiteimport requestsfrom bs4 import BeautifulSoup+@flowsint_enricherclass WebsiteToText(Enricher):"""Extracts the texts in a webpage."""diff --git a/flowsint-enrichers/src/flowsint_enrichers/website/to_webtrackers.py b/flowsint-enrichers/src/flowsint_enrichers/website/to_webtrackers.pyindex 6a40e8e7..90920c0d 100644--- a/flowsint-enrichers/src/flowsint_enrichers/website/to_webtrackers.py+++ b/flowsint-enrichers/src/flowsint_enrichers/website/to_webtrackers.py@@ -1,5 +1,6 @@from typing import List, Dict, Any, Union, Optionalfrom flowsint_core.core.enricher_base import Enricher+from flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.website import Websitefrom flowsint_types.web_tracker import WebTrackerfrom flowsint_core.core.logger import Logger@@ -8,6 +9,7 @@ from flowsint_core.core.vault import VaultProtocolfrom recontrack import TrackingCodeExtractor+@flowsint_enricherclass WebsiteToWebtrackersEnricher(Enricher):"""From website to webtrackers."""diff --git a/flowsint-enrichers/tests/enrichers/__init__.py b/flowsint-enrichers/tests/enrichers/__init__.pyindex e69de29b..a0a08f66 100644--- a/flowsint-enrichers/tests/enrichers/__init__.py+++ b/flowsint-enrichers/tests/enrichers/__init__.py@@ -0,0 +1,3 @@+from flowsint_enrichers import load_all_enrichers++load_all_enrichers()diff --git a/flowsint-enrichers/tests/enrichers/crypto/__init__.py b/flowsint-enrichers/tests/enrichers/crypto/__init__.pydeleted file mode 100644index e69de29b..00000000diff --git a/flowsint-enrichers/tests/enrichers/crypto/wallet_to_nfts.py b/flowsint-enrichers/tests/enrichers/crypto/wallet_to_nfts.pydeleted file mode 100644index 7f1dc065..00000000--- a/flowsint-enrichers/tests/enrichers/crypto/wallet_to_nfts.py+++ /dev/null@@ -1,73 +0,0 @@-from flowsint_enrichers.crypto.wallet_to_nfts import CryptoWalletAddressToNFTs-from flowsint_types.wallet import CryptoWallet, CryptoNFT-from pydantic import HttpUrl--enricher = CryptoWalletAddressToNFTs("sketch_123", "scan_123")---def test_wallet_address_to_transactions_name():- assert enricher.name() == "wallet_to_nfts"---def test_wallet_address_to_transactions_category():- assert enricher.category() == "crypto"---def test_wallet_address_to_transactions_key():- assert enricher.key() == "address"---def test_preprocess_with_string():- input_data = ["0x742d35Cc6634C0532925a3b844Bc454e4438f44e"]- result = enricher.preprocess(input_data)- assert len(result) == 1- assert isinstance(result[0], CryptoWallet)- assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"---def test_preprocess_with_dict():- input_data = [{"address": "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"}]- result = enricher.preprocess(input_data)- assert len(result) == 1- assert isinstance(result[0], CryptoWallet)- assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"---def test_preprocess_with_wallet_object():- wallet = CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")- input_data = [wallet]- result = enricher.preprocess(input_data)- assert len(result) == 1- assert isinstance(result[0], CryptoWallet)- assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"---def test_scan_mocked_transactions(monkeypatch):- # Mock the _get_transactions method- def mock_get_nfts(address):- return [- CryptoNFT(- wallet=CryptoWallet(- address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e"- ),- contract_address="0x123",- token_id="1",- collection_name="Test Collection",- metadata_url="https://example.com/metadata.json",- image_url="https://example.com/image.png",- name="Test NFT",- )- ]-- monkeypatch.setattr(enricher, "_get_nfts", mock_get_nfts)-- input_data = [CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")]- result = enricher.scan(input_data)-- assert len(result) == 1- assert len(result[0]) == 1- assert result[0][0].contract_address == "0x123"- assert result[0][0].collection_name == "Test Collection"- assert result[0][0].metadata_url == HttpUrl("https://example.com/metadata.json")- assert result[0][0].image_url == HttpUrl("https://example.com/image.png")- assert result[0][0].name == "Test NFT"diff --git a/flowsint-enrichers/tests/enrichers/crypto/wallet_to_transactions.py b/flowsint-enrichers/tests/enrichers/crypto/wallet_to_transactions.pydeleted file mode 100644index aa9134cc..00000000--- a/flowsint-enrichers/tests/enrichers/crypto/wallet_to_transactions.py+++ /dev/null@@ -1,106 +0,0 @@-import pytest-from flowsint_enrichers.crypto.wallet_to_transactions import (- CryptoWalletAddressToTransactions,-)-from flowsint_types.wallet import CryptoWallet, CryptoWalletTransaction--enricher = CryptoWalletAddressToTransactions(- "sketch_123",- "scan_123",- params={"ETHERSCAN_API_KEY": "ta-clef-api"},-)---def test_wallet_address_to_transactions_name():- assert enricher.name() == "wallet_to_transactions"---def test_wallet_address_to_transactions_category():- assert enricher.category() == "CryptoCryptoWallet"---def test_wallet_address_to_transactions_key():- assert enricher.key() == "address"---def test_preprocess_with_string():- input_data = ["0x742d35Cc6634C0532925a3b844Bc454e4438f44e"]- result = enricher.preprocess(input_data)- assert len(result) == 1- assert isinstance(result[0], CryptoWallet)- assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"---def test_preprocess_with_dict():- input_data = [{"address": "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"}]- result = enricher.preprocess(input_data)- assert len(result) == 1- assert isinstance(result[0], CryptoWallet)- assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"---def test_preprocess_with_wallet_object():- wallet = CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")- input_data = [wallet]- result = enricher.preprocess(input_data)- assert len(result) == 1- assert isinstance(result[0], CryptoWallet)- assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"---async def test_scan_mocked_transactions(monkeypatch):- # Mock the _get_transactions method - note it takes address and api_key parameters- async def mock_get_transactions(address, api_key):- return [- CryptoWalletTransaction(- hash="0x123",- source=CryptoWallet(- address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e"- ),- target=CryptoWallet(address="0x456"),- value=1.0, # 1 ETH- timestamp="1234567890",- block_number="12345",- block_hash="0xabc",- nonce="1",- transaction_index="0",- gas="21000",- gas_price="20000000000",- gas_used="21000",- cumulative_gas_used="21000",- input="0x",- contract_address=None,- )- ]-- monkeypatch.setattr(enricher, "_get_transactions", mock_get_transactions)-- input_data = [CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")]- result = await enricher.scan(input_data)-- assert len(result) == 1- assert len(result[0]) == 1- assert result[0][0].hash == "0x123"- assert result[0][0].source.address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"- assert result[0][0].target.address == "0x456"- assert result[0][0].value == 1.0- assert result[0][0].timestamp == "1234567890"---def test_enricher_requires_api_key():- """Test that the enricher validates required ETHERSCAN_API_KEY parameter at construction"""- with pytest.raises(- ValueError, match="Enricher wallet_to_transactions received invalid params"- ):- CryptoWalletAddressToTransactions("sketch_123", "scan_123", params={})---def test_enricher_with_invalid_api_key_type():- """Test that the enricher validates parameter types"""- with pytest.raises(- ValueError, match="Enricher wallet_to_transactions received invalid params"- ):- CryptoWalletAddressToTransactions(- "sketch_123", "scan_123", params={"ETHERSCAN_API_KEY": 123}- )diff --git a/flowsint-enrichers/tests/enrichers/domain/__init__.py b/flowsint-enrichers/tests/enrichers/domain/__init__.pydeleted file mode 100644index e69de29b..00000000diff --git a/flowsint-enrichers/tests/enrichers/domain/resolve.py b/flowsint-enrichers/tests/enrichers/domain/resolve.pydeleted file mode 100644index ce614366..00000000--- a/flowsint-enrichers/tests/enrichers/domain/resolve.py+++ /dev/null@@ -1,153 +0,0 @@-from flowsint_enrichers.domain.to_ip import ResolveEnricher-from flowsint_types.domain import Domain-from flowsint_types.ip import Ip-from typing import List-import pytest--enricher = ResolveEnricher("sketch_123", "scan_123")---def test_preprocess_valid_domains():- domains = [- Domain(domain="example.com"),- Domain(domain="example2.com"),- ]- result = enricher.preprocess(domains)-- result_domains = [d.domain for d in result]- expected_domains = [d.domain for d in domains]-- assert result_domains == expected_domains---def test_unprocessed_valid_domains():- domains = [- "example.com",- "example2.com",- ]- result = enricher.preprocess(domains)- result_domains = [d for d in result]- expected_domains = [Domain(domain=d) for d in domains]- assert result_domains == expected_domains---def test_preprocess_invalid_domains():- domains = [- Domain(domain="example.com"),- Domain(domain="invalid_domain"),- Domain(domain="example.org"),- ]- result = enricher.preprocess(domains)-- result_domains = [d.domain for d in result]- assert "example.com" in result_domains- assert "example.org" in result_domains- assert "invalid_domain" not in result_domains---def test_preprocess_multiple_formats():- domains = [- {"domain": "example.com"},- {"invalid_key": "example.io"},- Domain(domain="example.org"),- "example.org",- ]- result = enricher.preprocess(domains)-- result_domains = [d.domain for d in result]- assert "example.com" in result_domains- assert "example.org" in result_domains- assert "invalid_domain" not in result_domains- assert "example.io" not in result_domains---async def test_scan_returns_ip(monkeypatch):- # on crée une fonction mock qui retourne une IP- def mock_gethostbyname(domain):- return "12.23.34.45"-- monkeypatch.setattr("socket.gethostbyname", mock_gethostbyname)-- input_data = [Domain(domain="example.com")]- output = await enricher.execute(input_data)- print(output)- assert isinstance(output, list)- assert output[0].address == "12.23.34.45"---def test_schemas():- input_schema = enricher.input_schema()- output_schema = enricher.output_schema()-- # Test the structure and key properties rather than exact match- assert input_schema["type"] == "Domain"- assert isinstance(input_schema["properties"], list)- input_property_names = [prop["name"] for prop in input_schema["properties"]]- assert "domain" in input_property_names-- assert output_schema["type"] == "Ip"- assert isinstance(output_schema["properties"], list)- output_property_names = [prop["name"] for prop in output_schema["properties"]]- assert "address" in output_property_names---class TestResolveInputOutputTypes:- """Test the InputType/OutputType functionality for ResolveEnricher"""-- def test_input_output_types_are_defined(self):- """Test that InputType and OutputType are properly defined"""- assert hasattr(ResolveEnricher, "InputType")- assert hasattr(ResolveEnricher, "OutputType")- assert ResolveEnricher.InputType == List[Domain]- assert ResolveEnricher.OutputType == List[Ip]-- def test_schemas_use_generate_methods(self):- """Test that schema methods use the new generate methods"""- # These should work without error- input_schema = ResolveEnricher.generate_input_schema()- output_schema = ResolveEnricher.generate_output_schema()-- assert isinstance(input_schema, dict)- assert isinstance(output_schema, dict)- assert input_schema["type"] == "Domain"- assert output_schema["type"] == "Ip"-- def test_schema_methods_return_same_as_generate_methods(self):- """Test that input_schema() and output_schema() return the same as generate methods"""- assert ResolveEnricher.input_schema() == ResolveEnricher.generate_input_schema()- assert ResolveEnricher.output_schema() == ResolveEnricher.generate_output_schema()-- def test_input_schema_properties(self):- """Test input schema has expected properties"""- schema = ResolveEnricher.input_schema()-- properties = schema["properties"]- property_names = [p["name"] for p in properties]-- # Domain should have these properties- assert "domain" in property_names-- def test_output_schema_properties(self):- """Test output schema has expected properties"""- schema = ResolveEnricher.output_schema()-- properties = schema["properties"]- property_names = [p["name"] for p in properties]-- # Ip should have these properties- assert "address" in property_names-- def test_type_accessibility_from_instance(self):- """Test that types are accessible from enricher instance"""- enricher_instance = ResolveEnricher("test", "test")-- assert enricher_instance.InputType == List[Domain]- assert enricher_instance.OutputType == List[Ip]-- # Should be able to generate schemas from instance- input_schema = enricher_instance.generate_input_schema()- output_schema = enricher_instance.generate_output_schema()-- assert input_schema["type"] == "Domain"- assert output_schema["type"] == "Ip"diff --git a/flowsint-enrichers/tests/enrichers/domain/subdomains.py b/flowsint-enrichers/tests/enrichers/domain/subdomains.pydeleted file mode 100644index ba547e09..00000000--- a/flowsint-enrichers/tests/enrichers/domain/subdomains.py+++ /dev/null@@ -1,95 +0,0 @@-from flowsint_enrichers.domains.subdomains import SubdomainEnricher-from flowsint_types.domain import Domain, Domain--enricher = SubdomainEnricher("sketch_123", "scan_123")---def test_preprocess_valid_domains():- domains = [- Domain(domain="example.com"),- Domain(domain="example2.com"),- ]- result = enricher.preprocess(domains)-- result_domains = [d.domain for d in result]- expected_domains = [d.domain for d in domains]-- assert result_domains == expected_domains---def test_unprocessed_valid_domains():- domains = [- "example.com",- "example2.com",- ]- result = enricher.preprocess(domains)- result_domains = [d for d in result]- expected_domains = [Domain(domain=d) for d in domains]- assert result_domains == expected_domains---def test_preprocess_invalid_domains():- domains = [- Domain(domain="example.com"),- Domain(domain="invalid_domain"),- Domain(domain="example.org"),- ]- result = enricher.preprocess(domains)-- result_domains = [d.domain for d in result]- assert "example.com" in result_domains- assert "example.org" in result_domains- assert "invalid_domain" not in result_domains---def test_preprocess_multiple_formats():- domains = [- {"domain": "example.com"},- {"invalid_key": "example.io"},- Domain(domain="example.org"),- "example.org",- ]- result = enricher.preprocess(domains)-- result_domains = [d.domain for d in result]- assert "example.com" in result_domains- assert "example.org" in result_domains- assert "invalid_domain" not in result_domains- assert "example.io" not in result_domains---def test_scan_extracts_subdomains(monkeypatch):- mock_response = [- {"name_value": "mail.example.com\nwww.example.com"},- {"name_value": "api.example.com"},- {"name_value": "invalid_domain"}, # devrait être ignoré- ]-- class MockRequestsResponse:- def __init__(self, json_data):- self._json_data = json_data- self.status_code = 200-- def json(self):- return self._json_data-- @property- def ok(self):- return True-- def mock_get(url, timeout):- assert "example.com" in url- return MockRequestsResponse(mock_response)-- # Patch la requête réseau dans le module enricher- monkeypatch.setattr("requests.get", mock_get)-- input_data = [Domain(domain="example.com")]- domains = enricher.execute(input_data)- assert isinstance(domains, list)- for sub in domains:- print(sub)- assert isinstance(sub, Domain)- expected = sorted(["mail.example.com", "www.example.com", "api.example.com"])- print(domains)- # assert domains[0].subdomains == expecteddiff --git a/flowsint-enrichers/tests/enrichers/domain/test_to_asn_preprocess.py b/flowsint-enrichers/tests/enrichers/domain/test_to_asn_preprocess.pydeleted file mode 100644index 1789dd1f..00000000--- a/flowsint-enrichers/tests/enrichers/domain/test_to_asn_preprocess.py+++ /dev/null@@ -1,98 +0,0 @@-import pytest-from flowsint_enrichers.domain.to_asn import DomainToAsnEnricher-from flowsint_types.domain import Domain---def enricher():- """Create enricher instance for testing."""- return DomainToAsnEnricher(sketch_id="test_sketch", scan_id="test_scan")---def test_preprocess_valid_objects(enricher):- """Test preprocess with valid Domain objects."""- inputs = [- Domain(domain="example.com"),- Domain(domain="google.com"),- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Domain) for item in result)- assert result[0].domain == "example.com"- assert result[1].domain == "google.com"---def test_preprocess_strings(enricher):- """Test preprocess with string inputs (converted via primary field)."""- inputs = ["example.com", "google.com"]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Domain) for item in result)- assert result[0].domain == "example.com"- assert result[1].domain == "google.com"---def test_preprocess_dicts(enricher):- """Test preprocess with dict inputs."""- inputs = [- {"domain": "example.com"},- {"domain": "google.com"},- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Domain) for item in result)- assert result[0].domain == "example.com"- assert result[1].domain == "google.com"---def test_preprocess_invalid_filtered(enricher):- """Test that invalid items are filtered out."""- inputs = [- Domain(domain="example.com"),- "not a domain", # Invalid- Domain(domain="google.com"),- ]- result = enricher.preprocess(inputs)-- # Only valid items should remain- assert len(result) == 2- assert result[0].domain == "example.com"- assert result[1].domain == "google.com"---def test_preprocess_mixed_formats(enricher):- """Test preprocess with mixed input formats."""- inputs = [- Domain(domain="example.com"), # Object- "google.com", # String- {"domain": "github.com"}, # Dict- {"invalid_key": "wrong.com"}, # Invalid dict (wrong key)- ]- result = enricher.preprocess(inputs)-- assert len(result) == 3 # Should have 3 valid items- assert all(isinstance(item, Domain) for item in result)- assert result[0].domain == "example.com"- assert result[1].domain == "google.com"- assert result[2].domain == "github.com"---def test_preprocess_empty_list(enricher):- """Test preprocess with empty input."""- result = enricher.preprocess([])- assert result == []---def test_preprocess_all_invalid(enricher):- """Test preprocess when all items are invalid."""- inputs = [- "not a domain",- "domain with spaces",- "",- ]- result = enricher.preprocess(inputs)-- assert len(result) == 0diff --git a/flowsint-enrichers/tests/enrichers/domain/test_to_ip_preprocess.py b/flowsint-enrichers/tests/enrichers/domain/test_to_ip_preprocess.pydeleted file mode 100644index a9b96f96..00000000--- a/flowsint-enrichers/tests/enrichers/domain/test_to_ip_preprocess.py+++ /dev/null@@ -1,83 +0,0 @@-import pytest-from flowsint_enrichers.domain.to_ip import ResolveEnricher-from flowsint_types.domain import Domain---def enricher():- """Create enricher instance for testing."""- return ResolveEnricher(sketch_id="test_sketch", scan_id="test_scan")---def test_preprocess_valid_objects(enricher):- """Test preprocess with valid Domain objects."""- inputs = [- Domain(domain="example.com"),- Domain(domain="google.com"),- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Domain) for item in result)- assert result[0].domain == "example.com"- assert result[1].domain == "google.com"---def test_preprocess_strings(enricher):- """Test preprocess with string inputs (converted via primary field)."""- inputs = ["example.com", "google.com"]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Domain) for item in result)- assert result[0].domain == "example.com"- assert result[1].domain == "google.com"---def test_preprocess_dicts(enricher):- """Test preprocess with dict inputs."""- inputs = [- {"domain": "example.com"},- {"domain": "google.com"},- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Domain) for item in result)- assert result[0].domain == "example.com"- assert result[1].domain == "google.com"---def test_preprocess_invalid_filtered(enricher):- """Test that invalid items are filtered out."""- inputs = [- Domain(domain="example.com"),- "invalid domain with spaces", # Invalid- Domain(domain="google.com"),- ]- result = enricher.preprocess(inputs)-- # Only valid items should remain- assert len(result) == 2- assert result[0].domain == "example.com"- assert result[1].domain == "google.com"---def test_preprocess_mixed_formats(enricher):- """Test preprocess with mixed input formats."""- inputs = [- Domain(domain="example.com"), # Object- "google.com", # String- {"domain": "github.com"}, # Dict- {"invalid_key": "wrong.com"}, # Invalid dict (wrong key)- ]- result = enricher.preprocess(inputs)-- assert len(result) == 3 # Should have 3 valid items- assert all(isinstance(item, Domain) for item in result)---def test_preprocess_empty_list(enricher):- """Test preprocess with empty input."""- result = enricher.preprocess([])- assert result == []diff --git a/flowsint-enrichers/tests/enrichers/domain/test_to_root_domain_preprocess.py b/flowsint-enrichers/tests/enrichers/domain/test_to_root_domain_preprocess.pydeleted file mode 100644index ae7db5d7..00000000--- a/flowsint-enrichers/tests/enrichers/domain/test_to_root_domain_preprocess.py+++ /dev/null@@ -1,83 +0,0 @@-import pytest-from flowsint_enrichers.domain.to_root_domain import DomainToRootDomain-from flowsint_types.domain import Domain---def enricher():- """Create enricher instance for testing."""- return DomainToRootDomain(sketch_id="test_sketch", scan_id="test_scan")---def test_preprocess_valid_objects(enricher):- """Test preprocess with valid Domain objects."""- inputs = [- Domain(domain="subdomain.example.com"),- Domain(domain="www.google.com"),- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Domain) for item in result)- assert result[0].domain == "subdomain.example.com"- assert result[1].domain == "www.google.com"---def test_preprocess_strings(enricher):- """Test preprocess with string inputs (converted via primary field)."""- inputs = ["subdomain.example.com", "www.google.com"]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Domain) for item in result)- assert result[0].domain == "subdomain.example.com"- assert result[1].domain == "www.google.com"---def test_preprocess_dicts(enricher):- """Test preprocess with dict inputs."""- inputs = [- {"domain": "subdomain.example.com"},- {"domain": "www.google.com"},- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Domain) for item in result)- assert result[0].domain == "subdomain.example.com"- assert result[1].domain == "www.google.com"---def test_preprocess_invalid_filtered(enricher):- """Test that invalid items are filtered out."""- inputs = [- Domain(domain="example.com"),- "not a domain", # Invalid- Domain(domain="google.com"),- ]- result = enricher.preprocess(inputs)-- # Only valid items should remain- assert len(result) == 2- assert result[0].domain == "example.com"- assert result[1].domain == "google.com"---def test_preprocess_mixed_formats(enricher):- """Test preprocess with mixed input formats."""- inputs = [- Domain(domain="subdomain.example.com"), # Object- "www.google.com", # String- {"domain": "api.github.com"}, # Dict- {"invalid_key": "wrong.com"}, # Invalid dict (wrong key)- ]- result = enricher.preprocess(inputs)-- assert len(result) == 3 # Should have 3 valid items- assert all(isinstance(item, Domain) for item in result)---def test_preprocess_empty_list(enricher):- """Test preprocess with empty input."""- result = enricher.preprocess([])- assert result == []diff --git a/flowsint-enrichers/tests/enrichers/domain/test_to_website.py b/flowsint-enrichers/tests/enrichers/domain/test_to_website.pydeleted file mode 100644index 9fe94124..00000000--- a/flowsint-enrichers/tests/enrichers/domain/test_to_website.py+++ /dev/null@@ -1,113 +0,0 @@-import pytest-from flowsint_enrichers.domain.to_website import DomainToWebsiteEnricher-from flowsint_types.domain import Domain-from flowsint_types.website import Website-from pydantic import HttpUrl---def enricher():- """Create enricher instance for testing."""- return DomainToWebsiteEnricher(sketch_id="test_sketch", scan_id="test_scan")---def test_website_object_serialization(enricher):- """Test that Website objects with HttpUrl can be properly serialized."""- # Create a website object with HttpUrl- domain = Domain(domain="example.com")- website = Website(- url=HttpUrl("https://example.com"),- domain=domain,- active=True,- redirects=[HttpUrl("https://www.example.com")]- )-- # Verify the Website object is created correctly- assert isinstance(website.url, HttpUrl)- assert len(website.redirects) == 1- assert isinstance(website.redirects[0], HttpUrl)-- # Test model_dump with mode="json" properly serializes HttpUrl to strings- dumped = website.model_dump(mode="json")- assert isinstance(dumped["url"], str)- assert dumped["url"] == "https://example.com/"- assert isinstance(dumped["redirects"][0], str)- assert dumped["redirects"][0] == "https://www.example.com/"---def test_create_node_with_website(enricher):- """Test that create_node can handle Website objects with HttpUrl fields."""- domain = Domain(domain="example.com")- website = Website(- url=HttpUrl("https://example.com"),- domain=domain,- active=True- )-- # This should not raise an error about unsupported types- # The create_node method should properly serialize HttpUrl to string- try:- enricher.create_node(website)- # If we get here without exception, the serialization works- success = True- except Exception as e:- success = False- error_msg = str(e)-- # Note: This test won't actually create a node if Neo4j isn't connected,- # but it will verify the serialization doesn't fail- assert success or "Neo4j" in error_msg or "connection" in error_msg.lower()---def test_create_relationship_with_website(enricher):- """Test that create_relationship can handle Website objects with HttpUrl fields."""- domain = Domain(domain="example.com")- website = Website(- url=HttpUrl("https://example.com"),- domain=domain,- active=True- )-- # This should not raise an error about unsupported HttpUrl types- # The create_relationship method should properly serialize HttpUrl to string- try:- enricher.create_relationship(domain, website, "HAS_WEBSITE")- # If we get here without exception, the serialization works- success = True- except Exception as e:- success = False- error_msg = str(e)-- # Note: This test won't actually create a relationship if Neo4j isn't connected,- # but it will verify the serialization doesn't fail- assert success or "Neo4j" in error_msg or "connection" in error_msg.lower()---def test_extract_primary_value_with_httpurl(enricher):- """Test that _extract_primary_value properly serializes HttpUrl."""- website = Website(- url=HttpUrl("https://example.com"),- domain=Domain(domain="example.com"),- active=True- )-- # Extract the primary value (url field)- primary_value = enricher._extract_primary_value(website)-- # Should be a string, not an HttpUrl object- assert isinstance(primary_value, str)- assert primary_value == "https://example.com/"---def test_preprocess_valid_domains(enricher):- """Test preprocess with valid Domain objects."""- inputs = [- Domain(domain="example.com"),- Domain(domain="google.com"),- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Domain) for item in result)- assert result[0].domain == "example.com"- assert result[1].domain == "google.com"diff --git a/flowsint-enrichers/tests/enrichers/domain/to_history.py b/flowsint-enrichers/tests/enrichers/domain/to_history.pydeleted file mode 100644index 57853ede..00000000--- a/flowsint-enrichers/tests/enrichers/domain/to_history.py+++ /dev/null@@ -1,393 +0,0 @@-import pytest-import json-import os-from unittest.mock import Mock-from flowsint_enrichers.domain.to_history import DomainToHistoryEnricher-from flowsint_types.domain import Domain---class MockNeo4jConn:- def __init__(self):- self.nodes_created = []- self.relationships_created = []-- def create_node(self, label, key, value, **kwargs):- node_info = {"label": label, "key": key, "value": value, **kwargs}- self.nodes_created.append(node_info)-- def create_relationship(- self,- from_label,- from_key,- from_value,- to_label,- to_key,- to_value,- relationship_type,- ):- rel_info = {- "from": f"{from_label}:{from_value}",- "to": f"{to_label}:{to_value}",- "type": relationship_type,- }- self.relationships_created.append(rel_info)-- def query(self, query, params):- """Mock query method to avoid errors."""- pass---class MockEnricher(DomainToHistoryEnricher):- def __init__(self):- self.sketch_id = "test_sketch_123"- self.neo4j_conn = MockNeo4jConn()- self._extracted_data = []- self._extracted_individuals = []-- def log_graph_message(self, message):- """Mock log_graph_message method."""- pass---def enricher():- """Create an enricher instance for testing."""- enricher = MockEnricher()- return enricher---def test_data():- """Load test data from data.json."""- current_dir = os.path.dirname(os.path.abspath(__file__))- data_file = os.path.join(current_dir, "..", "..", "test_data", "data.json")- with open(data_file, "r") as f:- return json.load(f)---def test_preprocess_valid_domains(enricher):- """Test preprocessing with valid domains."""- domains = [- Domain(domain="example.com"),- Domain(domain="example2.com"),- ]- result = enricher.preprocess(domains)-- result_domains = [d.domain for d in result]- expected_domains = [d.domain for d in domains]-- assert result_domains == expected_domains---def test_preprocess_string_domains(enricher):- """Test preprocessing with string domains."""- domains = ["example.com", "example2.com"]- result = enricher.preprocess(domains)-- assert len(result) == 2- assert all(isinstance(d, Domain) for d in result)- assert result[0].domain == "example.com"- assert result[1].domain == "example2.com"---def test_preprocess_dict_domains(enricher):- """Test preprocessing with dict domains."""- domains = [{"domain": "example.com"}, {"domain": "example2.com"}]- result = enricher.preprocess(domains)-- assert len(result) == 2- assert all(isinstance(d, Domain) for d in result)- assert result[0].domain == "example.com"- assert result[1].domain == "example2.com"---def test_preprocess_invalid_domains(enricher):- """Test preprocessing with invalid domains."""- domains = [- Domain(domain="example.com"),- Domain(domain="invalid_domain"),- Domain(domain="example.org"),- ]- result = enricher.preprocess(domains)-- result_domains = [d.domain for d in result]- assert "example.com" in result_domains- assert "example.org" in result_domains- assert "invalid_domain" not in result_domains---def test_is_redacted(enricher):- """Test the __is_redacted method."""- # Should be redacted- assert enricher._DomainToHistoryEnricher__is_redacted("REDACTED FOR PRIVACY")- assert enricher._DomainToHistoryEnricher__is_redacted("redacted for privacy")- assert enricher._DomainToHistoryEnricher__is_redacted("Some text with PRIVACY in it")-- # Should NOT be redacted- assert not enricher._DomainToHistoryEnricher__is_redacted("JOHN DOE")- assert not enricher._DomainToHistoryEnricher__is_redacted("[email protected]")- assert not enricher._DomainToHistoryEnricher__is_redacted("123 JOHN STREET")- assert not enricher._DomainToHistoryEnricher__is_redacted("DOE CITY")---def test_has_non_redacted_info(enricher):- """Test the __has_non_redacted_info method."""- # Contact with valid information- valid_contact = {- "full_name": "JOHN DOE",- "email_address": "[email protected], [email protected]",- "phone_number": "+123456789",- "mailing_address": "123 JOHN STREET",- "city_name": "DOE CITY",- "zip_code": "12345",- "country_name": "United States",- }- assert enricher._DomainToHistoryEnricher__has_non_redacted_info(valid_contact)-- # Contact with all redacted information- redacted_contact = {- "full_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- }- assert not enricher._DomainToHistoryEnricher__has_non_redacted_info(redacted_contact)-- # Empty contact- assert not enricher._DomainToHistoryEnricher__has_non_redacted_info({})---def test_extract_individual_from_contact(enricher):- """Test the __extract_individual_from_contact method."""- # Valid contact- valid_contact = {- "full_name": "JOHN DOE",- "email_address": "[email protected], [email protected]",- "phone_number": "+123456789",- "mailing_address": "123 JOHN STREET",- "city_name": "DOE CITY",- "zip_code": "12345",- "country_name": "United States",- }-- individual = enricher._DomainToHistoryEnricher__extract_individual_from_contact(- valid_contact, "REGISTRANT"- )-- assert individual is not None- assert individual.first_name == "MARC"- assert individual.last_name == "DESCOLLONGES"- assert individual.full_name == "JOHN DOE"- assert len(individual.email_addresses) == 2- assert "[email protected]" in individual.email_addresses- assert "[email protected]" in individual.email_addresses- assert individual.phone_numbers == ["+123456789"]---def test_extract_individual_redacted_name(enricher):- """Test that individuals with redacted names are skipped."""- redacted_contact = {- "full_name": "REDACTED FOR PRIVACY",- "email_address": "[email protected]",- "phone_number": "+1234567890",- }-- individual = enricher._DomainToHistoryEnricher__extract_individual_from_contact(- redacted_contact, "REGISTRANT"- )- assert individual is None---def test_is_valid_email(enricher):- """Test the __is_valid_email method."""- # Valid emails- assert enricher._DomainToHistoryEnricher__is_valid_email("[email protected]")- assert enricher._DomainToHistoryEnricher__is_valid_email("[email protected]")- assert enricher._DomainToHistoryEnricher__is_valid_email("[email protected]")-- # Invalid emails- assert not enricher._DomainToHistoryEnricher__is_valid_email("invalid-email")- assert not enricher._DomainToHistoryEnricher__is_valid_email("@example.com")- assert not enricher._DomainToHistoryEnricher__is_valid_email("test@")- assert not enricher._DomainToHistoryEnricher__is_valid_email("")---def test_extract_physical_address(enricher):- """Test the __extract_physical_address method."""- # Valid address- valid_contact = {- "mailing_address": "123 JOHN STREET",- "city_name": "DOE CITY",- "zip_code": "12345",- "country_name": "United States",- }-- address = enricher._DomainToHistoryEnricher__extract_physical_address(valid_contact)-- assert address is not None- assert address.address == "123 JOHN STREET"- assert address.city == "DOE CITY"- assert address.zip == "12345"- assert address.country == "United States"-- # Address with redacted parts- redacted_contact = {- "mailing_address": "123 JOHN STREET",- "city_name": "REDACTED FOR PRIVACY",- "zip_code": "12345",- "country_name": "United States",- }-- address = enricher._DomainToHistoryEnricher__extract_physical_address(- redacted_contact- )- assert address is None---async def test_scan_with_test_data(enricher, test_data, monkeypatch):- """Test the scan method with test data."""-- # Mock the __get_infos_from_whoxy method to return test data- def mock_get_infos(domain):- if domain == "epios.com":- return test_data- return {}-- monkeypatch.setattr(- enricher, "_DomainToHistoryEnricher__get_infos_from_whoxy", mock_get_infos- )-- # Test with epios.com domain- input_domains = [Domain(domain="epios.com")]- results = await enricher.scan(input_domains)-- # Should find the domain (one for each WHOIS record)- assert len(results) == 16 # 16 WHOIS records in the test data- assert all(r.domain == "epios.com" for r in results)-- # Should have extracted data- assert len(enricher._extracted_data) == 16-- # Should have extracted individuals- assert (- len(enricher._extracted_individuals) > 0- ), "Should have extracted some individuals"-- # Check that JOHN DOE is in the extracted individuals- marc_found = False- marc_individuals = []-- for individual_info in enricher._extracted_individuals:- individual = individual_info["individual"]- if "JOHN DOE" in individual.full_name:- marc_found = True- marc_individuals.append(individual_info)- print(- f"Found MARC: {individual.full_name} ({individual_info['contact_type']})"- )- print(f" Emails: {individual.email_addresses}")- print(f" Phones: {individual.phone_numbers}")-- assert marc_found, "JOHN DOE should be found in the extracted individuals"- assert (- len(marc_individuals) > 0- ), f"Expected to find JOHN DOE, but found {len(marc_individuals)} instances"-- # Print summary of all extracted individuals- print(f"\n=== Summary of extracted individuals ===")- for individual_info in enricher._extracted_individuals:- individual = individual_info["individual"]- print(- f"- {individual.full_name} ({individual_info['contact_type']}) for {individual_info['domain_name']}"- )- if individual.email_addresses:- print(f" Emails: {individual.email_addresses}")- if individual.phone_numbers:- print(f" Phones: {individual.phone_numbers}")---def test_postprocess_creates_nodes_and_relationships(enricher, test_data, monkeypatch):- """Test that postprocess creates the expected nodes and relationships."""-- # Mock the __get_infos_from_whoxy method- def mock_get_infos(domain):- if domain == "epios.com":- return test_data- return {}-- monkeypatch.setattr(- enricher, "_DomainToHistoryEnricher__get_infos_from_whoxy", mock_get_infos- )-- # First run scan to populate _extracted_data and _extracted_individuals- import asyncio-- loop = asyncio.new_event_loop()- asyncio.set_event_loop(loop)-- try:- input_domains = [Domain(domain="epios.com")]- results = loop.run_until_complete(enricher.scan(input_domains))-- # Debug: Check what individuals were extracted- print(- f"\n=== DEBUG: _extracted_individuals has {len(enricher._extracted_individuals)} individuals ==="- )- for i, individual_info in enumerate(enricher._extracted_individuals):- individual = individual_info["individual"]- print(- f"Individual {i+1}: {individual.full_name} ({individual_info['contact_type']}) for {individual_info['domain_name']}"- )- if individual.email_addresses:- print(f" Emails: {individual.email_addresses}")- if individual.phone_numbers:- print(f" Phones: {individual.phone_numbers}")-- # Now run postprocess- print(f"\n=== Running postprocess ===")- enricher.postprocess(results, input_domains)-- # Debug: Check what happened during postprocess- print(f"=== Postprocess completed ===")- print(f"Nodes created: {len(enricher.neo4j_conn.nodes_created)}")- print(f"Relationships created: {len(enricher.neo4j_conn.relationships_created)}")-- # Should have created some nodes- assert len(enricher.neo4j_conn.nodes_created) > 0-- # Should have created some relationships- assert len(enricher.neo4j_conn.relationships_created) > 0-- # Check for domain node- domain_nodes = [- n for n in enricher.neo4j_conn.nodes_created if n["label"] == "domain"- ]- assert len(domain_nodes) > 0-- # Check for individual nodes (should include JOHN DOE)- individual_nodes = [- n for n in enricher.neo4j_conn.nodes_created if n["label"] == "individual"- ]- assert len(individual_nodes) > 0-- # Check that JOHN DOE is in the individual nodes- marc_nodes = [n for n in individual_nodes if "JOHN DOE" in n["value"]]- assert (- len(marc_nodes) > 0- ), "JOHN DOE should be in the individual nodes"-- finally:- loop.close()---def test_schemas(enricher):- """Test that the enricher has the expected schemas."""- input_schema = enricher.input_schema()- output_schema = enricher.output_schema()-- assert input_schema is not None- assert output_schema is not Nonediff --git a/flowsint-enrichers/tests/enrichers/domain/whois.py b/flowsint-enrichers/tests/enrichers/domain/whois.pydeleted file mode 100644index 094ac804..00000000--- a/flowsint-enrichers/tests/enrichers/domain/whois.py+++ /dev/null@@ -1,103 +0,0 @@-from flowsint_enrichers.domains.whois import WhoisEnricher-from flowsint_types.domain import Domain--enricher = WhoisEnricher("sketch_123", "scan_123")---def test_preprocess_valid_domains():- domains = [- Domain(domain="example.com"),- Domain(domain="example2.com"),- ]- result = enricher.preprocess(domains)-- result_domains = [d.domain for d in result]- expected_domains = [d.domain for d in domains]-- assert result_domains == expected_domains---def test_unprocessed_valid_domains():- domains = [- "example.com",- "example2.com",- ]- result = enricher.preprocess(domains)- result_domains = [d for d in result]- expected_domains = [Domain(domain=d) for d in domains]- assert result_domains == expected_domains---def test_preprocess_invalid_domains():- domains = [- Domain(domain="example.com"),- Domain(domain="invalid_domain"),- Domain(domain="example.org"),- ]- result = enricher.preprocess(domains)-- result_domains = [d.domain for d in result]- assert "example.com" in result_domains- assert "example.org" in result_domains- assert "invalid_domain" not in result_domains---def test_preprocess_multiple_formats():- domains = [- {"domain": "example.com"},- {"invalid_key": "example.io"},- Domain(domain="example.org"),- "example.org",- ]- result = enricher.preprocess(domains)-- result_domains = [d.domain for d in result]- assert "example.com" in result_domains- assert "example.org" in result_domains- assert "invalid_domain" not in result_domains- assert "example.io" not in result_domains---def test_scan_returns_whois_objects(monkeypatch):- # Patch `whois.whois` to avoid real network call- mock_whois = lambda domain: {- "registrar": "MockRegistrar",- "org": "MockOrg",- "city": "MockCity",- "country": "MockCountry",- "emails": ["[email protected]"],- "creation_date": "2020-01-01",- "expiration_date": "2030-01-01",- }-- monkeypatch.setattr("whois.whois", mock_whois)-- input_data = [Domain(domain="example.com")]- output = enricher.execute(input_data)- assert isinstance(output, list)- assert isinstance(output[0], Domain)- assert output[0].whois.org == "MockOrg"- assert output[0].whois.email.email == "[email protected]"---def test_schemas():- input_schema = enricher.input_schema()- output_schema = enricher.output_schema()- assert input_schema == {- "type": "Domain",- "properties": [- {"name": "domain", "type": "string"},- {"name": "subdomains", "type": "array | null"},- {"name": "ips", "type": "array | null"},- {"name": "whois", "type": "Whois | null"},- ],- }- assert output_schema == {- "type": "Domain",- "properties": [- {"name": "domain", "type": "string"},- {"name": "subdomains", "type": "array | null"},- {"name": "ips", "type": "array | null"},- {"name": "whois", "type": "Whois | null"},- ],- }diff --git a/flowsint-enrichers/tests/enrichers/email/__init__.py b/flowsint-enrichers/tests/enrichers/email/__init__.pydeleted file mode 100644index e69de29b..00000000diff --git a/flowsint-enrichers/tests/enrichers/email/test_to_gravatar_preprocess.py b/flowsint-enrichers/tests/enrichers/email/test_to_gravatar_preprocess.pydeleted file mode 100644index be447c24..00000000--- a/flowsint-enrichers/tests/enrichers/email/test_to_gravatar_preprocess.py+++ /dev/null@@ -1,98 +0,0 @@-import pytest-from flowsint_enrichers.email.to_gravatar import EmailToGravatarEnricher-from flowsint_types.email import Email---def enricher():- """Create enricher instance for testing."""- return EmailToGravatarEnricher(sketch_id="test_sketch", scan_id="test_scan")---def test_preprocess_valid_objects(enricher):- """Test preprocess with valid Email objects."""- inputs = [- Email(email="[email protected]"),- Email(email="[email protected]"),- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Email) for item in result)- assert result[0].email == "[email protected]"- assert result[1].email == "[email protected]"---def test_preprocess_strings(enricher):- """Test preprocess with string inputs (converted via primary field)."""- inputs = ["[email protected]", "[email protected]"]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Email) for item in result)- assert result[0].email == "[email protected]"- assert result[1].email == "[email protected]"---def test_preprocess_dicts(enricher):- """Test preprocess with dict inputs."""- inputs = [- {"email": "[email protected]"},- {"email": "[email protected]"},- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Email) for item in result)- assert result[0].email == "[email protected]"- assert result[1].email == "[email protected]"---def test_preprocess_invalid_filtered(enricher):- """Test that invalid items are filtered out."""- inputs = [- Email(email="[email protected]"),- "not-an-email", # Invalid- Email(email="[email protected]"),- ]- result = enricher.preprocess(inputs)-- # Only valid items should remain- assert len(result) == 2- assert result[0].email == "[email protected]"- assert result[1].email == "[email protected]"---def test_preprocess_mixed_formats(enricher):- """Test preprocess with mixed input formats."""- inputs = [- Email(email="[email protected]"), # Object- "[email protected]", # String- {"email": "[email protected]"}, # Dict- {"invalid_key": "[email protected]"}, # Invalid dict (wrong key)- ]- result = enricher.preprocess(inputs)-- assert len(result) == 3 # Should have 3 valid items- assert all(isinstance(item, Email) for item in result)- assert result[0].email == "[email protected]"- assert result[1].email == "[email protected]"- assert result[2].email == "[email protected]"---def test_preprocess_empty_list(enricher):- """Test preprocess with empty input."""- result = enricher.preprocess([])- assert result == []---def test_preprocess_all_invalid(enricher):- """Test preprocess when all items are invalid."""- inputs = [- "not-an-email",- "missing@",- "invalid",- ]- result = enricher.preprocess(inputs)-- assert len(result) == 0diff --git a/flowsint-enrichers/tests/enrichers/email/to_gravatar.py b/flowsint-enrichers/tests/enrichers/email/to_gravatar.pydeleted file mode 100644index 06ab5822..00000000--- a/flowsint-enrichers/tests/enrichers/email/to_gravatar.py+++ /dev/null@@ -1,336 +0,0 @@-import hashlib-from unittest.mock import Mock, patch-from flowsint_enrichers.emails.to_gravatar import EmailToGravatarEnricher-from flowsint_types.email import Email-from flowsint_types.gravatar import Gravatar--enricher = EmailToGravatarEnricher("sketch_123", "scan_123")---class TestEmailToGravatarEnricher:- """Test suite for EmailToGravatarEnricher"""-- def test_name(self):- """Test the enricher name"""- assert EmailToGravatarEnricher.name() == "to_gravatar"-- def test_category(self):- """Test the enricher category"""- assert EmailToGravatarEnricher.category() == "Email"-- def test_key(self):- """Test the enricher key"""- assert EmailToGravatarEnricher.key() == "email"-- def test_input_schema(self):- """Test the input schema generation"""- schema = EmailToGravatarEnricher.input_schema()- assert schema["type"] == "Email"- assert "properties" in schema- # Check that email property is present- email_prop = next(- (prop for prop in schema["properties"] if prop["name"] == "email"), None- )- assert email_prop is not None- assert email_prop["type"] == "string"-- def test_output_schema(self):- """Test the output schema generation"""- schema = EmailToGravatarEnricher.output_schema()- assert schema["type"] == "Gravatar"- assert "properties" in schema- # Check that required properties are present- src_prop = next(- (prop for prop in schema["properties"] if prop["name"] == "src"), None- )- hash_prop = next(- (prop for prop in schema["properties"] if prop["name"] == "hash"), None- )- assert src_prop is not None- assert hash_prop is not None-- def test_preprocess_string_emails(self):- """Test preprocessing with string emails"""- emails = [- "[email protected]",- "[email protected]",- ]- result = enricher.preprocess(emails)- assert len(result) == 2- assert all(isinstance(email, Email) for email in result)- assert result[0].email == "[email protected]"- assert result[1].email == "[email protected]"-- def test_preprocess_dict_emails(self):- """Test preprocessing with dictionary emails"""- emails = [- {"email": "[email protected]"},- {"email": "[email protected]"},- ]- result = enricher.preprocess(emails)- assert len(result) == 2- assert all(isinstance(email, Email) for email in result)- assert result[0].email == "[email protected]"- assert result[1].email == "[email protected]"-- def test_preprocess_email_objects(self):- """Test preprocessing with Email objects"""- emails = [- Email(email="[email protected]"),- Email(email="[email protected]"),- ]- result = enricher.preprocess(emails)- assert len(result) == 2- assert all(isinstance(email, Email) for email in result)- assert result[0].email == "[email protected]"- assert result[1].email == "[email protected]"-- def test_preprocess_mixed_formats(self):- """Test preprocessing with mixed input formats"""- emails = [- "[email protected]",- {"email": "[email protected]"},- Email(email="[email protected]"),- ]- result = enricher.preprocess(emails)- assert len(result) == 3- assert all(isinstance(email, Email) for email in result)- assert result[0].email == "[email protected]"- assert result[1].email == "[email protected]"- assert result[2].email == "[email protected]"-- def test_preprocess_invalid_inputs(self):- """Test preprocessing with invalid inputs"""- emails = [- "not-an-email",- {"invalid_key": "[email protected]"},- {"email": "invalid-email"},- None,- 123,- ]- result = enricher.preprocess(emails)- # The preprocess method doesn't validate email format, it just creates Email objects- # for valid string inputs and dicts with email key- assert len(result) == 2 # "not-an-email" and "invalid-email" are processed- assert result[0].email == "not-an-email"- assert result[1].email == "invalid-email"-- def test_preprocess_empty_list(self):- """Test preprocessing with empty list"""- result = enricher.preprocess([])- assert result == []-- @patch("requests.get")- def test_scan_successful_gravatar(self, mock_get):- """Test successful gravatar retrieval"""- # Mock successful response- mock_response = Mock()- mock_response.status_code = 200- mock_get.return_value = mock_response-- emails = [Email(email="[email protected]")]- result = enricher.scan(emails)-- assert len(result) == 1- assert isinstance(result[0], Gravatar)- assert result[0].hash == hashlib.md5("[email protected]".encode()).hexdigest()- assert "gravatar.com/avatar/" in str(result[0].src)-- @patch("requests.get")- def test_scan_failed_request(self, mock_get):- """Test handling of failed HTTP requests"""- # Mock failed response- mock_response = Mock()- mock_response.status_code = 404- mock_get.return_value = mock_response-- emails = [Email(email="[email protected]")]- result = enricher.scan(emails)-- assert len(result) == 0-- @patch("requests.get")- def test_scan_request_exception(self, mock_get):- """Test handling of request exceptions"""- # Mock exception- mock_get.side_effect = Exception("Network error")-- emails = [Email(email="[email protected]")]- result = enricher.scan(emails)-- assert len(result) == 0-- @patch("requests.get")- def test_scan_multiple_emails(self, mock_get):- """Test scanning multiple emails"""- # Mock successful responses- mock_response = Mock()- mock_response.status_code = 200- mock_get.return_value = mock_response-- emails = [- Email(email="[email protected]"),- Email(email="[email protected]"),- Email(email="[email protected]"),- ]- result = enricher.scan(emails)-- assert len(result) == 3- assert all(isinstance(gravatar, Gravatar) for gravatar in result)- assert mock_get.call_count == 3-- @patch("requests.get")- def test_scan_mixed_success_failure(self, mock_get):- """Test scanning with mixed success and failure"""-- # Mock mixed responses - check the actual URL being called- def side_effect(url, *args, **kwargs):- mock_response = Mock()- # Check if the URL contains the hash for [email protected]- test1_hash = hashlib.md5("[email protected]".encode()).hexdigest()- if test1_hash in url:- mock_response.status_code = 200- else:- mock_response.status_code = 404- return mock_response-- mock_get.side_effect = side_effect-- emails = [- Email(email="[email protected]"),- Email(email="[email protected]"),- ]- result = enricher.scan(emails)-- # Should get 1 result for the first email (success) and 0 for the second (failure)- assert len(result) == 1- assert result[0].hash == hashlib.md5("[email protected]".encode()).hexdigest()-- def test_postprocess_with_neo4j_connection(self):- """Test postprocessing with Neo4j connection"""- # Mock Neo4j connection- mock_neo4j = Mock()- enricher_with_neo4j = EmailToGravatarEnricher(- "sketch_123", "scan_123", neo4j_conn=mock_neo4j- )-- gravatars = [- Gravatar(src="https://www.gravatar.com/avatar/hash1", hash="hash1"),- Gravatar(src="https://www.gravatar.com/avatar/hash2", hash="hash2"),- ]- original_input = [- Email(email="[email protected]"),- Email(email="[email protected]"),- ]-- result = enricher_with_neo4j.postprocess(gravatars, original_input)-- # Verify Neo4j queries were executed- assert mock_neo4j.query.call_count == 2-- # Check that results are returned unchanged- assert result == gravatars-- def test_postprocess_without_neo4j_connection(self):- """Test postprocessing without Neo4j connection"""- gravatars = [- Gravatar(src="https://www.gravatar.com/avatar/hash1", hash="hash1"),- ]- original_input = [Email(email="[email protected]")]-- result = enricher.postprocess(gravatars, original_input)-- # Should return results unchanged- assert result == gravatars-- def test_postprocess_missing_original_input(self):- """Test postprocessing with missing original input"""- gravatars = [- Gravatar(src="https://www.gravatar.com/avatar/hash1", hash="hash1"),- ]- original_input = [] # Empty list-- result = enricher.postprocess(gravatars, original_input)-- # Should handle gracefully and return results- assert result == gravatars-- def test_postprocess_none_original_input(self):- """Test postprocessing with None original input"""- gravatars = [- Gravatar(src="https://www.gravatar.com/avatar/hash1", hash="hash1"),- ]-- # The postprocess method doesn't handle None input properly- # Let's test with an empty list instead- result = enricher.postprocess(gravatars, [])-- # Should handle gracefully and return results- assert result == gravatars-- def test_execute_full_workflow(self):- """Test the complete execute workflow"""- with patch("requests.get") as mock_get:- # Mock successful response- mock_response = Mock()- mock_response.status_code = 200- mock_get.return_value = mock_response-- emails = ["[email protected]"]- result = enricher.execute(emails)-- assert len(result) == 1- assert isinstance(result[0], Gravatar)- assert (- result[0].hash == hashlib.md5("[email protected]".encode()).hexdigest()- )-- def test_execute_with_invalid_input(self):- """Test execute with invalid input"""- emails = ["not-an-email", "also-invalid"]-- with patch("requests.get") as mock_get:- # Mock successful response for any request- mock_response = Mock()- mock_response.status_code = 200- mock_get.return_value = mock_response-- result = enricher.execute(emails)-- # The enricher processes any string as an email, so it will create Email objects- # and attempt to get gravatars for them- assert len(result) == 2- assert all(isinstance(gravatar, Gravatar) for gravatar in result)-- def test_gravatar_hash_calculation(self):- """Test that gravatar hash is calculated correctly"""- email = "[email protected]"- expected_hash = hashlib.md5(email.encode()).hexdigest()-- with patch("requests.get") as mock_get:- mock_response = Mock()- mock_response.status_code = 200- mock_get.return_value = mock_response-- emails = [Email(email=email)]- result = enricher.scan(emails)-- assert len(result) == 1- assert result[0].hash == expected_hash-- def test_gravatar_url_format(self):- """Test that gravatar URL is formatted correctly"""- email = "[email protected]"- expected_hash = hashlib.md5(email.encode()).hexdigest()- expected_url = f"https://www.gravatar.com/avatar/{expected_hash}"-- with patch("requests.get") as mock_get:- mock_response = Mock()- mock_response.status_code = 200- mock_get.return_value = mock_response-- emails = [Email(email=email)]- result = enricher.scan(emails)-- assert len(result) == 1- assert str(result[0].src) == expected_urldiff --git a/flowsint-enrichers/tests/enrichers/email/to_leaks.py b/flowsint-enrichers/tests/enrichers/email/to_leaks.pydeleted file mode 100644index 5a20fd84..00000000--- a/flowsint-enrichers/tests/enrichers/email/to_leaks.py+++ /dev/null@@ -1,175 +0,0 @@-import pytest-from unittest.mock import patch, MagicMock-from flowsint_enrichers.emails.to_leaks import EmailToBreachesEnricher-from flowsint_types.email import Email-from flowsint_types.breach import Breach--enricher = EmailToBreachesEnricher("sketch_123", "scan_123")---def test_enricher_name():- assert EmailToBreachesEnricher.name() == "to_leaks"---def test_enricher_category():- assert EmailToBreachesEnricher.category() == "Email"---def test_enricher_key():- assert EmailToBreachesEnricher.key() == "email"---def test_preprocess_string_emails():- emails = [- "[email protected]",- "[email protected]",- ]- result = enricher.preprocess(emails)- expected_emails = [Email(email=email) for email in emails]- assert result == expected_emails---def test_preprocess_dict_emails():- emails = [- {"email": "[email protected]"},- {"email": "[email protected]"},- ]- result = enricher.preprocess(emails)- expected_emails = [Email(email=email["email"]) for email in emails]- assert result == expected_emails---def test_preprocess_email_objects():- emails = [- Email(email="[email protected]"),- Email(email="[email protected]"),- ]- result = enricher.preprocess(emails)- assert result == emails---def test_preprocess_mixed_formats():- emails = [- "[email protected]",- {"email": "[email protected]"},- Email(email="[email protected]"),- {"invalid_key": "[email protected]"},- ]- result = enricher.preprocess(emails)-- result_emails = [email.email for email in result]- assert "[email protected]" in result_emails- assert "[email protected]" in result_emails- assert "[email protected]" in result_emails- assert "[email protected]" not in result_emails---@patch("src.enrichers.emails.to_leaks.requests.get")-def test_scan_successful_response(mock_get):- # Mock successful API response- mock_response = MagicMock()- mock_response.status_code = 200- mock_response.json.return_value = [- {"Name": "TestBreach", "Title": "Test Breach", "Domain": "test.com"},- {"Name": "AnotherBreach", "Title": "Another Breach", "Domain": "another.com"},- ]- mock_get.return_value = mock_response-- emails = [Email(email="[email protected]")]- result = enricher.scan(emails)-- assert len(result) == 2- assert isinstance(result[0], Breach)- assert isinstance(result[1], Breach)- assert result[0].name == "testbreach"- assert result[1].name == "anotherbreach"- assert result[0].breach["name"] == "testbreach"- assert result[1].breach["name"] == "anotherbreach"---@patch("src.enrichers.emails.to_leaks.requests.get")-def test_scan_no_breaches_found(mock_get):- # Mock 404 response (no breaches found)- mock_response = MagicMock()- mock_response.status_code = 404- mock_get.return_value = mock_response-- emails = [Email(email="[email protected]")]- result = enricher.scan(emails)-- assert len(result) == 0---@patch("src.enrichers.emails.to_leaks.requests.get")-def test_scan_api_error(mock_get):- # Mock API error- mock_get.side_effect = Exception("API Error")-- emails = [Email(email="[email protected]")]- result = enricher.scan(emails)-- assert len(result) == 0---@patch("src.enrichers.emails.to_leaks.requests.get")-def test_scan_missing_name_field(mock_get):- # Mock API response with missing "Name" field- mock_response = MagicMock()- mock_response.status_code = 200- mock_response.json.return_value = [- {"Title": "Test Breach", "Domain": "test.com"}, # Missing "Name" field- {"Name": "ValidBreach", "Title": "Valid Breach", "Domain": "valid.com"},- ]- mock_get.return_value = mock_response-- emails = [Email(email="[email protected]")]- result = enricher.scan(emails)-- assert len(result) == 2- assert result[0].name == "unknown" # Should default to "unknown"- assert result[1].name == "validbreach" # Should use the provided name- assert result[0].breach["title"] == "Test Breach"- assert result[1].breach["name"] == "validbreach"---@patch("src.enrichers.emails.to_leaks.HIBP_API_KEY", None)-def test_scan_no_api_key():- """Test that enricher raises ValueError when HIBP_API_KEY is not set."""- emails = [Email(email="[email protected]")]-- with pytest.raises(ValueError, match="HIBP_API_KEY not set"):- enricher.scan(emails)---def test_postprocess():- # Test postprocess method with mocked neo4j connection- enricher.neo4j_conn = MagicMock()-- # Create breach objects with the new structure- breach1 = Breach(- name="testbreach",- title="Test Breach",- domain="test.com",- pwncount=1000,- breach={"name": "testbreach", "title": "Test Breach"},- )- breach2 = Breach(- name="anotherbreach",- title="Another Breach",- domain="another.com",- pwncount=2000,- breach={"name": "anotherbreach", "title": "Another Breach"},- )-- breaches = [breach1, breach2]- original_input = [Email(email="[email protected]")]-- result = enricher.postprocess(breaches, original_input)-- assert result == breaches- # Verify that neo4j queries were called:- # - 2 breach node creation queries- # - 1 email node creation query- # - 2 relationship creation queries- # Total: 5 queries- assert enricher.neo4j_conn.query.call_count == 5diff --git a/flowsint-enrichers/tests/enrichers/ip/__init__.py b/flowsint-enrichers/tests/enrichers/ip/__init__.pydeleted file mode 100644index e69de29b..00000000diff --git a/flowsint-enrichers/tests/enrichers/ip/asn_to_cidrs.py b/flowsint-enrichers/tests/enrichers/ip/asn_to_cidrs.pydeleted file mode 100644index e83b4de0..00000000--- a/flowsint-enrichers/tests/enrichers/ip/asn_to_cidrs.py+++ /dev/null@@ -1,77 +0,0 @@-from flowsint_enrichers.ips.asn_to_cidrs import AsnToCidrsEnricher-from flowsint_types.asn import ASN--enricher = AsnToCidrsEnricher("sketch_123", "scan_123")---def test_preprocess_valid_asns():- asns = [- ASN(number=15169),- ASN(number=13335),- ]- result = enricher.preprocess(asns)-- result_numbers = [asn.number for asn in result]- expected_numbers = [asn.number for asn in asns]-- assert result_numbers == expected_numbers---def test_unprocessed_valid_asns():- asns = [- "15169",- "13335",- ]- result = enricher.preprocess(asns)- result_asns = [asn for asn in result]- expected_asns = [ASN(number=int(asn)) for asn in asns]- assert result_asns == expected_asns---def test_preprocess_invalid_asns():- asns = [- ASN(number=15169),- ASN(number=999999999999), # Invalid ASN number- ASN(number=13335),- ]- result = enricher.preprocess(asns)-- result_numbers = [asn.number for asn in result]- assert 15169 in result_numbers- assert 13335 in result_numbers- assert 999999999999 not in result_numbers---def test_preprocess_multiple_formats():- asns = [- {"number": 15169},- {"invalid_key": 13335},- ASN(number=13335),- "15169",- ]- result = enricher.preprocess(asns)-- result_numbers = [asn.number for asn in result]- assert 15169 in result_numbers- assert 13335 in result_numbers- assert (- "invalid_key" not in result_numbers- ) # Should be filtered out due to invalid key---def test_schemas():- input_schema = enricher.input_schema()- output_schema = enricher.output_schema()-- # Input schema should have number field- assert "properties" in input_schema- number_prop = next(- (prop for prop in input_schema["properties"] if prop["name"] == "number"), None- )- assert number_prop is not None- assert number_prop["type"] == "integer"-- # Output schema should have network field- assert "properties" in output_schema- prop_names = [prop["name"] for prop in output_schema["properties"]]- assert "network" in prop_namesdiff --git a/flowsint-enrichers/tests/enrichers/ip/cidr_to_ips.py b/flowsint-enrichers/tests/enrichers/ip/cidr_to_ips.pydeleted file mode 100644index 031274ed..00000000--- a/flowsint-enrichers/tests/enrichers/ip/cidr_to_ips.py+++ /dev/null@@ -1,122 +0,0 @@-from flowsint_enrichers.ips.cidr_to_ips import CidrToIpsEnricher-from flowsint_types.cidr import CIDR-from flowsint_types.ip import Ip-from tests.logger import TestLogger--logger = TestLogger()-enricher = CidrToIpsEnricher("sketch_123", "scan_123", logger)---def test_preprocess_valid_cidrs():- cidrs = [- CIDR(network="8.8.8.0/24"),- CIDR(network="1.1.1.0/24"),- ]- result = enricher.preprocess(cidrs)-- result_networks = [cidr.network for cidr in result]- expected_networks = [cidr.network for cidr in cidrs]-- assert result_networks == expected_networks---def test_preprocess_unprocessed_valid_cidrs():- cidrs = [- "8.8.8.0/24",- "1.1.1.0/24",- ]- result = enricher.preprocess(cidrs)- result_cidrs = [c for c in result]- expected_cidrs = [CIDR(network=c) for c in cidrs]- assert result_cidrs == expected_cidrs---def test_preprocess_invalid_cidrs():- cidrs = [- CIDR(network="8.8.8.0/24"),- "invalid-cidr",- "not-a-cidr",- ]- result = enricher.preprocess(cidrs)- result_networks = [str(cidr.network) for cidr in result]- assert "8.8.8.0/24" in result_networks- assert "invalid-cidr" not in result_networks- assert "not-a-cidr" not in result_networks---def test_preprocess_multiple_formats():- cidrs = [- {"network": "8.8.8.0/24"},- {"invalid_key": "1.1.1.0/24"},- CIDR(network="9.9.9.0/24"),- "InvalidCIDR",- ]- result = enricher.preprocess(cidrs)- result_networks = [str(cidr.network) for cidr in result]- assert "8.8.8.0/24" in result_networks- assert "9.9.9.0/24" in result_networks- assert "1.1.1.0/24" not in result_networks- assert "InvalidCIDR" not in result_networks---def test_scan_extracts_ips(monkeypatch):- mock_dnsx_output = """8.35.200.12-8.35.200.112-8.35.200.16-8.35.200.170"""-- class MockSubprocessResult:- def __init__(self, stdout):- self.stdout = stdout- self.returncode = 0-- def mock_subprocess_run(cmd, shell, capture_output, text, timeout):- assert "dnsx" in cmd- assert "-ptr" in cmd- return MockSubprocessResult(mock_dnsx_output)-- # Patch the subprocess call in the enricher- monkeypatch.setattr("subprocess.run", mock_subprocess_run)-- input_data = [CIDR(network="8.35.200.0/24")]- ips = enricher.scan(input_data)-- assert isinstance(ips, list)- assert len(ips) == 4-- expected_ips = ["8.35.200.12", "8.35.200.112", "8.35.200.16", "8.35.200.170"]-- for ip in ips:- assert isinstance(ip, Ip)- assert ip.address in expected_ips---def test_scan_handles_empty_output(monkeypatch):- class MockSubprocessResult:- def __init__(self):- self.stdout = ""- self.returncode = 0-- def mock_subprocess_run(cmd, shell, capture_output, text, timeout):- return MockSubprocessResult()-- monkeypatch.setattr("subprocess.run", mock_subprocess_run)-- input_data = [CIDR(network="8.8.8.0/24")]- ips = enricher.scan(input_data)-- assert isinstance(ips, list)- assert len(ips) == 0---def test_scan_handles_subprocess_exception(monkeypatch):- def mock_subprocess_run(cmd, shell, capture_output, text, timeout):- raise Exception("Subprocess failed")-- monkeypatch.setattr("subprocess.run", mock_subprocess_run)-- input_data = [CIDR(network="8.8.8.0/24")]- ips = enricher.scan(input_data)-- assert isinstance(ips, list)- assert len(ips) == 0diff --git a/flowsint-enrichers/tests/enrichers/ip/ip_to_asn.py b/flowsint-enrichers/tests/enrichers/ip/ip_to_asn.pydeleted file mode 100644index 373a85be..00000000--- a/flowsint-enrichers/tests/enrichers/ip/ip_to_asn.py+++ /dev/null@@ -1,267 +0,0 @@-import json-from unittest.mock import Mock-from flowsint_enrichers.ips.ip_to_asn import IpToAsnEnricher-from flowsint_types.ip import Ip-from flowsint_types.asn import ASN-from flowsint_types.cidr import CIDR-from tests.logger import TestLogger--logger = TestLogger()-# The enricher will get a mock logger from conftest.py automatically-enricher = IpToAsnEnricher("sketch_123", "scan_123", logger)---def test_preprocess_valid_ips():- ips = [- Ip(address="8.8.8.8"),- Ip(address="1.1.1.1"),- ]- result = enricher.preprocess(ips)-- result_addresses = [ip.address for ip in result]- expected_addresses = [ip.address for ip in ips]-- assert result_addresses == expected_addresses---def test_unprocessed_valid_ips():- ips = [- "8.8.8.8",- "1.1.1.1",- ]- result = enricher.preprocess(ips)- result_ips = [ip for ip in result]- expected_ips = [Ip(address=ip) for ip in ips]- assert result_ips == expected_ips---def test_preprocess_invalid_ips():- ips = [- Ip(address="8.8.8.8"),- Ip(address="invalid_ip"),- Ip(address="192.168.1.1"),- ]- result = enricher.preprocess(ips)-- result_addresses = [ip.address for ip in result]- assert "8.8.8.8" in result_addresses- assert "192.168.1.1" in result_addresses- assert "invalid_ip" not in result_addresses---def test_preprocess_multiple_formats():- ips = [- {"address": "8.8.8.8"},- {"invalid_key": "1.1.1.1"},- Ip(address="192.168.1.1"),- "10.0.0.1",- ]- result = enricher.preprocess(ips)-- result_addresses = [ip.address for ip in result]- assert "8.8.8.8" in result_addresses- assert "192.168.1.1" in result_addresses- assert "10.0.0.1" in result_addresses- assert (- "1.1.1.1" not in result_addresses- ) # Should be filtered out due to invalid key---def test_scan_extracts_asn_info(monkeypatch):- mock_asnmap_output = {- "input": "8.8.8.8",- "as_number": "AS15169",- "as_name": "GOOGLE",- "as_country": "US",- "as_range": ["8.8.8.0/24", "8.8.4.0/24"],- }-- class MockSubprocessResult:- def __init__(self, stdout):- self.stdout = stdout- self.returncode = 0-- def mock_subprocess_run(cmd, input, capture_output, text, timeout):- assert "asnmap" in cmd- assert input == "8.8.8.8"- return MockSubprocessResult(json.dumps(mock_asnmap_output))-- # Patch the subprocess call in the enricher- monkeypatch.setattr("subprocess.run", mock_subprocess_run)-- input_data = [Ip(address="8.8.8.8")]- asns = enricher.scan(input_data)-- assert isinstance(asns, list)- assert len(asns) == 1-- asn = asns[0]- assert isinstance(asn, ASN)- assert asn.number == 15169 # AS15169 -> 15169- assert asn.name == "GOOGLE"- assert asn.country == "US"- assert len(asn.cidrs) == 2- assert str(asn.cidrs[0].network) == "8.8.8.0/24"- assert str(asn.cidrs[1].network) == "8.8.4.0/24"---def test_scan_handles_no_asn_found(monkeypatch):- class MockSubprocessResult:- def __init__(self, stdout):- self.stdout = stdout- self.returncode = 0-- def mock_subprocess_run(cmd, input, capture_output, text, timeout):- # Return empty output to simulate no ASN found- return MockSubprocessResult("")-- monkeypatch.setattr("subprocess.run", mock_subprocess_run)-- input_data = [Ip(address="192.168.1.1")]- asns = enricher.scan(input_data)-- assert isinstance(asns, list)- assert len(asns) == 1-- asn = asns[0]- assert isinstance(asn, ASN)- assert asn.number == 0- assert asn.name == "Unknown"- assert asn.country == "Unknown"- assert len(asn.cidrs) == 0---def test_scan_handles_subprocess_exception(monkeypatch):- def mock_subprocess_run(cmd, input, capture_output, text, timeout):- raise Exception("Subprocess failed")-- monkeypatch.setattr("subprocess.run", mock_subprocess_run)-- input_data = [Ip(address="8.8.8.8")]- asns = enricher.scan(input_data)-- assert isinstance(asns, list)- assert len(asns) == 1-- asn = asns[0]- assert isinstance(asn, ASN)- assert asn.number == 0- assert asn.name == "Unknown"- assert asn.country == "Unknown"---def test_scan_multiple_ips(monkeypatch):- mock_responses = {- "8.8.8.8": {- "input": "8.8.8.8",- "as_number": "AS15169",- "as_name": "GOOGLE",- "as_country": "US",- "as_range": ["8.8.8.0/24"],- },- "1.1.1.1": {- "input": "1.1.1.1",- "as_number": "AS13335",- "as_name": "CLOUDFLARE",- "as_country": "US",- "as_range": ["1.1.1.0/24"],- },- }-- class MockSubprocessResult:- def __init__(self, stdout):- self.stdout = stdout- self.returncode = 0-- def mock_subprocess_run(cmd, input, capture_output, text, timeout):- if input in mock_responses:- return MockSubprocessResult(json.dumps(mock_responses[input]))- return MockSubprocessResult("")-- monkeypatch.setattr("subprocess.run", mock_subprocess_run)-- input_data = [Ip(address="8.8.8.8"), Ip(address="1.1.1.1")]- asns = enricher.scan(input_data)-- assert len(asns) == 2-- # Check first ASN- assert asns[0].number == 15169- assert asns[0].name == "GOOGLE"-- # Check second ASN- assert asns[1].number == 13335- assert asns[1].name == "CLOUDFLARE"---def test_schemas():- input_schema = enricher.input_schema()- output_schema = enricher.output_schema()-- # Input schema should have address field- assert "properties" in input_schema- address_prop = next(- (prop for prop in input_schema["properties"] if prop["name"] == "address"), None- )- assert address_prop is not None- assert address_prop["type"] == "string"-- # Output schema should have ASN fields- assert "properties" in output_schema- prop_names = [prop["name"] for prop in output_schema["properties"]]- assert "number" in prop_names- assert "name" in prop_names- assert "country" in prop_names- assert "cidrs" in prop_names---def test_postprocess_creates_neo4j_relationships(monkeypatch):- # Mock Neo4j connection- mock_neo4j = Mock()- enricher.neo4j_conn = mock_neo4j-- input_data = [Ip(address="8.8.8.8")]- asn_results = [- ASN(- number=15169,- name="GOOGLE",- country="US",- cidrs=[CIDR(network="8.8.8.0/24")],- )- ]-- result = enricher.postprocess(asn_results, input_data)-- # Verify Neo4j query was called- mock_neo4j.query.assert_called_once()-- # Check the query parameters- call_args = mock_neo4j.query.call_args- params = call_args[0][1]- assert params["ip_address"] == "8.8.8.8"- assert params["asn_number"] == 15169- assert params["asn_name"] == "GOOGLE"- assert params["asn_country"] == "US"- assert params["sketch_id"] == "sketch_123"-- # Should return the same results- assert result == asn_results---def test_postprocess_skips_unknown_asns(monkeypatch):- # Mock Neo4j connection- mock_neo4j = Mock()- enricher.neo4j_conn = mock_neo4j-- input_data = [Ip(address="192.168.1.1")]- asn_results = [- ASN(number=0, name="Unknown", country="Unknown", cidrs=[]) # Unknown ASN- ]-- result = enricher.postprocess(asn_results, input_data)-- # Verify Neo4j query was NOT called for unknown ASN- mock_neo4j.query.assert_not_called()-- # Should return the same results- assert result == asn_resultsdiff --git a/flowsint-enrichers/tests/enrichers/ip/ip_to_infos.py b/flowsint-enrichers/tests/enrichers/ip/ip_to_infos.pydeleted file mode 100644index 85767b57..00000000--- a/flowsint-enrichers/tests/enrichers/ip/ip_to_infos.py+++ /dev/null@@ -1,103 +0,0 @@-from flowsint_enrichers.ips.geolocation import IpToInfosEnricher-from flowsint_types.ip import Ip, Ip--enricher = IpToInfosEnricher("sketch_123", "scan_123")---def test_preprocess_valid_ips():- ips = [- Ip(address="8.8.8.8"),- Ip(address="1.1.1.1"),- ]- result = enricher.preprocess(ips)- result_ips = [d.address for d in result]- expected_ips = [d.address for d in ips]- assert result_ips == expected_ips---def test_preprocess_string_ips():- ips = [- "8.8.8.8",- "1.1.1.1",- ]- result = enricher.preprocess(ips)- result_ips = [d.address for d in result]- expected_ips = [d for d in ips]- assert [ip.address for ip in result] == expected_ips---def test_preprocess_invalid_ips():- ips = [- Ip(address="8.8.8.8"),- Ip(address="invalid_ip"),- Ip(address="1.1.1.1"),- ]- result = enricher.preprocess(ips)- result_ips = [d.address for d in result]- assert "8.8.8.8" in result_ips- assert "1.1.1.1" in result_ips- assert "invalid_ip" not in result_ips---def test_preprocess_multiple_formats():- ips = [- {"address": "8.8.8.8"},- {"invalid_key": "1.2.3.4"},- Ip(address="1.1.1.1"),- "1.1.1.1",- ]- result = enricher.preprocess(ips)- result_ips = [d.address for d in result]- assert "8.8.8.8" in result_ips- assert "1.1.1.1" in result_ips- assert "1.2.3.4" not in result_ips---def test_scan_returns_ip(monkeypatch):- # Mock of get_location_data- def mock_get_location_data(address):- return {- "latitude": 37.386,- "longitude": -122.0838,- "country": "US",- "city": "Mountain View",- "isp": "Google LLC",- }-- monkeypatch.setattr(enricher, "get_location_data", mock_get_location_data)-- input_data = [Ip(address="8.8.8.8")]- output = enricher.execute(input_data)- assert isinstance(output, list)- assert isinstance(output[0], Ip)- assert output[0].address == "8.8.8.8"- assert output[0].city == "Mountain View"- assert output[0].country == "US"- assert output[0].isp == "Google LLC"---def test_schemas():- input_schema = enricher.input_schema()- output_schema = enricher.output_schema()- assert input_schema == {- "type": "Ip",- "properties": [- {"name": "address", "type": "string"},- {"name": "latitude", "type": "number | null"},- {"name": "longitude", "type": "number | null"},- {"name": "country", "type": "string | null"},- {"name": "city", "type": "string | null"},- {"name": "isp", "type": "string | null"},- ],- }- assert output_schema == {- "type": "Ip",- "properties": [- {"name": "address", "type": "string"},- {"name": "latitude", "type": "number | null"},- {"name": "longitude", "type": "number | null"},- {"name": "country", "type": "string | null"},- {"name": "city", "type": "string | null"},- {"name": "isp", "type": "string | null"},- ],- }diff --git a/flowsint-enrichers/tests/enrichers/ip/test_to_asn_preprocess.py b/flowsint-enrichers/tests/enrichers/ip/test_to_asn_preprocess.pydeleted file mode 100644index 4b346ecc..00000000--- a/flowsint-enrichers/tests/enrichers/ip/test_to_asn_preprocess.py+++ /dev/null@@ -1,83 +0,0 @@-import pytest-from flowsint_enrichers.ip.to_asn import IpToAsnEnricher-from flowsint_types.ip import Ip---def enricher():- """Create enricher instance for testing."""- return IpToAsnEnricher(sketch_id="test_sketch", scan_id="test_scan")---def test_preprocess_valid_objects(enricher):- """Test preprocess with valid Ip objects."""- inputs = [- Ip(address="8.8.8.8"),- Ip(address="1.1.1.1"),- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Ip) for item in result)- assert result[0].address == "8.8.8.8"- assert result[1].address == "1.1.1.1"---def test_preprocess_strings(enricher):- """Test preprocess with string inputs (converted via primary field)."""- inputs = ["8.8.8.8", "1.1.1.1"]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Ip) for item in result)- assert result[0].address == "8.8.8.8"- assert result[1].address == "1.1.1.1"---def test_preprocess_dicts(enricher):- """Test preprocess with dict inputs."""- inputs = [- {"address": "8.8.8.8"},- {"address": "1.1.1.1"},- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Ip) for item in result)- assert result[0].address == "8.8.8.8"- assert result[1].address == "1.1.1.1"---def test_preprocess_invalid_filtered(enricher):- """Test that invalid items are filtered out."""- inputs = [- Ip(address="8.8.8.8"),- "not-an-ip", # Invalid- Ip(address="1.1.1.1"),- ]- result = enricher.preprocess(inputs)-- # Only valid items should remain- assert len(result) == 2- assert result[0].address == "8.8.8.8"- assert result[1].address == "1.1.1.1"---def test_preprocess_mixed_formats(enricher):- """Test preprocess with mixed input formats."""- inputs = [- Ip(address="8.8.8.8"), # Object- "1.1.1.1", # String- {"address": "192.168.1.1"}, # Dict- {"invalid_key": "10.0.0.1"}, # Invalid dict (wrong key)- ]- result = enricher.preprocess(inputs)-- assert len(result) == 3 # Should have 3 valid items- assert all(isinstance(item, Ip) for item in result)---def test_preprocess_empty_list(enricher):- """Test preprocess with empty input."""- result = enricher.preprocess([])- assert result == []diff --git a/flowsint-enrichers/tests/enrichers/ip/test_to_infos_preprocess.py b/flowsint-enrichers/tests/enrichers/ip/test_to_infos_preprocess.pydeleted file mode 100644index b4d1bc61..00000000--- a/flowsint-enrichers/tests/enrichers/ip/test_to_infos_preprocess.py+++ /dev/null@@ -1,98 +0,0 @@-import pytest-from flowsint_enrichers.ip.to_infos import IpToInfosEnricher-from flowsint_types.ip import Ip---def enricher():- """Create enricher instance for testing."""- return IpToInfosEnricher(sketch_id="test_sketch", scan_id="test_scan")---def test_preprocess_valid_objects(enricher):- """Test preprocess with valid Ip objects."""- inputs = [- Ip(address="8.8.8.8"),- Ip(address="1.1.1.1"),- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Ip) for item in result)- assert result[0].address == "8.8.8.8"- assert result[1].address == "1.1.1.1"---def test_preprocess_strings(enricher):- """Test preprocess with string inputs (converted via primary field)."""- inputs = ["8.8.8.8", "1.1.1.1"]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Ip) for item in result)- assert result[0].address == "8.8.8.8"- assert result[1].address == "1.1.1.1"---def test_preprocess_dicts(enricher):- """Test preprocess with dict inputs."""- inputs = [- {"address": "8.8.8.8"},- {"address": "1.1.1.1"},- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Ip) for item in result)- assert result[0].address == "8.8.8.8"- assert result[1].address == "1.1.1.1"---def test_preprocess_invalid_filtered(enricher):- """Test that invalid items are filtered out."""- inputs = [- Ip(address="8.8.8.8"),- "999.999.999.999", # Invalid- Ip(address="1.1.1.1"),- ]- result = enricher.preprocess(inputs)-- # Only valid items should remain- assert len(result) == 2- assert result[0].address == "8.8.8.8"- assert result[1].address == "1.1.1.1"---def test_preprocess_mixed_formats(enricher):- """Test preprocess with mixed input formats."""- inputs = [- Ip(address="8.8.8.8"), # Object- "1.1.1.1", # String- {"address": "192.168.1.1"}, # Dict- {"invalid_key": "10.0.0.1"}, # Invalid dict (wrong key)- ]- result = enricher.preprocess(inputs)-- assert len(result) == 3 # Should have 3 valid items- assert all(isinstance(item, Ip) for item in result)- assert result[0].address == "8.8.8.8"- assert result[1].address == "1.1.1.1"- assert result[2].address == "192.168.1.1"---def test_preprocess_empty_list(enricher):- """Test preprocess with empty input."""- result = enricher.preprocess([])- assert result == []---def test_preprocess_all_invalid(enricher):- """Test preprocess when all items are invalid."""- inputs = [- "not-an-ip",- "999.999.999.999",- "invalid",- ]- result = enricher.preprocess(inputs)-- assert len(result) == 0diff --git a/flowsint-enrichers/tests/enrichers/ip/to_ports.py b/flowsint-enrichers/tests/enrichers/ip/to_ports.pydeleted file mode 100644index b58e8004..00000000--- a/flowsint-enrichers/tests/enrichers/ip/to_ports.py+++ /dev/null@@ -1,89 +0,0 @@-import pytest-from flowsint_enrichers.ip.to_ports import IpToPortsEnricher-from flowsint_types.ip import Ip---def enricher():- """Create an enricher instance for testing"""- return IpToPortsEnricher(- params={- "mode": "passive",- "top_ports": "100",- "service_detection": "false",- }- )---def test_name():- assert IpToPortsEnricher.name() == "ip_to_ports"---def test_category():- assert IpToPortsEnricher.category() == "Ip"---def test_key():- assert IpToPortsEnricher.key() == "address"---def test_required_params():- assert IpToPortsEnricher.required_params() == False---def test_params_schema():- schema = IpToPortsEnricher.get_params_schema()- assert isinstance(schema, list)- assert len(schema) > 0- # Check that mode param exists- mode_param = next((p for p in schema if p["name"] == "mode"), None)- assert mode_param is not None- assert mode_param["type"] == "select"- assert mode_param["default"] == "passive"---def test_preprocess_string(enricher):- """Test preprocessing with string input"""- input_data = ["192.168.1.1", "10.0.0.1"]- result = enricher.preprocess(input_data)- assert len(result) == 2- assert all(isinstance(ip, Ip) for ip in result)- assert result[0].address == "192.168.1.1"- assert result[1].address == "10.0.0.1"---def test_preprocess_dict(enricher):- """Test preprocessing with dict input"""- input_data = [{"address": "192.168.1.1"}, {"address": "10.0.0.1"}]- result = enricher.preprocess(input_data)- assert len(result) == 2- assert all(isinstance(ip, Ip) for ip in result)---def test_preprocess_ip_objects(enricher):- """Test preprocessing with Ip objects"""- input_data = [Ip(address="192.168.1.1"), Ip(address="10.0.0.1")]- result = enricher.preprocess(input_data)- assert len(result) == 2- assert all(isinstance(ip, Ip) for ip in result)---def test_preprocess_invalid_ip(enricher):- """Test preprocessing filters out invalid IPs"""- input_data = ["192.168.1.1", "not-an-ip", "10.0.0.1"]- result = enricher.preprocess(input_data)- assert len(result) == 2- assert result[0].address == "192.168.1.1"- assert result[1].address == "10.0.0.1"---async def test_scan():- """Test the scan method (requires API key for passive mode)"""- # This test would require actual API credentials and network access- # For now, just verify the method exists and has correct signature- enricher = IpToPortsEnricher(- params={"mode": "passive", "top_ports": "100"}- )- assert hasattr(enricher, "scan")- assert callable(enricher.scan)diff --git a/flowsint-enrichers/tests/enrichers/organization/__init__.py b/flowsint-enrichers/tests/enrichers/organization/__init__.pydeleted file mode 100644index e69de29b..00000000diff --git a/flowsint-enrichers/tests/enrichers/organization/test_to_asn_preprocess.py b/flowsint-enrichers/tests/enrichers/organization/test_to_asn_preprocess.pydeleted file mode 100644index c7704a6f..00000000--- a/flowsint-enrichers/tests/enrichers/organization/test_to_asn_preprocess.py+++ /dev/null@@ -1,87 +0,0 @@-import pytest-from flowsint_enrichers.organization.to_asn import OrgToAsnEnricher-from flowsint_types.organization import Organization---def enricher():- """Create enricher instance for testing."""- return OrgToAsnEnricher(sketch_id="test_sketch", scan_id="test_scan")---def test_preprocess_valid_objects(enricher):- """Test preprocess with valid Organization objects."""- inputs = [- Organization(name="Acme Corp"),- Organization(name="Google LLC"),- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Organization) for item in result)- assert result[0].name == "Acme Corp"- assert result[1].name == "Google LLC"---def test_preprocess_strings(enricher):- """Test preprocess with string inputs (converted via primary field)."""- inputs = ["Acme Corp", "Google LLC"]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Organization) for item in result)- assert result[0].name == "Acme Corp"- assert result[1].name == "Google LLC"---def test_preprocess_dicts(enricher):- """Test preprocess with dict inputs."""- inputs = [- {"name": "Acme Corp"},- {"name": "Google LLC"},- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Organization) for item in result)- assert result[0].name == "Acme Corp"- assert result[1].name == "Google LLC"---def test_preprocess_mixed_formats(enricher):- """Test preprocess with mixed input formats."""- inputs = [- Organization(name="Acme Corp"), # Object- "Google LLC", # String- {"name": "Microsoft Corporation"}, # Dict- {"invalid_key": "Apple Inc"}, # Invalid dict (wrong key)- ]- result = enricher.preprocess(inputs)-- assert len(result) == 3 # Should have 3 valid items- assert all(isinstance(item, Organization) for item in result)- assert result[0].name == "Acme Corp"- assert result[1].name == "Google LLC"- assert result[2].name == "Microsoft Corporation"---def test_preprocess_empty_list(enricher):- """Test preprocess with empty input."""- result = enricher.preprocess([])- assert result == []---def test_preprocess_with_additional_fields(enricher):- """Test preprocess with dicts containing additional fields."""- inputs = [- {"name": "Acme Corp", "nom_complet": "Acme Corporation"},- {"name": "Google LLC", "siren": "123456789"},- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Organization) for item in result)- assert result[0].name == "Acme Corp"- assert result[0].nom_complet == "Acme Corporation"- assert result[1].name == "Google LLC"- assert result[1].siren == "123456789"diff --git a/flowsint-enrichers/tests/enrichers/organization/to_infos.py b/flowsint-enrichers/tests/enrichers/organization/to_infos.pydeleted file mode 100644index f2dc22ff..00000000--- a/flowsint-enrichers/tests/enrichers/organization/to_infos.py+++ /dev/null@@ -1,29 +0,0 @@-from flowsint_enrichers.organizations.to_infos import OrgToInfosEnricher-from flowsint_types.organization import Organization--enricher = OrgToInfosEnricher("sketch_123", "scan_123")---def test_preprocess_valid_names():- data = [Organization(name="OpenAI"), {"name": "Inria"}, "OVH"]- result = enricher.preprocess(data)- result_names = [org.name for org in result]-- assert result_names == ["OpenAI", "Inria", "OVH"]---# def test_preprocess_invalid_entries():-# data = [-# {"wrong_key": "value"},-# 123,-# None,-# "",-# {"name": ""},-# ]-# result = enricher.preprocess(data)-# assert result == []---def test_execute():- enricher.execute(["Karim Terrache"])- assert Truediff --git a/flowsint-enrichers/tests/enrichers/phone/test_to_infos_preprocess.py b/flowsint-enrichers/tests/enrichers/phone/test_to_infos_preprocess.pydeleted file mode 100644index 135c79d0..00000000--- a/flowsint-enrichers/tests/enrichers/phone/test_to_infos_preprocess.py+++ /dev/null@@ -1,95 +0,0 @@-import pytest-from flowsint_enrichers.phone.to_infos import IgnorantEnricher-from flowsint_types.phone import Phone---def enricher():- """Create enricher instance for testing."""- return IgnorantEnricher(sketch_id="test_sketch", scan_id="test_scan")---def test_preprocess_valid_objects(enricher):- """Test preprocess with valid Phone objects."""- inputs = [- Phone(number="+33612345678"),- Phone(number="+14155552671"),- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Phone) for item in result)- assert result[0].number == "+33612345678"- assert result[1].number == "+14155552671"---def test_preprocess_strings(enricher):- """Test preprocess with string inputs (converted via primary field)."""- inputs = ["+33612345678", "+14155552671"]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Phone) for item in result)- assert result[0].number == "+33612345678"- assert result[1].number == "+14155552671"---def test_preprocess_dicts(enricher):- """Test preprocess with dict inputs."""- inputs = [- {"number": "+33612345678"},- {"number": "+14155552671"},- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Phone) for item in result)- assert result[0].number == "+33612345678"- assert result[1].number == "+14155552671"---def test_preprocess_invalid_filtered(enricher):- """Test that invalid items are filtered out."""- inputs = [- Phone(number="+33612345678"),- "123", # Invalid (too short)- Phone(number="+14155552671"),- ]- result = enricher.preprocess(inputs)-- # Only valid items should remain- assert len(result) == 2- assert result[0].number == "+33612345678"- assert result[1].number == "+14155552671"---def test_preprocess_mixed_formats(enricher):- """Test preprocess with mixed input formats."""- inputs = [- Phone(number="+33612345678"), # Object- "+14155552671", # String- {"number": "+447911123456"}, # Dict- {"invalid_key": "+15555555555"}, # Invalid dict (wrong key)- ]- result = enricher.preprocess(inputs)-- assert len(result) == 3 # Should have 3 valid items- assert all(isinstance(item, Phone) for item in result)---def test_preprocess_empty_list(enricher):- """Test preprocess with empty input."""- result = enricher.preprocess([])- assert result == []---def test_preprocess_all_invalid(enricher):- """Test preprocess when all items are invalid."""- inputs = [- "123",- "not-a-phone",- "",- ]- result = enricher.preprocess(inputs)-- assert len(result) == 0diff --git a/flowsint-enrichers/tests/enrichers/social/__init__.py b/flowsint-enrichers/tests/enrichers/social/__init__.pydeleted file mode 100644index e69de29b..00000000diff --git a/flowsint-enrichers/tests/enrichers/social/maigret.py b/flowsint-enrichers/tests/enrichers/social/maigret.pydeleted file mode 100644index 10a3203f..00000000--- a/flowsint-enrichers/tests/enrichers/social/maigret.py+++ /dev/null@@ -1,57 +0,0 @@-from pathlib import Path-from flowsint_enrichers.socials.maigret import MaigretEnricher-from flowsint_types.social import SocialAccount--enricher = MaigretEnricher("sketch_123", "scan_123")---def test_unprocessed_valid_usernames():- usernames = [- "toto123",- "DorianXd78",- ]- result = enricher.preprocess(usernames)- result_usernames = [d for d in result]- expected_usernames = [SocialAccount(username=d) for d in usernames]- assert result_usernames == expected_usernames---def test_preprocess_invalid_usernames():- usernames = [- SocialAccount(username="toto123"),- SocialAccount(username="DorianXd78_Official"),- SocialAccount(username="This is not a username"),- ]- result = enricher.preprocess(usernames)-- result_usernames = [d.username for d in result]- assert "toto123" in result_usernames- assert "DorianXd78_Official" in result_usernames- assert "This is not a username" not in result_usernames---def test_preprocess_multiple_formats():- usernames = [- {"username": "toto123"},- {"invalid_key": "ValId_UseRnAme"},- SocialAccount(username="DorianXd78_Official"),- "MySimpleUsername",- ]- result = enricher.preprocess(usernames)-- result_usernames = [d.username for d in result]- assert "toto123" in result_usernames- assert "DorianXd78_Official" in result_usernames- assert "ValId_UseRnAme" not in result_usernames- assert "MySimpleUsername" in result_usernames---def test_parsing_invalid_output_file():- results = enricher.parse_maigret_output("toto123", Path("/this/path/does/not/exist"))- assert results == []---def test_parsing():- results = enricher.parse_maigret_output("toto123", Path("/tmp/maigret_test.json"))- print(results)- assert len(results) == 2diff --git a/flowsint-enrichers/tests/enrichers/social/test_to_maigret_preprocess.py b/flowsint-enrichers/tests/enrichers/social/test_to_maigret_preprocess.pydeleted file mode 100644index 7ba58732..00000000--- a/flowsint-enrichers/tests/enrichers/social/test_to_maigret_preprocess.py+++ /dev/null@@ -1,111 +0,0 @@-import pytest-from flowsint_enrichers.social.to_maigret import MaigretEnricher-from flowsint_types.username import Username---def enricher():- """Create enricher instance for testing."""- return MaigretEnricher(sketch_id="test_sketch", scan_id="test_scan")---def test_preprocess_valid_objects(enricher):- """Test preprocess with valid Username objects."""- inputs = [- Username(value="john_doe"),- Username(value="user123"),- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Username) for item in result)- assert result[0].value == "john_doe"- assert result[1].value == "user123"---def test_preprocess_strings(enricher):- """Test preprocess with string inputs (converted via primary field)."""- inputs = ["john_doe", "user123"]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Username) for item in result)- assert result[0].value == "john_doe"- assert result[1].value == "user123"---def test_preprocess_dicts(enricher):- """Test preprocess with dict inputs."""- inputs = [- {"value": "john_doe"},- {"value": "user123"},- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Username) for item in result)- assert result[0].value == "john_doe"- assert result[1].value == "user123"---def test_preprocess_invalid_filtered(enricher):- """Test that invalid items are filtered out."""- inputs = [- Username(value="john_doe"),- "ab", # Invalid (too short, must be 3-30 chars)- Username(value="user123"),- ]- result = enricher.preprocess(inputs)-- # Only valid items should remain- assert len(result) == 2- assert result[0].value == "john_doe"- assert result[1].value == "user123"---def test_preprocess_mixed_formats(enricher):- """Test preprocess with mixed input formats."""- inputs = [- Username(value="john_doe"), # Object- "user123", # String- {"value": "alice_2023"}, # Dict- {"invalid_key": "bob_smith"}, # Invalid dict (wrong key)- ]- result = enricher.preprocess(inputs)-- assert len(result) == 3 # Should have 3 valid items- assert all(isinstance(item, Username) for item in result)---def test_preprocess_empty_list(enricher):- """Test preprocess with empty input."""- result = enricher.preprocess([])- assert result == []---def test_preprocess_all_invalid(enricher):- """Test preprocess when all items are invalid."""- inputs = [- "ab", # Too short- "user with spaces", # Invalid characters- "", # Empty- ]- result = enricher.preprocess(inputs)-- assert len(result) == 0---def test_preprocess_with_platform(enricher):- """Test preprocess with username including platform."""- inputs = [- {"value": "john_doe", "platform": "twitter"},- {"value": "user123", "platform": "github"},- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Username) for item in result)- assert result[0].value == "john_doe"- assert result[0].platform == "twitter"- assert result[1].value == "user123"- assert result[1].platform == "github"diff --git a/flowsint-enrichers/tests/enrichers/social/test_to_sherlock_preprocess.py b/flowsint-enrichers/tests/enrichers/social/test_to_sherlock_preprocess.pydeleted file mode 100644index 8c501857..00000000--- a/flowsint-enrichers/tests/enrichers/social/test_to_sherlock_preprocess.py+++ /dev/null@@ -1,95 +0,0 @@-import pytest-from flowsint_enrichers.social.to_sherlock import SherlockEnricher-from flowsint_types.username import Username---def enricher():- """Create enricher instance for testing."""- return SherlockEnricher(sketch_id="test_sketch", scan_id="test_scan")---def test_preprocess_valid_objects(enricher):- """Test preprocess with valid Username objects."""- inputs = [- Username(value="john_doe"),- Username(value="user123"),- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Username) for item in result)- assert result[0].value == "john_doe"- assert result[1].value == "user123"---def test_preprocess_strings(enricher):- """Test preprocess with string inputs (converted via primary field)."""- inputs = ["john_doe", "user123"]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Username) for item in result)- assert result[0].value == "john_doe"- assert result[1].value == "user123"---def test_preprocess_dicts(enricher):- """Test preprocess with dict inputs."""- inputs = [- {"value": "john_doe"},- {"value": "user123"},- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Username) for item in result)- assert result[0].value == "john_doe"- assert result[1].value == "user123"---def test_preprocess_invalid_filtered(enricher):- """Test that invalid items are filtered out."""- inputs = [- Username(value="john_doe"),- "ab", # Invalid (too short, must be 3-30 chars)- Username(value="user123"),- ]- result = enricher.preprocess(inputs)-- # Only valid items should remain- assert len(result) == 2- assert result[0].value == "john_doe"- assert result[1].value == "user123"---def test_preprocess_mixed_formats(enricher):- """Test preprocess with mixed input formats."""- inputs = [- Username(value="john_doe"), # Object- "user123", # String- {"value": "alice_2023"}, # Dict- {"invalid_key": "bob_smith"}, # Invalid dict (wrong key)- ]- result = enricher.preprocess(inputs)-- assert len(result) == 3 # Should have 3 valid items- assert all(isinstance(item, Username) for item in result)---def test_preprocess_empty_list(enricher):- """Test preprocess with empty input."""- result = enricher.preprocess([])- assert result == []---def test_preprocess_all_invalid(enricher):- """Test preprocess when all items are invalid."""- inputs = [- "ab", # Too short (min 3 chars)- "user with spaces", # Invalid characters- "a" * 81, # Too long (max 80 chars)- ]- result = enricher.preprocess(inputs)-- assert len(result) == 0diff --git a/flowsint-enrichers/tests/enrichers/test_registry.py b/flowsint-enrichers/tests/enrichers/test_registry.pynew file mode 100644index 00000000..23ce366f--- /dev/null+++ b/flowsint-enrichers/tests/enrichers/test_registry.py@@ -0,0 +1,13 @@+import pytest+from flowsint_enrichers import ENRICHER_REGISTRY+++def test_enricher_registry_enricher_found():+ enricher = ENRICHER_REGISTRY.get_enricher("domain_to_ip", "123", "123")+ assert enricher.name() == "domain_to_ip"+++def test_enricher_registry_enricher_not_found():+ with pytest.raises(Exception) as error:+ ENRICHER_REGISTRY.get_enricher("enricher_does_not_exist", "123", "123")+ assert "not found" in str(error.value)diff --git a/flowsint-enrichers/tests/enrichers/website/__init__.py b/flowsint-enrichers/tests/enrichers/website/__init__.pydeleted file mode 100644index e69de29b..00000000diff --git a/flowsint-enrichers/tests/enrichers/website/test_to_domain_preprocess.py b/flowsint-enrichers/tests/enrichers/website/test_to_domain_preprocess.pydeleted file mode 100644index 55375d24..00000000--- a/flowsint-enrichers/tests/enrichers/website/test_to_domain_preprocess.py+++ /dev/null@@ -1,95 +0,0 @@-import pytest-from flowsint_enrichers.website.to_domain import WebsiteToDomainEnricher-from flowsint_types.website import Website---def enricher():- """Create enricher instance for testing."""- return WebsiteToDomainEnricher(sketch_id="test_sketch", scan_id="test_scan")---def test_preprocess_valid_objects(enricher):- """Test preprocess with valid Website objects."""- inputs = [- Website(url="https://example.com"),- Website(url="https://google.com"),- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Website) for item in result)- assert str(result[0].url) == "https://example.com/"- assert str(result[1].url) == "https://google.com/"---def test_preprocess_strings(enricher):- """Test preprocess with string inputs (converted via primary field)."""- inputs = ["https://example.com", "https://google.com"]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Website) for item in result)- assert str(result[0].url) == "https://example.com/"- assert str(result[1].url) == "https://google.com/"---def test_preprocess_dicts(enricher):- """Test preprocess with dict inputs."""- inputs = [- {"url": "https://example.com"},- {"url": "https://google.com"},- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Website) for item in result)- assert str(result[0].url) == "https://example.com/"- assert str(result[1].url) == "https://google.com/"---def test_preprocess_invalid_filtered(enricher):- """Test that invalid items are filtered out."""- inputs = [- Website(url="https://example.com"),- "not-a-url", # Invalid- Website(url="https://google.com"),- ]- result = enricher.preprocess(inputs)-- # Only valid items should remain- assert len(result) == 2- assert str(result[0].url) == "https://example.com/"- assert str(result[1].url) == "https://google.com/"---def test_preprocess_mixed_formats(enricher):- """Test preprocess with mixed input formats."""- inputs = [- Website(url="https://example.com"), # Object- "https://google.com", # String- {"url": "https://github.com"}, # Dict- {"invalid_key": "https://wrong.com"}, # Invalid dict (wrong key)- ]- result = enricher.preprocess(inputs)-- assert len(result) == 3 # Should have 3 valid items- assert all(isinstance(item, Website) for item in result)---def test_preprocess_empty_list(enricher):- """Test preprocess with empty input."""- result = enricher.preprocess([])- assert result == []---def test_preprocess_all_invalid(enricher):- """Test preprocess when all items are invalid."""- inputs = [- "not-a-url",- "ht tp://broken.com",- "",- ]- result = enricher.preprocess(inputs)-- assert len(result) == 0diff --git a/flowsint-enrichers/tests/enrichers/website/test_to_text_preprocess.py b/flowsint-enrichers/tests/enrichers/website/test_to_text_preprocess.pydeleted file mode 100644index 15103fc9..00000000--- a/flowsint-enrichers/tests/enrichers/website/test_to_text_preprocess.py+++ /dev/null@@ -1,83 +0,0 @@-import pytest-from flowsint_enrichers.website.to_text import WebsiteToText-from flowsint_types.website import Website---def enricher():- """Create enricher instance for testing."""- return WebsiteToText(sketch_id="test_sketch", scan_id="test_scan")---def test_preprocess_valid_objects(enricher):- """Test preprocess with valid Website objects."""- inputs = [- Website(url="https://example.com"),- Website(url="https://google.com"),- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Website) for item in result)- assert str(result[0].url) == "https://example.com/"- assert str(result[1].url) == "https://google.com/"---def test_preprocess_strings(enricher):- """Test preprocess with string inputs (converted via primary field)."""- inputs = ["https://example.com", "https://google.com"]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Website) for item in result)- assert str(result[0].url) == "https://example.com/"- assert str(result[1].url) == "https://google.com/"---def test_preprocess_dicts(enricher):- """Test preprocess with dict inputs."""- inputs = [- {"url": "https://example.com"},- {"url": "https://google.com"},- ]- result = enricher.preprocess(inputs)-- assert len(result) == 2- assert all(isinstance(item, Website) for item in result)- assert str(result[0].url) == "https://example.com/"- assert str(result[1].url) == "https://google.com/"---def test_preprocess_invalid_filtered(enricher):- """Test that invalid items are filtered out."""- inputs = [- Website(url="https://example.com"),- "not-a-url", # Invalid- Website(url="https://google.com"),- ]- result = enricher.preprocess(inputs)-- # Only valid items should remain- assert len(result) == 2- assert str(result[0].url) == "https://example.com/"- assert str(result[1].url) == "https://google.com/"---def test_preprocess_mixed_formats(enricher):- """Test preprocess with mixed input formats."""- inputs = [- Website(url="https://example.com"), # Object- "https://google.com", # String- {"url": "https://github.com"}, # Dict- {"invalid_key": "https://wrong.com"}, # Invalid dict (wrong key)- ]- result = enricher.preprocess(inputs)-- assert len(result) == 3 # Should have 3 valid items- assert all(isinstance(item, Website) for item in result)---def test_preprocess_empty_list(enricher):- """Test preprocess with empty input."""- result = enricher.preprocess([])- assert result == []diff --git a/flowsint-enrichers/tests/enrichers/website/to_links.py b/flowsint-enrichers/tests/enrichers/website/to_links.pydeleted file mode 100644index 42e18747..00000000--- a/flowsint-enrichers/tests/enrichers/website/to_links.py+++ /dev/null@@ -1,275 +0,0 @@-import pytest-from unittest.mock import Mock, patch-from flowsint_enrichers.websites.to_links import WebsiteToLinks-from flowsint_types.website import Website---class MockCrawlResults:- def __init__(self, internal=None, external=None):- self.internal = internal or []- self.external = external or []---class MockCrawler:- def __init__(self, url, recursive=True, verbose=False, _on_result_callback=None):- self.url = url- self.callback = _on_result_callback-- def fetch(self):- pass-- def extract_urls(self):- # Simulate callback calls- if self.callback:- self.callback("https://example.com/page1", is_external=False)- self.callback("https://example.com/page2", is_external=False)- self.callback("https://external.com/page", is_external=True)- self.callback("https://another-external.org/resource", is_external=True)-- def get_results(self):- return MockCrawlResults(- internal=["https://example.com/page1", "https://example.com/page2"],- external=[- "https://external.com/page",- "https://another-external.org/resource",- ],- )---async def test_website_to_links_real_time_neo4j_creation():- """Test that Neo4j nodes are created in real-time during the callback."""- enricher = WebsiteToLinks(sketch_id="test", scan_id="test")-- # Mock neo4j connection and methods- enricher.neo4j_conn = Mock()- enricher.create_node = Mock()- enricher.create_relationship = Mock()- enricher.log_graph_message = Mock()-- # Test input- websites = [Website(url="https://example.com")]-- with patch("src.enrichers.websites.to_links.Crawler", MockCrawler):- results = await enricher.scan(websites)-- # Verify main website and domain nodes were created upfront- enricher.create_node.assert_any_call(- "website",- "url",- "https://example.com",- caption="https://example.com",- type="website",- )- enricher.create_node.assert_any_call(- "domain", "name", "example.com", caption="example.com", type="domain"- )-- # Verify main website to domain relationship- enricher.create_relationship.assert_any_call(- "website",- "url",- "https://example.com",- "domain",- "name",- "example.com",- "BELONGS_TO_DOMAIN",- )-- # Verify internal website nodes were created in callback- enricher.create_node.assert_any_call(- "website",- "url",- "https://example.com/page1",- caption="https://example.com/page1",- type="website",- )- enricher.create_node.assert_any_call(- "website",- "url",- "https://example.com/page2",- caption="https://example.com/page2",- type="website",- )-- # Verify internal website relationships- enricher.create_relationship.assert_any_call(- "website",- "url",- "https://example.com",- "website",- "url",- "https://example.com/page1",- "LINKS_TO",- )- enricher.create_relationship.assert_any_call(- "website",- "url",- "https://example.com",- "website",- "url",- "https://example.com/page2",- "LINKS_TO",- )-- # Verify external website nodes were created in callback- enricher.create_node.assert_any_call(- "website",- "url",- "https://external.com/page",- caption="https://external.com/page",- type="website",- )- enricher.create_node.assert_any_call(- "website",- "url",- "https://another-external.org/resource",- caption="https://another-external.org/resource",- type="website",- )-- # Verify external domain nodes were created in callback- enricher.create_node.assert_any_call(- "domain", "name", "external.com", caption="external.com", type="domain"- )- enricher.create_node.assert_any_call(- "domain",- "name",- "another-external.org",- caption="another-external.org",- type="domain",- )-- # Verify main website to external website relationships- enricher.create_relationship.assert_any_call(- "website",- "url",- "https://example.com",- "website",- "url",- "https://external.com/page",- "LINKS_TO",- )- enricher.create_relationship.assert_any_call(- "website",- "url",- "https://example.com",- "website",- "url",- "https://another-external.org/resource",- "LINKS_TO",- )-- # Verify external website to domain relationships- enricher.create_relationship.assert_any_call(- "website",- "url",- "https://external.com/page",- "domain",- "name",- "external.com",- "BELONGS_TO_DOMAIN",- )- enricher.create_relationship.assert_any_call(- "website",- "url",- "https://another-external.org/resource",- "domain",- "name",- "another-external.org",- "BELONGS_TO_DOMAIN",- )-- # Verify main website to external domain relationships- enricher.create_relationship.assert_any_call(- "website",- "url",- "https://example.com",- "domain",- "name",- "external.com",- "LINKS_TO_DOMAIN",- )- enricher.create_relationship.assert_any_call(- "website",- "url",- "https://example.com",- "domain",- "name",- "another-external.org",- "LINKS_TO_DOMAIN",- )---async def test_website_to_links_error_handling_with_neo4j():- """Test that main nodes are still created even when crawling fails."""- enricher = WebsiteToLinks(sketch_id="test", scan_id="test")-- # Mock neo4j connection and methods- enricher.neo4j_conn = Mock()- enricher.create_node = Mock()- enricher.create_relationship = Mock()- enricher.log_graph_message = Mock()-- # Mock crawler that raises an exception- def mock_crawler_error(*args, **kwargs):- raise Exception("Test error")-- websites = [Website(url="https://example.com")]-- with patch("src.enrichers.websites.to_links.Crawler", mock_crawler_error):- results = await enricher.scan(websites)-- # Verify main website and domain nodes were still created despite error- enricher.create_node.assert_any_call(- "website",- "url",- "https://example.com",- caption="https://example.com",- type="website",- )- enricher.create_node.assert_any_call(- "domain", "name", "example.com", caption="example.com", type="domain"- )-- # Verify main website to domain relationship was created- enricher.create_relationship.assert_any_call(- "website",- "url",- "https://example.com",- "domain",- "name",- "example.com",- "BELONGS_TO_DOMAIN",- )-- # Verify result structure- assert len(results) == 1- result = results[0]- assert result["website"] == "https://example.com"- assert result["main_domain"] == "example.com"- assert result["internal_urls"] == []- assert result["external_urls"] == []- assert result["external_domains"] == []---def test_postprocess_simplified():- """Test that postprocess now just returns results as-is."""- enricher = WebsiteToLinks(sketch_id="test", scan_id="test")-- original_input = [Website(url="https://example.com")]- results = [- {- "website": "https://example.com",- "main_domain": "example.com",- "internal_urls": ["https://example.com/page1"],- "external_urls": ["https://external.com/page"],- "external_domains": ["external.com"],- }- ]-- processed_results = enricher.postprocess(results, original_input)-- # Should just return the same results since Neo4j work is done in real-time- assert processed_results == resultsdiff --git a/flowsint-enrichers/tests/test_data/data.json b/flowsint-enrichers/tests/test_data/data.jsondeleted file mode 100644index 2322a9df..00000000--- a/flowsint-enrichers/tests/test_data/data.json+++ /dev/null@@ -1,934 +0,0 @@-{- "status": 1,- "api_query": "whois_history",- "total_records_found": 16,- "whois_records": [- {- "num": 1,- "domain_name": "epios.com",- "query_time": "2016-08-21 03:27:59",- "create_date": "1998-09-01",- "update_date": "2016-08-02",- "expiry_date": "2017-08-31",- "domain_registrar": {- "iana_id": 48,- "registrar_name": "eNom, Inc.",- "whois_server": "whois.enom.com",- "website_url": "http://www.enom.com",- "registrar_status": "Accredited"- },- "registrant_contact": {- "full_name": "JOHN DOE",- "mailing_address": "123 JOHN STREET",- "city_name": "DOE CITY",- "state_name": "CA",- "zip_code": "12345",- "country_name": "United States",- "country_code": "US",- "email_address": "[email protected], [email protected]",- "phone_number": "+123456789"- },- "administrative_contact": {- "full_name": "JOHN DOE",- "mailing_address": "123 JOHN STREET",- "city_name": "DOE CITY",- "state_name": "CA",- "zip_code": "12345",- "country_name": "United States",- "country_code": "US",- "email_address": "[email protected], [email protected]",- "phone_number": "+123456789"- },- "technical_contact": {- "full_name": "JOHN DOE",- "mailing_address": "123 JOHN STREET",- "city_name": "DOE CITY",- "state_name": "CA",- "zip_code": "12345",- "country_name": "United States",- "country_code": "US",- "email_address": "[email protected], [email protected]",- "phone_number": "+123456789"- },- "name_servers": [- "ns1.server270.com",- "ns2.server270.com"- ],- "domain_status": [- "clientTransferProhibited"- ]- },- {- "num": 2,- "domain_name": "epios.com",- "query_time": "2017-10-13 22:27:51",- "create_date": "1998-09-01",- "update_date": "2016-08-02",- "expiry_date": "2019-08-31",- "domain_registrar": {- "iana_id": 48,- "registrar_name": "eNom, Inc.",- "whois_server": "whois.enom.com",- "website_url": "http://www.enom.com",- "registrar_status": "Accredited"- },- "registrant_contact": {- "full_name": "JOHN DOE",- "mailing_address": "123 JOHN STREET",- "city_name": "DOE CITY",- "state_name": "CA",- "zip_code": "12345",- "country_name": "United States",- "country_code": "US",- "email_address": "[email protected], [email protected]",- "phone_number": "+123456789"- },- "administrative_contact": {- "full_name": "JOHN DOE",- "mailing_address": "123 JOHN STREET",- "city_name": "DOE CITY",- "state_name": "CA",- "zip_code": "12345",- "country_name": "United States",- "country_code": "US",- "email_address": "[email protected], [email protected]",- "phone_number": "+123456789"- },- "technical_contact": {- "full_name": "JOHN DOE",- "mailing_address": "123 JOHN STREET",- "city_name": "DOE CITY",- "state_name": "CA",- "zip_code": "12345",- "country_name": "United States",- "country_code": "US",- "email_address": "[email protected], [email protected]",- "phone_number": "+123456789"- },- "name_servers": [- "ns1.server270.com",- "ns2.server270.com"- ],- "domain_status": [- "clientTransferProhibited"- ]- },- {- "num": 3,- "domain_name": "epios.com",- "query_time": "2023-06-02 21:14:04",- "create_date": "1998-09-01",- "update_date": "2022-08-01",- "expiry_date": "2023-08-31",- "domain_registrar": {- "iana_id": 48,- "registrar_name": "eNom, Inc.",- "whois_server": "whois.enom.com",- "website_url": "http://www.enom.com",- "registrar_status": "Accredited"- },- "registrant_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "CA",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "United States",- "country_code": "US",- "email_address": "https://tieredaccess.com/contact/2d4e326b-fc3d-4269-a998-b62ba07aee51",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "administrative_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "technical_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "name_servers": [- "dns1.name-services.com",- "dns2.name-services.com",- "dns3.name-services.com",- "dns4.name-services.com"- ],- "domain_status": [- "clientTransferProhibited"- ]- },- {- "num": 4,- "domain_name": "epios.com",- "query_time": "2023-08-09 19:21:55",- "create_date": "1998-09-01",- "update_date": "2022-08-01",- "expiry_date": "2023-08-31",- "domain_registrar": {- "iana_id": 48,- "registrar_name": "eNom, Inc.",- "whois_server": "whois.enom.com",- "website_url": "http://www.enom.com",- "registrar_status": "Accredited"- },- "registrant_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "CA",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "United States",- "country_code": "US",- "email_address": "https://tieredaccess.com/contact/e811856a-515d-486d-bfa3-2771ec4b8bb4",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "administrative_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "technical_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "name_servers": [- "dns1.name-services.com",- "dns2.name-services.com",- "dns3.name-services.com",- "dns4.name-services.com"- ],- "domain_status": [- "clientTransferProhibited"- ]- },- {- "num": 5,- "domain_name": "epios.com",- "query_time": "2023-09-10 05:34:39",- "create_date": "1998-09-01",- "update_date": "2023-08-16",- "expiry_date": "2024-08-31",- "domain_registrar": {- "iana_id": 48,- "registrar_name": "eNom, Inc.",- "whois_server": "whois.enom.com",- "website_url": "http://www.enom.com",- "registrar_status": "Accredited"- },- "registrant_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "CA",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "United States",- "country_code": "US",- "email_address": "https://tieredaccess.com/contact/e966535a-18b4-4fc2-ba62-6b773211ea7d",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "administrative_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "technical_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "name_servers": [- "dns1.name-services.com",- "dns2.name-services.com",- "dns3.name-services.com",- "dns4.name-services.com"- ],- "domain_status": [- "clientTransferProhibited"- ]- },- {- "num": 6,- "domain_name": "epios.com",- "query_time": "2023-11-26 06:52:14",- "create_date": "1998-09-01",- "update_date": "2023-08-16",- "expiry_date": "2024-08-31",- "domain_registrar": {- "iana_id": 48,- "registrar_name": "eNom, Inc.",- "whois_server": "whois.enom.com",- "website_url": "http://www.enom.com",- "registrar_status": "Accredited"- },- "registrant_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "CA",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "United States",- "country_code": "US",- "email_address": "https://tieredaccess.com/contact/85a6b638-dbc5-43ca-8e81-1564fc0e1718",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "administrative_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "technical_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "name_servers": [- "dns1.name-services.com",- "dns2.name-services.com",- "dns3.name-services.com",- "dns4.name-services.com"- ],- "domain_status": [- "clientTransferProhibited"- ]- },- {- "num": 7,- "domain_name": "epios.com",- "query_time": "2024-02-08 04:00:50",- "create_date": "1998-09-01",- "update_date": "2023-08-16",- "expiry_date": "2024-08-31",- "domain_registrar": {- "iana_id": 48,- "registrar_name": "eNom, Inc.",- "whois_server": "whois.enom.com",- "website_url": "http://www.enom.com",- "registrar_status": "Accredited"- },- "name_servers": [- "dns1.name-services.com",- "dns2.name-services.com",- "dns3.name-services.com",- "dns4.name-services.com"- ],- "domain_status": [- "clientTransferProhibited"- ]- },- {- "num": 8,- "domain_name": "epios.com",- "query_time": "2024-02-13 01:44:26",- "create_date": "1998-09-01",- "update_date": "2023-08-16",- "expiry_date": "2024-08-31",- "domain_registrar": {- "iana_id": 48,- "registrar_name": "eNom, Inc.",- "whois_server": "whois.enom.com",- "website_url": "http://www.enom.com",- "registrar_status": "Accredited"- },- "registrant_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "CA",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "United States",- "country_code": "US",- "email_address": "https://tieredaccess.com/contact/fd6e5334-dad3-4247-9970-ef471bc3d887",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "administrative_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "technical_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "name_servers": [- "dns1.name-services.com",- "dns2.name-services.com",- "dns3.name-services.com",- "dns4.name-services.com"- ],- "domain_status": [- "clientTransferProhibited"- ]- },- {- "num": 9,- "domain_name": "epios.com",- "query_time": "2024-02-22 07:49:19",- "create_date": "1998-09-01",- "update_date": "2023-08-16",- "expiry_date": "2024-08-31",- "domain_registrar": {- "iana_id": 48,- "registrar_name": "eNom, Inc.",- "whois_server": "whois.enom.com",- "website_url": "http://www.enom.com",- "registrar_status": "Accredited"- },- "registrant_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "CA",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "United States",- "country_code": "US",- "email_address": "https://tieredaccess.com/contact/7f676f3b-cac7-4336-9787-1295d039c7fb",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "administrative_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "technical_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "name_servers": [- "dns1.name-services.com",- "dns2.name-services.com",- "dns3.name-services.com",- "dns4.name-services.com"- ],- "domain_status": [- "clientTransferProhibited"- ]- },- {- "num": 10,- "domain_name": "epios.com",- "query_time": "2024-04-06 03:08:26",- "create_date": "1998-09-01",- "update_date": "2023-08-16",- "expiry_date": "2024-08-31",- "domain_registrar": {- "iana_id": 48,- "registrar_name": "eNom, Inc.",- "whois_server": "whois.enom.com",- "website_url": "http://www.enom.com",- "registrar_status": "Accredited"- },- "registrant_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "CA",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "United States",- "country_code": "US",- "email_address": "https://tieredaccess.com/contact/a5ff0f8e-eb63-4648-86c7-d7a25b822b25",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "administrative_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "technical_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "name_servers": [- "dns1.name-services.com",- "dns2.name-services.com",- "dns3.name-services.com",- "dns4.name-services.com"- ],- "domain_status": [- "clientTransferProhibited"- ]- },- {- "num": 11,- "domain_name": "epios.com",- "query_time": "2024-06-05 00:00:58",- "create_date": "1998-09-01",- "update_date": "2023-08-16",- "expiry_date": "2024-08-31",- "domain_registrar": {- "iana_id": 48,- "registrar_name": "eNom, Inc.",- "whois_server": "whois.enom.com",- "website_url": "http://www.enom.com",- "registrar_status": "Accredited"- },- "registrant_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "CA",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "United States",- "country_code": "US",- "email_address": "https://tieredaccess.com/contact/c24d86c9-870b-425e-9650-38052534f131",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "administrative_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "technical_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "name_servers": [- "dns1.name-services.com",- "dns2.name-services.com",- "dns3.name-services.com",- "dns4.name-services.com"- ],- "domain_status": [- "clientTransferProhibited"- ]- },- {- "num": 12,- "domain_name": "epios.com",- "query_time": "2024-06-13 02:54:54",- "create_date": "1998-09-01",- "update_date": "2023-08-16",- "expiry_date": "2024-08-31",- "domain_registrar": {- "iana_id": 48,- "registrar_name": "eNom, Inc.",- "whois_server": "whois.enom.com",- "website_url": "http://www.enom.com",- "registrar_status": "Accredited"- },- "registrant_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "CA",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "United States",- "country_code": "US",- "email_address": "https://tieredaccess.com/contact/2d35cecc-0e79-4a18-b9d7-9cb7029442f1",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "administrative_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "technical_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "name_servers": [- "dns1.name-services.com",- "dns2.name-services.com",- "dns3.name-services.com",- "dns4.name-services.com"- ],- "domain_status": [- "clientTransferProhibited"- ]- },- {- "num": 13,- "domain_name": "epios.com",- "query_time": "2024-08-05 15:03:48",- "create_date": "1998-09-01",- "update_date": "2024-08-01",- "expiry_date": "2025-08-31",- "domain_registrar": {- "iana_id": 48,- "registrar_name": "eNom, Inc.",- "whois_server": "whois.enom.com",- "website_url": "http://www.enom.com",- "registrar_status": "Accredited"- },- "registrant_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "CA",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "United States",- "country_code": "US",- "email_address": "https://tieredaccess.com/contact/7654cb74-0744-475d-bba1-d5bb6f91b5aa",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "administrative_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "technical_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "name_servers": [- "dns1.name-services.com",- "dns2.name-services.com",- "dns3.name-services.com",- "dns4.name-services.com"- ],- "domain_status": [- "clientTransferProhibited",- "renewPeriod"- ]- },- {- "num": 14,- "domain_name": "epios.com",- "query_time": "2024-10-01 09:11:19",- "create_date": "1998-09-01",- "update_date": "2024-08-01",- "expiry_date": "2025-08-31",- "domain_registrar": {- "iana_id": 48,- "registrar_name": "eNom, Inc.",- "whois_server": "whois.enom.com",- "website_url": "http://www.enom.com",- "registrar_status": "Accredited"- },- "registrant_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "CA",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "United States",- "country_code": "US",- "email_address": "https://tieredaccess.com/contact/db7c6dea-a389-4e83-ad2c-98c7c2f8cc18",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "administrative_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "technical_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "name_servers": [- "dns1.name-services.com",- "dns2.name-services.com",- "dns3.name-services.com",- "dns4.name-services.com"- ],- "domain_status": [- "clientTransferProhibited"- ]- },- {- "num": 15,- "domain_name": "epios.com",- "query_time": "2024-11-24 18:45:22",- "create_date": "1998-09-01",- "update_date": "2024-11-24",- "expiry_date": "2025-08-31",- "domain_registrar": {- "iana_id": 48,- "registrar_name": "eNom, Inc.",- "whois_server": "whois.enom.com",- "website_url": "http://www.enom.com",- "registrar_status": "Accredited"- },- "registrant_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "CA",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "United States",- "country_code": "US",- "email_address": "https://tieredaccess.com/contact/a5e74127-d4d4-450c-a314-1ebbe1254e20",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "administrative_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "technical_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "redacted for privacy",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "name_servers": [- "dns1.name-services.com",- "dns2.name-services.com",- "dns3.name-services.com",- "dns4.name-services.com"- ],- "domain_status": [- "PENDINGTRANSFER"- ]- },- {- "num": 16,- "domain_name": "epios.com",- "query_time": "2025-08-24 08:10:49",- "create_date": "1998-09-01",- "update_date": "2024-11-30",- "expiry_date": "2026-08-31",- "domain_registrar": {- "iana_id": 433,- "registrar_name": "OVH sas",- "whois_server": "whois.ovh.com",- "website_url": "http://www.ovh.com",- "registrar_status": "Accredited"- },- "registrant_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "Epieos",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "France",- "country_code": "FR",- "email_address": "REDACTED FOR PRIVACY - Send message to contact by visiting https://www.ovhcloud.com/en/lp/request-ov",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "administrative_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "REDACTED FOR PRIVACY - Send message to contact by visiting https://www.ovhcloud.com/en/lp/request-ov",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "technical_contact": {- "full_name": "REDACTED FOR PRIVACY",- "company_name": "REDACTED FOR PRIVACY",- "mailing_address": "REDACTED FOR PRIVACY",- "city_name": "REDACTED FOR PRIVACY",- "state_name": "REDACTED FOR PRIVACY",- "zip_code": "REDACTED FOR PRIVACY",- "country_name": "REDACTED FOR PRIVACY",- "email_address": "REDACTED FOR PRIVACY - Send message to contact by visiting https://www.ovhcloud.com/en/lp/request-ov",- "phone_number": "REDACTED FOR PRIVACY",- "fax_number": "REDACTED FOR PRIVACY"- },- "name_servers": [- "dns200.anycast.me",- "ns200.anycast.me"- ],- "domain_status": [- "clientDeleteProhibited",- "clientTransferProhibited"- ]- }- ],- "api_execution_time": 0.01-}\ No newline at end of filediff --git a/flowsint-enrichers/tests/test_vault_integration.py b/flowsint-enrichers/tests/test_vault_integration.pyindex d379d889..dc1997de 100644--- a/flowsint-enrichers/tests/test_vault_integration.py+++ b/flowsint-enrichers/tests/test_vault_integration.py@@ -1,7 +1,7 @@"""Integration tests for enrichers using vault secrets."""import pytestimport uuid-from unittest.mock import Mock, MagicMock+from unittest.mock import Mockfrom flowsint_enrichers.domain.to_history import DomainToHistoryEnricherfrom flowsint_enrichers.individual.to_domains import IndividualToDomainsEnricherfrom flowsint_enrichers.email.to_domains import EmailToDomainsEnricher