A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat: type validation + simplified transform system
dextmorgn7b91bb244241
diff --git a/flowsint-api/app/api/routes/types.py b/flowsint-api/app/api/routes/types.pyindex 1ddb7dbc..5db1c6b9 100644--- a/flowsint-api/app/api/routes/types.py+++ b/flowsint-api/app/api/routes/types.py@@ -282,6 +282,8 @@ def extract_input_schema("fields": [resolve_field(prop, details=info, schema=schema)for prop, info in details.get("properties", {}).items()+ # exclude label from properties to fill+ if prop != "label"],}diff --git a/flowsint-api/app/utils.py b/flowsint-api/app/utils.pyindex 1a0bd16d..c9b1260a 100644--- a/flowsint-api/app/utils.py+++ b/flowsint-api/app/utils.py@@ -8,7 +8,6 @@ import reimport sslimport socketfrom typing import Dict, Any, List, Type-from pydantic import BaseModelimport inspectfrom typing import Any, Dict, Typefrom pydantic import BaseModel, TypeAdapterdiff --git a/flowsint-app/src/components/graphs/details-panel/details-panel.tsx b/flowsint-app/src/components/graphs/details-panel/details-panel.tsxindex b4dc9100..b9c464bb 100644--- a/flowsint-app/src/components/graphs/details-panel/details-panel.tsx+++ b/flowsint-app/src/components/graphs/details-panel/details-panel.tsx@@ -113,7 +113,7 @@ function KeyValueDisplay({ data, className }: KeyValueDisplayProps) {{data &&Object.entries(data).filter(- ([key]) => !['sketch_id', 'caption', 'size', 'color', 'description', 'x', 'y'].includes(key)+ ([key]) => !['id', 'sketch_id', 'caption', 'size', 'color', 'description', 'x', 'y'].includes(key)).map(([key, value], index) => {let val: string | null = nulldiff --git a/flowsint-core/src/flowsint_core/core/registry.py b/flowsint-core/src/flowsint_core/core/registry.pyindex 3198932f..564169c1 100644--- a/flowsint-core/src/flowsint_core/core/registry.py+++ b/flowsint-core/src/flowsint_core/core/registry.py@@ -51,7 +51,6 @@ from flowsint_transforms.email.to_gravatar import EmailToGravatarTransformfrom flowsint_transforms.email.to_leaks import EmailToBreachesTransform# Phone-related transforms-from flowsint_transforms.phone.to_leaks import PhoneToBreachesTransform# Individual-related transformsfrom flowsint_transforms.individual.to_org import IndividualToOrgTransform@@ -192,9 +191,6 @@ TransformRegistry.register(EmailToGravatarTransform)TransformRegistry.register(EmailToBreachesTransform)TransformRegistry.register(EmailToDomainsTransform)-# Phone-related transforms-TransformRegistry.register(PhoneToBreachesTransform)-# Individual-related transformsTransformRegistry.register(IndividualToOrgTransform)TransformRegistry.register(IndividualToDomainsTransform)diff --git a/flowsint-core/src/flowsint_core/core/transform_base.py b/flowsint-core/src/flowsint_core/core/transform_base.pyindex 1dd3ebf1..b09384cb 100644--- a/flowsint-core/src/flowsint_core/core/transform_base.py+++ b/flowsint-core/src/flowsint_core/core/transform_base.py@@ -132,7 +132,7 @@ class Transform(ABC):self._graph_service = create_graph_service(sketch_id=self.sketch_id,neo4j_connection=neo4j_conn,- enable_batching=True+ enable_batching=True,)# Params is filled synchronously by the constructor. This params is generally constructed of@@ -355,35 +355,49 @@ class Transform(ABC):def preprocess(self, values: List) -> List:"""Generic preprocess that validates and converts input using InputType.- Automatically handles str, dict, and object inputs, filtering invalids silently.-- Subclasses can override this method for custom preprocessing logic.+ Automatically handles dicts, objects, and strings (using the model's primary field).+ Invalid items are skipped silently."""- # If InputType is not defined, return as-is (backward compatibility)if self.InputType is NotImplemented:return valuesfrom typing import get_args- # Extract base type from List[Type]type_args = get_args(self.InputType)if not type_args:- # If no type args, return as-isreturn valuesbase_type = type_args[0]-- # Use TypeAdapter for validation and conversionadapter = TypeAdapter(base_type)++ # Trouver le champ primaire marqué par Field(..., primary=True)+ primary_field = None+ if issubclass(base_type, BaseModel):+ for name, field in base_type.model_fields.items():+ if field.json_schema_extra and field.json_schema_extra.get("primary"):+ primary_field = name+ break+ if primary_field is None:+ # fallback : premier champ requis ou premier champ disponible+ for name, field in base_type.model_fields.items():+ if field.is_required():+ primary_field = name+ break+ if primary_field is None:+ primary_field = next(iter(base_type.model_fields.keys()))+cleaned = []for item in values:try:- # TypeAdapter handles str, dict, and object automatically+ # Si item est une string, transformer en dict {primary_field: string}+ if isinstance(item, str) and primary_field:+ item = {primary_field: item}+validated = adapter.validate_python(item)cleaned.append(validated)except Exception:- # Skip invalid items silently+ # Ignore les items invalidescontinuereturn cleaned@@ -423,7 +437,7 @@ class Transform(ABC):return []def create_node(- self, node_type: str, key_prop: str, key_value: str, **properties+ self, node_type_or_obj, key_prop=None, key_value=None, **properties) -> None:"""Create a single Neo4j node.@@ -434,24 +448,53 @@ class Transform(ABC):The following properties are automatically added to every node:- type: Lowercase version of node_type- sketch_id: Current sketch ID from transform context- - label: Defaults to key_value if not provided+ - label: Automatically computed by FlowsintType, or defaults to key_value if not provided- created_at: ISO 8601 UTC timestamp (only on creation, not updates)+ Best Practice - Use Pydantic object directly:+ The simplest way is to pass a Pydantic object directly. The node type,+ key property, and key value are automatically inferred:++ ```python+ # Best: pass the Pydantic object directly+ self.create_node(ip)++ # Also good if you need to override properties+ self.create_node(domain, type="subdomain")+ ```+Args:- node_type: Node label (e.g., "domain", "ip")- key_prop: Property name used as unique identifier- key_value: Value of the key property- **properties: Additional node properties+ node_type_or_obj: Either a Pydantic object (FlowsintType), or node label string (e.g., "domain", "ip")+ key_prop: Property name used as unique identifier (optional if passing Pydantic object)+ key_value: Value of the key property (optional if passing Pydantic object)+ **properties: Additional node properties or property overridesNote:Uses MERGE semantics - if a node with the same (key_prop, sketch_id)exists, it will be updated. The created_at field is only set on creation."""+ # Check if first argument is a Pydantic object+ if isinstance(node_type_or_obj, BaseModel):+ obj = node_type_or_obj++ # Infer node_type from class name (e.g., Ip -> "ip", Domain -> "domain")+ node_type = obj.__class__.__name__.lower()++ # Get the primary field and its value+ primary_field = self._get_primary_field(obj)+ key_prop = primary_field+ key_value = getattr(obj, primary_field)++ # Merge object properties with any overrides+ obj_properties = obj.__dict__.copy()+ obj_properties.update(properties)+ properties = obj_properties+ else:+ # Legacy signature: node_type_or_obj is the node_type string+ node_type = node_type_or_obj+self._graph_service.create_node(- node_type=node_type,- key_prop=key_prop,- key_value=key_value,- **properties+ node_type=node_type, key_prop=key_prop, key_value=key_value, **properties)def _serialize_properties(self, properties: dict) -> dict:@@ -468,17 +511,18 @@ class Transform(ABC):Dictionary of serialized properties"""from .graph_serializer import GraphSerializer+return GraphSerializer.serialize_properties(properties)def create_relationship(self,- from_type: str,- from_key: str,- from_value: str,- to_type: str,- to_key: str,- to_value: str,- rel_type: str,+ from_type_or_obj,+ from_key_or_to_obj,+ from_value_or_rel_type=None,+ to_type=None,+ to_key=None,+ to_value=None,+ rel_type=None,) -> None:"""Create a relationship between two nodes.@@ -486,24 +530,95 @@ class Transform(ABC):This method now uses the GraphService for improved performance andbetter separation of concerns.+ Best Practice - Use Pydantic objects directly:+ The simplest way is to pass two Pydantic objects and the relationship type:++ ```python+ # Best: pass Pydantic objects directly+ self.create_relationship(individual, domain, "HAS_DOMAIN")+ self.create_relationship(email, breach, "FOUND_IN_BREACH")+ ```++ Legacy Usage:+ You can still use the explicit signature for backward compatibility:++ ```python+ # Legacy: explicit signature+ self.create_relationship(+ "individual", "full_name", individual.full_name,+ "domain", "domain", domain_name,+ "HAS_DOMAIN"+ )+ ```+Args:- from_type: Source node label- from_key: Source node key property- from_value: Source node key value- to_type: Target node label- to_key: Target node key property- to_value: Target node key value- rel_type: Relationship type+ from_type_or_obj: Either a Pydantic object (source node) or source node label string+ from_key_or_to_obj: Either a Pydantic object (target node) or source node key property+ from_value_or_rel_type: Either relationship type string (if using objects) or source node key value+ to_type: Target node label (only for legacy signature)+ to_key: Target node key property (only for legacy signature)+ to_value: Target node key value (only for legacy signature)+ rel_type: Relationship type (only for legacy signature)"""- self._graph_service.create_relationship(- from_type=from_type,- from_key=from_key,- from_value=from_value,- to_type=to_type,- to_key=to_key,- to_value=to_value,- rel_type=rel_type- )+ # Check if using new signature (Pydantic objects)+ if isinstance(from_type_or_obj, BaseModel) and isinstance(from_key_or_to_obj, BaseModel):+ from_obj = from_type_or_obj+ to_obj = from_key_or_to_obj+ relationship_type = from_value_or_rel_type++ # Extract from_node info+ from_node_type = from_obj.__class__.__name__.lower()+ from_primary_field = self._get_primary_field(from_obj)+ from_key_value = getattr(from_obj, from_primary_field)++ # Extract to_node info+ to_node_type = to_obj.__class__.__name__.lower()+ to_primary_field = self._get_primary_field(to_obj)+ to_key_value = getattr(to_obj, to_primary_field)++ self._graph_service.create_relationship(+ from_type=from_node_type,+ from_key=from_primary_field,+ from_value=from_key_value,+ to_type=to_node_type,+ to_key=to_primary_field,+ to_value=to_key_value,+ rel_type=relationship_type,+ )+ else:+ # Legacy signature+ self._graph_service.create_relationship(+ from_type=from_type_or_obj,+ from_key=from_key_or_to_obj,+ from_value=from_value_or_rel_type,+ to_type=to_type,+ to_key=to_key,+ to_value=to_value,+ rel_type=rel_type,+ )++ def _get_primary_field(self, obj: BaseModel) -> str:+ """Helper method to get the primary field of a Pydantic object."""+ # Access model_fields from the class, not the instance+ model_fields = obj.__class__.model_fields++ # Find the primary field (marked with json_schema_extra={"primary": True})+ primary_field = None+ for field_name, field_info in model_fields.items():+ if field_info.json_schema_extra and field_info.json_schema_extra.get("primary"):+ primary_field = field_name+ break++ # Fallback: use first required field or first field+ if primary_field is None:+ for field_name, field_info in model_fields.items():+ if field_info.is_required():+ primary_field = field_name+ break+ if primary_field is None:+ primary_field = next(iter(model_fields.keys()))++ return primary_fielddef log_graph_message(self, message: str) -> None:"""diff --git a/flowsint-core/src/flowsint_core/core/vault.py b/flowsint-core/src/flowsint_core/core/vault.pyindex 8cc6ef58..34138eb1 100644--- a/flowsint-core/src/flowsint_core/core/vault.py+++ b/flowsint-core/src/flowsint_core/core/vault.py@@ -1,4 +1,3 @@-from ast import Strimport osfrom typing import Protocol, Optionalimport uuiddiff --git a/flowsint-core/tests/test_transform_base_simplified_api.py b/flowsint-core/tests/test_transform_base_simplified_api.pynew file mode 100644index 00000000..abcecbb9--- /dev/null+++ b/flowsint-core/tests/test_transform_base_simplified_api.py@@ -0,0 +1,117 @@+"""Test simplified API for create_node and create_relationship."""+import pytest+from flowsint_core.core.transform_base import Transform+from flowsint_types.domain import Domain+from flowsint_types.email import Email+from flowsint_types.individual import Individual+from typing import List+++class TestTransform(Transform):+ """Simple transform for testing."""++ InputType = List[Domain]+ OutputType = List[Domain]++ @classmethod+ def name(cls) -> str:+ return "test_transform"++ @classmethod+ def category(cls) -> str:+ return "Test"++ @classmethod+ def key(cls) -> str:+ return "domain"++ async def scan(self, data: InputType) -> OutputType:+ return data+++def test_create_node_with_pydantic_object():+ """Test that create_node works with Pydantic objects."""+ transform = TestTransform(sketch_id="test", scan_id="test")++ # Create a domain object+ domain = Domain(domain="example.com")++ # This should not raise an error+ transform.create_node(domain)++ # Verify the helper method works+ primary_field = transform._get_primary_field(domain)+ assert primary_field == "domain"+++def test_create_relationship_with_pydantic_objects():+ """Test that create_relationship works with Pydantic objects."""+ transform = TestTransform(sketch_id="test", scan_id="test")++ # Create objects+ individual = Individual(+ first_name="John",+ last_name="Doe",+ full_name="John Doe"+ )+ domain = Domain(domain="example.com")++ # This should not raise an error+ transform.create_relationship(individual, domain, "HAS_DOMAIN")+++def test_create_node_legacy_signature():+ """Test that legacy create_node signature still works."""+ transform = TestTransform(sketch_id="test", scan_id="test")++ domain = Domain(domain="example.com")++ # Legacy signature should still work+ transform.create_node("domain", "domain", "example.com", **domain.__dict__)+++def test_create_relationship_legacy_signature():+ """Test that legacy create_relationship signature still works."""+ transform = TestTransform(sketch_id="test", scan_id="test")++ # Legacy signature should still work+ transform.create_relationship(+ "individual",+ "full_name",+ "John Doe",+ "domain",+ "domain",+ "example.com",+ "HAS_DOMAIN"+ )+++def test_get_primary_field():+ """Test the _get_primary_field helper method."""+ transform = TestTransform(sketch_id="test", scan_id="test")++ # Test with Domain (has primary field marked)+ domain = Domain(domain="example.com")+ assert transform._get_primary_field(domain) == "domain"++ # Test with Email (has primary field marked)+ email = Email(email="[email protected]")+ assert transform._get_primary_field(email) == "email"++ # Test with Individual (no primary field marked, falls back to first required field)+ individual = Individual(+ first_name="John",+ last_name="Doe",+ full_name="John Doe"+ )+ assert transform._get_primary_field(individual) == "first_name"+++def test_create_node_with_property_override():+ """Test that property overrides work with Pydantic objects."""+ transform = TestTransform(sketch_id="test", scan_id="test")++ domain = Domain(domain="example.com")++ # Should be able to override properties+ transform.create_node(domain, type="subdomain")diff --git a/flowsint-core/tests/transforms/base.py b/flowsint-core/tests/transforms/base.pynew file mode 100644index 00000000..470622b2--- /dev/null+++ b/flowsint-core/tests/transforms/base.py@@ -0,0 +1,71 @@+from typing import List+from flowsint_core.core.transform_base import Transform+from flowsint_types import Phone+from flowsint_types.domain import Domain+from flowsint_types.ip import Ip+++class ResolveTransform(Transform):+ InputType = List[Domain]+ OutputType = List[Ip]++ @classmethod+ def name(cls) -> str:+ return "domain_to_ip"++ @classmethod+ def category(cls) -> str:+ return "Domain"++ @classmethod+ def key(cls) -> str:+ return "domain"++ async def scan(self, data: InputType) -> OutputType:+ return []++ def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ return []+++# Make types available at module level for easy access+InputType = ResolveTransform.InputType+OutputType = ResolveTransform.OutputType++transform = ResolveTransform("sketch_123", "scan_123")+++def test_correct_preprocess():+ inputs = [+ Domain(domain="mydomain.com"),+ {"domain": "blog.mydomain2.com"},+ "mydomain3.com",+ "notADomaiN",+ ]+ preprocessed = transform.preprocess(inputs)+ assert len(preprocessed) == 3 # 3 valid domains+ assert preprocessed[0].domain == "mydomain.com"+ assert preprocessed[0].label == "mydomain.com"+ assert preprocessed[0].root == True++ assert preprocessed[1].domain == "blog.mydomain2.com"+ assert preprocessed[1].label == "blog.mydomain2.com"+ assert preprocessed[1].root == False++ assert preprocessed[2].domain == "mydomain3.com"+ assert preprocessed[2].label == "mydomain3.com"+ assert preprocessed[2].root == True+++def test_incorrect_preprocess():+ inputs = [+ Phone(number="+33634565423"),+ {"name": "JohnDoe"},+ "mydomain.com",+ "notADomaiN",+ ]+ preprocessed = transform.preprocess(inputs)+ assert len(preprocessed) == 1 # 1 valid domain+ assert preprocessed[0].domain == "mydomain.com"+ assert preprocessed[0].root == True+ assert preprocessed[0].label == "mydomain.com"diff --git a/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.py b/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.pyindex 251613fb..75f9b10a 100644--- a/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.py+++ b/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.py@@ -62,26 +62,6 @@ class AsnToCidrsTransform(Transform):def key(cls) -> str:return "number"- def preprocess(- self, data: Union[List[str], List[int], List[dict], InputType]- ) -> InputType:- cleaned: InputType = []- for item in data:- asn_obj = None- try:- if isinstance(item, (str, int)):- asn_obj = ASN(number=parse_asn(str(item)))- elif isinstance(item, dict) and "number" in item:- asn_obj = ASN(number=parse_asn(str(item["number"])))- elif isinstance(item, ASN):- asn_obj = item- if asn_obj and is_valid_asn(str(asn_obj.number)):- cleaned.append(asn_obj)- except ValueError:- Logger.warn(self.sketch_id, {"message": f"Invalid ASN format: {item}"})- continue- return cleaned-async def scan(self, data: InputType) -> OutputType:"""Find CIDR from ASN using asnmap."""cidrs: OutputType = []@@ -145,33 +125,11 @@ class AsnToCidrsTransform(Transform):if str(cidr.network) == "0.0.0.0/0":continue # Skip default CIDR for unknown ASNif self.neo4j_conn:- self.create_node(- "asn",- "number",- asn.number,- label=f"AS{asn.number}",- caption=f"AS{asn.number}",- type="asn",- )+ self.create_node(asn)- self.create_node(- "cidr",- "network",- str(cidr.network),- label=str(cidr.network),- caption=str(cidr.network),- type="cidr",- )+ self.create_node(cidr)- self.create_relationship(- "asn",- "number",- asn.number,- "cidr",- "network",- str(cidr.network),- "ANNOUNCES",- )+ self.create_relationship(asn, cidr, "ANNOUNCES")self.log_graph_message(f"AS{asn.number} announces CIDR {cidr.network}"@@ -200,15 +158,9 @@ class AsnToCidrsTransform(Transform):type="cidr",)- self.create_relationship(- "asn",- "number",- asn.number,- "cidr",- "network",- str(cidr.network),- "ANNOUNCES",- )+ asn_obj = ASN(number=asn.number)+ cidr_obj = CIDR(network=str(cidr.network))+ self.create_relationship(asn_obj, cidr_obj, "ANNOUNCES")self.log_graph_message(f"AS{asn.number} announces CIDR {cidr.network}"diff --git a/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.py b/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.pyindex 237701f3..bf1599e0 100644--- a/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.py+++ b/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.py@@ -60,24 +60,6 @@ class CidrToIpsTransform(Transform):def category(cls) -> str:return "Cidr"- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- cidr_obj = None- try:- if isinstance(item, str):- cidr_obj = CIDR(network=item)- elif isinstance(item, dict) and "network" in item:- cidr_obj = CIDR(network=item["network"])- elif isinstance(item, CIDR):- cidr_obj = item- if cidr_obj:- cleaned.append(cidr_obj)- except ValueError:- Logger.warn(self.sketch_id, {"message": f"Invalid CIDR format: {item}"})- continue- return cleaned-async def scan(self, data: InputType) -> OutputType:"""Find IP addresses from CIDR using mapcidr."""ips: OutputType = []@@ -141,69 +123,25 @@ class CidrToIpsTransform(Transform):for ip in ip_list:if self.neo4j_conn:# Create CIDR node- self.create_node(- "cidr",- "network",- str(cidr.network),- label=str(cidr.network),- caption=str(cidr.network),- type="cidr",- )+ self.create_node(cidr)# Create IP node- self.create_node(- "ip",- "address",- ip.address,- label=ip.address,- caption=ip.address,- type="ip",- )+ self.create_node(ip)# Create relationship- self.create_relationship(- "cidr",- "network",- str(cidr.network),- "ip",- "address",- ip.address,- "CONTAINS",- )+ self.create_relationship(cidr, ip, "CONTAINS")else:# Fallback: original behavior (one-to-one zip)for cidr, ip in zip(original_input, results):if self.neo4j_conn:# Create CIDR node- self.create_node(- "cidr",- "network",- str(cidr.network),- label=str(cidr.network),- caption=str(cidr.network),- type="cidr",- )-+ self.create_node(cidr)# Create IP node- self.create_node(- "ip",- "address",- ip.address,- label=ip.address,- caption=ip.address,- type="ip",- )-+ self.create_node(ip)# Create relationship- self.create_relationship(- "cidr",- "network",- str(cidr.network),- "ip",- "address",- ip.address,- "CONTAINS",- )+ cidr_obj = CIDR(network=str(cidr.network))+ ip_obj = Ip(address=ip.address)+ self.create_relationship(cidr_obj, ip_obj, "CONTAINS")self.log_graph_message(f"CIDR {cidr.network} contains IP {ip.address}"diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.pyindex e969735d..4e7459cc 100644--- a/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py@@ -73,24 +73,12 @@ class CryptoWalletAddressToNFTs(Transform):def key(cls) -> str:return "address"- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- wallet_obj = None- if isinstance(item, str):- wallet_obj = CryptoWallet(address=item)- elif isinstance(item, dict) and "address" in item:- wallet_obj = CryptoWallet(address=item["address"])- elif isinstance(item, CryptoWallet):- wallet_obj = item- if wallet_obj:- cleaned.append(wallet_obj)- return cleaned-async def scan(self, data: InputType) -> OutputType:results: OutputType = []api_key = self.get_secret("ETHERSCAN_API_KEY", os.getenv("ETHERSCAN_API_KEY"))- api_url = self.get_params().get("ETHERSCAN_API_URL", "https://api.etherscan.io/v2/api")+ api_url = self.get_params().get(+ "ETHERSCAN_API_URL", "https://api.etherscan.io/v2/api"+ )for d in data:try:nfts = self._get_nfts(d.address, api_key, api_url)@@ -137,40 +125,22 @@ class CryptoWalletAddressToNFTs(Transform):for nfts in results:for nft in nfts:# Create or update wallet node- self.create_node(- "cryptowallet",- "wallet",- nft.wallet.address,- caption=nft.wallet.address,- type="cryptowallet",- )-+ self.create_node(nft.wallet)# Create or update NFT nodenft_key = f"{nft.contract_address}_{nft.token_id}"- self.create_node(- "nft",- "nft_id",- nft_key,+ self.create_node(nft)+ # Create relationship from wallet to NFT+ wallet_obj = CryptoWallet(address=nft.wallet.address)+ nft_obj = CryptoNFT(+ wallet=wallet_obj,contract_address=nft.contract_address,token_id=nft.token_id,collection_name=nft.collection_name,metadata_url=nft.metadata_url,image_url=nft.image_url,name=nft.name,- caption=nft.name,- type="nft",- )-- # Create relationship from wallet to NFT- self.create_relationship(- "cryptowallet",- "wallet",- nft.wallet.address,- "nft",- "nft_id",- nft_key,- "OWNS",)+ self.create_relationship(wallet_obj, nft_obj, "OWNS")self.log_graph_message(f"Found NFT for {nft.wallet.address}: {nft.contract_address} - {nft.token_id}")diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.pyindex b520b051..50689812 100644--- a/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py@@ -79,20 +79,6 @@ class CryptoWalletAddressToTransactions(Transform):def key(cls) -> str:return "address"- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- wallet_obj = None- if isinstance(item, str):- wallet_obj = CryptoWallet(address=item)- elif isinstance(item, dict) and "address" in item:- wallet_obj = CryptoWallet(address=item["address"])- elif isinstance(item, CryptoWallet):- wallet_obj = item- if wallet_obj:- cleaned.append(wallet_obj)- return cleaned-async def scan(self, data: InputType) -> OutputType:results: OutputType = []api_key = self.get_secret("ETHERSCAN_API_KEY", os.getenv("ETHERSCAN_API_KEY"))@@ -194,21 +180,8 @@ class CryptoWalletAddressToTransactions(Transform):for transactions in results:for tx in transactions:# Create or update both wallet nodes- self.create_node(- "cryptowallet",- "wallet",- tx.source.address,- caption=tx.source.address,- type="cryptowallet",- )- self.create_node(- "cryptowallet",- "wallet",- tx.target.address,- caption=tx.target.address,- type="cryptowallet",- )-+ self.create_node(tx.source)+ self.create_node(tx.target.address)# Create transaction as an edge between wallets (keeping complex query for transaction properties)tx_query = """MATCH (source:cryptowallet {wallet: $source})diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_asn.py b/flowsint-transforms/src/flowsint_transforms/domain/to_asn.pyindex 83c6917e..3463077e 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_asn.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_asn.py@@ -63,22 +63,6 @@ class DomainToAsnTransform(Transform):def key(cls) -> str:return "domain"- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- domain_obj = None- if isinstance(item, str):- if is_valid_domain(item):- domain_obj = Domain(domain=item)- elif isinstance(item, dict) and "domain" in item:- if is_valid_domain(item["domain"]):- domain_obj = Domain(domain=item["domain"])- elif isinstance(item, Domain):- domain_obj = item- if domain_obj:- cleaned.append(domain_obj)- return cleaned-async def scan(self, data: InputType) -> OutputType:results: OutputType = []asnmap = AsnmapTool()@@ -135,35 +119,12 @@ class DomainToAsnTransform(Transform):if input_data and self.neo4j_conn:for domain, asn in zip(input_data, results):# Create domain node- self.create_node(- "domain",- "domain",- domain.domain,- label=domain.domain,- caption=domain.domain,- type="domain",- )-+ self.create_node(domain)# Create ASN node- self.create_node(- "asn",- "number",- asn.number,- label=f"AS{asn.number}",- caption=f"AS{asn.number}",- type="asn",- )+ self.create_node(asn)# Create relationship- self.create_relationship(- "domain",- "domain",- domain.domain,- "asn",- "number",- asn.number,- "HOSTED_IN",- )+ self.create_relationship(domain, asn, "HOSTED_IN")self.log_graph_message(f"Domain {domain.domain} is hosted in AS{asn.number} ({asn.name})"diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_history.py b/flowsint-transforms/src/flowsint_transforms/domain/to_history.pyindex ed3d895c..f6d07003 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_history.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_history.py@@ -6,6 +6,8 @@ from flowsint_core.core.transform_base import Transformfrom flowsint_types.domain import Domainfrom flowsint_types.individual import Individualfrom flowsint_types.organization import Organization+from flowsint_types.email import Email+from flowsint_types.phone import Phonefrom flowsint_core.utils import is_valid_domain, is_root_domainfrom flowsint_types.address import Locationfrom flowsint_core.core.logger import Logger@@ -67,25 +69,6 @@ class DomainToHistoryTransform(Transform):def key(cls) -> str:return "domain"- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- domain_obj = None- if isinstance(item, str):- domain_obj = Domain(domain=item, root=is_root_domain(item))- elif isinstance(item, dict) and "domain" in item:- domain_obj = Domain(- domain=item["domain"], root=is_root_domain(item["domain"])- )- elif isinstance(item, Domain):- # If the Domain object already exists, update its root field- domain_obj = Domain(- domain=item.domain, root=is_root_domain(item.domain)- )- if domain_obj and is_valid_domain(domain_obj.domain):- cleaned.append(domain_obj)- return cleaned-async def scan(self, data: InputType) -> OutputType:"""Find infos related to domains using whoxy api."""domains: OutputType = []@@ -452,25 +435,13 @@ class DomainToHistoryTransform(Transform):self.sketch_id,{"message": f"[WHOXY] Creating domain node: {domain_name}"},)- self.create_node(- "domain",- "domain",- domain_name,- label=domain_name,- caption=domain_name,- type="domain",- )+ domain_obj = Domain(domain=domain_name)+ self.create_node(domain_obj)# Create relationship between original domain and found domain- self.create_relationship(- "domain",- "domain",- original_domain_name,- "domain",- "domain",- domain_name,- "HAS_RELATED_DOMAIN",- )+ original_domain_obj = Domain(domain=original_domain_name)+ domain_obj_rel = Domain(domain=domain_name)+ self.create_relationship(original_domain_obj, domain_obj_rel, "HAS_RELATED_DOMAIN")# Create individual node if not already processedindividual_id = (@@ -484,24 +455,11 @@ class DomainToHistoryTransform(Transform):"message": f"[WHOXY] Creating individual node: {individual.full_name}"},)- self.create_node(- "individual",- "full_name",- individual.full_name,- caption=individual.full_name,- type="individual",- )+ self.create_node(individual)# Create relationship between individual and domain- self.create_relationship(- "individual",- "full_name",- individual.full_name,- "domain",- "domain",- domain_name,- f"IS_{contact_type.upper()}_CONTACT",- )+ domain_obj_contact = Domain(domain=domain_name)+ self.create_relationship(individual, domain_obj_contact, f"IS_{contact_type.upper()}_CONTACT")# Process email addressesif individual.email_addresses:@@ -513,22 +471,9 @@ class DomainToHistoryTransform(Transform):self.sketch_id,{"message": f"[WHOXY] Creating email node: {email_str}"},)- self.create_node(- "email",- "email",- email_str,- caption=email_str,- type="email",- )- self.create_relationship(- "individual",- "full_name",- individual.full_name,- "email",- "email",- email_str,- "HAS_EMAIL",- )+ email_node = Email(email=email_str)+ self.create_node(email_node)+ self.create_relationship(individual, email_node, "HAS_EMAIL")# Process phone numbersif individual.phone_numbers:@@ -540,22 +485,9 @@ class DomainToHistoryTransform(Transform):self.sketch_id,{"message": f"[WHOXY] Creating phone node: {phone_str}"},)- self.create_node(- "phone",- "number",- phone_str,- caption=phone_str,- type="phone",- )- self.create_relationship(- "individual",- "full_name",- individual.full_name,- "phone",- "number",- phone_str,- "HAS_PHONE",- )+ phone_node = Phone(number=phone_str)+ self.create_node(phone_node)+ self.create_relationship(individual, phone_node, "HAS_PHONE")# Process physical address from contact datacontact_data = individual_info["contact_data"]@@ -572,22 +504,8 @@ class DomainToHistoryTransform(Transform):"message": f"[WHOXY] Creating address node: {address.address}"},)- self.create_node(- "location",- "address",- address.address,- caption=f"{address.address}, {address.city}",- type="location",- )- self.create_relationship(- "individual",- "full_name",- individual.full_name,- "location",- "address",- address.address,- "LIVES_AT",- )+ self.create_node(address)+ self.create_relationship(individual, address, "LIVES_AT")self.log_graph_message(f"Processed individual {individual.full_name} ({contact_type}) for domain {domain_name}"@@ -614,25 +532,13 @@ class DomainToHistoryTransform(Transform):self.sketch_id,{"message": f"[WHOXY] Creating domain node: {domain_name}"},)- self.create_node(- "domain",- "domain",- domain_name,- label=domain_name,- caption=domain_name,- type="domain",- )+ domain_obj = Domain(domain=domain_name)+ self.create_node(domain_obj)# Create relationship between original domain and found domain- self.create_relationship(- "domain",- "domain",- original_domain_name,- "domain",- "domain",- domain_name,- "HAS_RELATED_DOMAIN",- )+ original_domain_obj3 = Domain(domain=original_domain_name)+ domain_obj_rel3 = Domain(domain=domain_name)+ self.create_relationship(original_domain_obj3, domain_obj_rel3, "HAS_RELATED_DOMAIN")# Create organization node if not already processedif organization.name not in processed_organizations:@@ -643,24 +549,11 @@ class DomainToHistoryTransform(Transform):"message": f"[WHOXY] Creating organization node: {organization.name}"},)- self.create_node(- "organization",- "name",- organization.name,- caption=organization.name,- type="organization",- )+ self.create_node(organization)# Create relationship between organization and domain- self.create_relationship(- "organization",- "name",- organization.name,- "domain",- "domain",- domain_name,- f"IS_{contact_type.upper()}_CONTACT",- )+ domain_obj_org = Domain(domain=domain_name)+ self.create_relationship(organization, domain_obj_org, f"IS_{contact_type.upper()}_CONTACT")self.log_graph_message(f"Processed organization {organization.name} ({contact_type}) for domain {domain_name}"diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py b/flowsint-transforms/src/flowsint_transforms/domain/to_ip.pyindex 8ac3eae7..55cc885b 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py@@ -1,16 +1,14 @@import socket-from typing import List, Union+from typing import Listfrom flowsint_core.core.logger import Loggerfrom flowsint_core.core.transform_base import Transformfrom flowsint_types.domain import Domainfrom flowsint_types.ip import Ip-from flowsint_core.utils import is_valid_domain, is_root_domainclass ResolveTransform(Transform):"""Resolve domain names to IP addresses."""- # Define the input and output types as class attributesInputType = List[Domain]OutputType = List[Ip]@@ -29,253 +27,7 @@ class ResolveTransform(Transform):@classmethoddef documentation(cls) -> str:"""Return formatted markdown documentation for the domain resolver transform."""- return """- # Domain Resolver Transform-- Resolve domain names to their corresponding IP addresses using DNS queries. This transform performs forward DNS resolution to discover the IP addresses associated with domain names and subdomains.-- ## Overview-- The Domain Resolver Transform takes domain names as input and returns their resolved IP addresses. It automatically handles different input formats and validates domains before attempting resolution.-- ## Input/Output Types-- - **Input**: `List[Domain]` - Array of domain objects to resolve- - **Output**: `List[Ip]` - Array of resolved IP addresses-- ## Input Format Support-- The transform accepts multiple input formats and automatically converts them:-- ### String Format-- ```python- ["example.com", "subdomain.example.com"]- ```-- ### Dictionary Format-- ```python- [- {"domain": "example.com"},- {"domain": "subdomain.example.com"}- ]- ```-- ### Domain Object Format-- ```python- [- Domain(domain="example.com", root=True),- Domain(domain="subdomain.example.com", root=False)- ]- ```-- ## Resolution Process-- ### 1. Input Validation-- - Validates domain format using built-in validation- - Determines if domain is root domain or subdomain- - Filters out invalid domains-- ### 2. DNS Resolution-- - Uses Python's `socket.gethostbyname()` for DNS queries- - Resolves each domain to its primary A record- - Handles resolution errors gracefully-- ### 3. Result Storage-- - Stores domain-to-IP relationships in Neo4j graph database- - Creates nodes for both domains and IP addresses- - Establishes `RESOLVES_TO` relationships-- ## Example Usage-- ### Basic Domain Resolution-- **Input:**- ```json- [- "google.com",- "github.com",- "stackoverflow.com"- ]- ```-- **Expected Output:**- ```json- [- {"address": "142.250.191.14"},- {"address": "140.82.113.4"},- {"address": "151.101.193.69"}- ]- ```-- ### Mixed Input Types-- **Input:**- ```json- [- "example.com",- {"domain": "subdomain.example.com"},- {"domain": "api.example.com", "root": false}- ]- ```-- ## Graph Database Storage-- ### Node Creation-- **Domain Node:**- ```cypher- MERGE (d:domain {domain: "example.com"})- SET d.sketch_id = "sketch-uuid",- d.label = "example.com",- d.type = "domain" // or "subdomain"- ```-- **IP Node:**- ```cypher- MERGE (ip:ip {address: "93.184.216.34"})- SET ip.sketch_id = "sketch-uuid",- ip.label = "93.184.216.34",- ip.type = "ip"- ```-- ### Relationship Creation-- ```cypher- MERGE (d)-[:RESOLVES_TO {sketch_id: "sketch-uuid"}]->(ip)- ```-- ## Domain Type Classification-- The transform automatically classifies domains:-- - **Root Domain**: `example.com` → `type: "domain"`- - **Subdomain**: `api.example.com` → `type: "subdomain"`-- ## Error Handling-- ### Resolution Failures-- When DNS resolution fails, the transform:-- - Logs the error with domain name- - Continues processing remaining domains- - Does not create nodes for failed resolutions-- Common resolution failures:-- - **NXDOMAIN**: Domain does not exist- - **Timeout**: DNS server not responding- - **Network errors**: Connectivity issues-- ### Invalid Input Handling-- The transform filters out:-- - Malformed domain names- - Empty strings- - Non-string, non-dict, non-Domain inputs-- ## Performance Considerations-- ### Resolution Speed-- - Sequential DNS queries (not parallelized)- - Typical resolution time: 10-100ms per domain- - Consider batch size for large domain lists-- ### DNS Caching-- - Relies on system DNS cache- - Results may vary based on TTL values- - Fresh queries may take longer than cached ones-- ## Use Cases-- ### Investigation Scenarios-- 1. **Domain Enumeration**: Resolve discovered subdomains to find active hosts- 2. **Infrastructure Mapping**: Map domain-to-IP relationships for target organization- 3. **CDN Detection**: Identify content delivery network usage patterns- 4. **IP Pivoting**: Find shared hosting infrastructure across domains-- ### Workflow Integration-- ```- [Domain Discovery] → [Domain Resolver] → [IP Geolocation]- → [Port Transform]- → [ASN Lookup]- ```-- ## Security Considerations-- - **DNS Leakage**: Resolution queries may be logged by DNS providers- - **Rate Limiting**: Some DNS servers may rate limit queries- - **Privacy**: Consider using secure DNS (DoH/DoT) for sensitive investigations-- ## Limitations-- - **IPv4 Only**: Currently resolves only A records (IPv4)- - **Single IP**: Returns only the first resolved IP address- - **No CNAME Following**: Does not follow CNAME chains- - **No Cache Control**: Cannot force fresh DNS queries-- ## Troubleshooting-- ### Common Issues-- 1. **No Results**: Check domain validity and DNS configuration- 2. **Timeouts**: Verify network connectivity and DNS server availability- 3. **Partial Results**: Some domains may fail while others succeed-- ### Debug Information-- The transform provides logging for:- - Input validation results- - DNS resolution attempts- - Graph database operations- - Error conditions-- Check Flowsint logs for detailed resolution information.-- ## Technical Details-- ### DNS Query Method-- - Uses Python's standard library `socket.gethostbyname()`- - Follows system DNS configuration- - Respects `/etc/hosts` file entries on Unix systems-- ### Graph Integration-- - Creates typed nodes in Neo4j- - Maintains investigation context via `sketch_id`- - Supports graph traversal for relationship analysis- """-- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- domain_obj = None- if isinstance(item, str):- domain_obj = Domain(domain=item, root=is_root_domain(item))- elif isinstance(item, dict) and "domain" in item:- domain_obj = Domain(- domain=item["domain"], root=is_root_domain(item["domain"])- )- elif isinstance(item, Domain):- # If the Domain object already exists, update its root field- domain_obj = Domain(- domain=item.domain, root=is_root_domain(item.domain)- )- if domain_obj and is_valid_domain(domain_obj.domain):- cleaned.append(domain_obj)- return cleaned+ return ""async def scan(self, data: InputType) -> OutputType:results: OutputType = []@@ -293,20 +45,11 @@ class ResolveTransform(Transform):def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:for domain_obj, ip_obj in zip(original_input, results):- self.create_node(- "domain",- "domain",- domain_obj.domain,- type="domain",- )- self.create_node("ip", "address", ip_obj.address, **ip_obj.__dict__)+ self.create_node(domain_obj)+ self.create_node(ip_obj)self.create_relationship(- "domain",- "domain",- domain_obj.domain,- "ip",- "address",- ip_obj.address,+ domain_obj,+ ip_obj,"RESOLVES_TO",)self.log_graph_message(@@ -315,6 +58,5 @@ class ResolveTransform(Transform):return results-# Make types available at module level for easy accessInputType = ResolveTransform.InputTypeOutputType = ResolveTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py b/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.pyindex c5e99eee..73fee128 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py@@ -28,22 +28,6 @@ class DomainToRootDomain(Transform):def key(cls) -> str:return "domain"- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- domain_obj = None- if isinstance(item, str):- if is_valid_domain(item):- domain_obj = Domain(domain=item)- elif isinstance(item, dict) and "domain" in item:- if is_valid_domain(item["domain"]):- domain_obj = Domain(domain=item["domain"])- elif isinstance(item, Domain):- domain_obj = item- if domain_obj:- cleaned.append(domain_obj)- return cleaned-async def scan(self, data: InputType) -> OutputType:results: OutputType = []self.domain_root_mapping = [] # Reset mapping@@ -73,22 +57,13 @@ class DomainToRootDomain(Transform):if not self.neo4j_conn:continue- # Create root domain node- self.create_node("domain", "domain", root_domain.domain, **root_domain.__dict__)-- # Create original domain node- self.create_node("domain", "domain", original_domain.domain, **original_domain.__dict__)+ # New simplified pattern: pass Pydantic objects directly+ # Override type when needed+ self.create_node(root_domain)+ self.create_node(original_domain)# Create relationship from root domain to original domain- self.create_relationship(- "domain",- "domain",- root_domain.domain,- "domain",- "domain",- original_domain.domain,- "HAS_SUBDOMAIN",- )+ self.create_relationship(root_domain, original_domain, "HAS_SUBDOMAIN")self.log_graph_message(f"{root_domain.domain} -> HAS_SUBDOMAIN -> {original_domain.domain}"diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py b/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.pyindex 4efb9272..6f0b6013 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py@@ -12,7 +12,7 @@ class SubdomainTransform(Transform):"""Transform to find subdomains associated with a domain."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[Domain | str]+ InputType = List[Domain]OutputType = List[Domain]@classmethod@@ -27,20 +27,6 @@ class SubdomainTransform(Transform):def key(cls) -> str:return "domain"- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- domain_obj = None- if isinstance(item, str):- domain_obj = Domain(domain=item)- elif isinstance(item, dict) and "domain" in item:- domain_obj = Domain(domain=item["domain"])- elif isinstance(item, Domain):- domain_obj = item- if domain_obj and is_valid_domain(domain_obj.domain):- cleaned.append(domain_obj)- return cleaned-async def scan(self, data: InputType) -> OutputType:"""Find subdomains using subfinder (Docker) or fallback to crt.sh."""domains: OutputType = []@@ -115,18 +101,12 @@ class SubdomainTransform(Transform):)# Create subdomain node- self.create_node("domain", "domain", subdomain, domain=subdomain)+ parent_domain_obj = Domain(domain=domain_obj["domain"])+ subdomain_obj = Domain(domain=subdomain)+ self.create_node(subdomain_obj)# Create relationship from parent domain to subdomain- self.create_relationship(- "domain",- "domain",- domain_obj["domain"],- "domain",- "domain",- subdomain,- "HAS_SUBDOMAIN",- )+ self.create_relationship(parent_domain_obj, subdomain_obj, "HAS_SUBDOMAIN")self.log_graph_message(f"{domain_obj['domain']} -> {len(domain_obj['subdomains'])} subdomain(s) found."diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_website.py b/flowsint-transforms/src/flowsint_transforms/domain/to_website.pyindex 271292b4..a90432bf 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_website.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_website.py@@ -26,22 +26,6 @@ class DomainToWebsiteTransform(Transform):def key(cls) -> str:return "domain"- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- domain_obj = None- if isinstance(item, str):- if is_valid_domain(item):- domain_obj = Domain(domain=item)- elif isinstance(item, dict) and "domain" in item:- if is_valid_domain(item["domain"]):- domain_obj = Domain(domain=item["domain"])- elif isinstance(item, Domain):- domain_obj = item- if domain_obj:- cleaned.append(domain_obj)- return cleaned-async def scan(self, data: InputType) -> OutputType:results: OutputType = []for domain in data:@@ -100,34 +84,13 @@ class DomainToWebsiteTransform(Transform):if self.neo4j_conn:# Create domain node- self.create_node(- "domain", "domain", website.domain.domain, type="domain"- )+ self.create_node(website.domain)# Create website node- self.create_node(- "website",- "url",- str(website.url),- active=website.active,- redirects=(- [str(redirect) for redirect in website.redirects]- if website.redirects- else []- ),- type="website",- )+ self.create_node(website)# Create relationship- self.create_relationship(- "domain",- "domain",- website.domain.domain,- "website",- "url",- str(website.url),- "HAS_WEBSITE",- )+ self.create_relationship(website.domain, website, "HAS_WEBSITE")is_active_str = "active" if website.active else "inactive"redirects_str = (diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py b/flowsint-transforms/src/flowsint_transforms/domain/to_whois.pyindex 681a0879..e00128cb 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py@@ -4,6 +4,7 @@ from flowsint_core.core.transform_base import Transformfrom flowsint_types.domain import Domainfrom flowsint_types.whois import Whoisfrom flowsint_types.email import Email+from flowsint_types.organization import Organizationfrom flowsint_core.core.logger import Logger@@ -77,13 +78,18 @@ class WhoisTransform(Transform):elif hasattr(whois_info, "domain_id") and whois_info.domain_id:registry_domain_id = str(whois_info.domain_id)+ # Create organization object if org info is available+ organization = None+ if whois_info.org:+ organization = Organization(name=str(whois_info.org))+whois_obj = Whois(domain=domain,registry_domain_id=registry_domain_id,registrar=(str(whois_info.registrar) if whois_info.registrar else None),- org=str(whois_info.org) if whois_info.org else None,+ organization=organization,city=str(whois_info.city) if whois_info.city else None,country=str(whois_info.country) if whois_info.country else None,email=emails[0] if emails else None,@@ -107,98 +113,62 @@ class WhoisTransform(Transform):continue# Create domain node- self.create_node(- "domain",- "domain",- whois_obj.domain.domain,- root=whois_obj.domain.root,- type="domain",- )+ self.create_node(whois_obj.domain)- # Create whois node+ # Create whois node with custom key (only primitive fields)whois_key = f"{whois_obj.domain.domain}_{self.sketch_id}"+ whois_props = {+ "registry_domain_id": whois_obj.registry_domain_id,+ "registrar": whois_obj.registrar,+ "city": whois_obj.city,+ "country": whois_obj.country,+ "creation_date": whois_obj.creation_date,+ "expiration_date": whois_obj.expiration_date,+ "label": whois_obj.label,+ }+ self.create_node("whois", "whois_id", whois_key, **whois_props)- if whois_obj.registry_domain_id:- whois_label = whois_obj.registry_domain_id- else:- whois_label = whois_obj.domain.domain- self.create_node(- "whois",- "whois_id",- whois_key,- domain=whois_obj.domain.domain,+ # Create relationship between domain and whois+ domain_obj = Domain(domain=whois_obj.domain.domain)+ whois_node = Whois(+ domain=domain_obj,registry_domain_id=whois_obj.registry_domain_id,registrar=whois_obj.registrar,- org=whois_obj.org,city=whois_obj.city,country=whois_obj.country,creation_date=whois_obj.creation_date,expiration_date=whois_obj.expiration_date,- email=whois_obj.email.email if whois_obj.email else None,- label=whois_label,- type="whois",)+ self.create_relationship(domain_obj, whois_node, "HAS_WHOIS")- # Create relationship between domain and whois- self.create_relationship(- "domain",- "domain",- whois_obj.domain.domain,- "whois",- "whois_id",- whois_key,- "HAS_WHOIS",- )-- # Create organization node if org information is available- if whois_obj.org:- self.create_node(- "organization",- "name",- whois_obj.org,- country=whois_obj.country,- founding_date=whois_obj.creation_date,- description=f"Organization from WHOIS data for {whois_obj.domain.domain}",- caption=whois_obj.org,- type="organization",- )+ # Create organization node if organization information is available+ if whois_obj.organization:+ self.create_node(whois_obj.organization)# Create relationship between organization and domain- self.create_relationship(- "organization",- "name",- whois_obj.org,- "domain",- "domain",- whois_obj.domain.domain,- "HAS_DOMAIN",- )+ domain_obj2 = Domain(domain=whois_obj.domain.domain)+ self.create_relationship(whois_obj.organization, domain_obj2, "HAS_DOMAIN")self.log_graph_message(- f"{whois_obj.domain.domain} -> {whois_obj.org} (organization)"+ f"{whois_obj.domain.domain} -> {whois_obj.organization.name} (organization)")+ # Create email node if email information is availableif whois_obj.email:- self.create_node(- "email", "email", whois_obj.email.email, **whois_obj.email.__dict__- )- self.create_relationship(- "whois",- "whois_id",- whois_key,- "email",- "email",- whois_obj.email.email,- "REGISTERED_BY",+ self.create_node(whois_obj.email)+ whois_node2 = Whois(+ domain=Domain(domain=whois_obj.domain.domain),+ registry_domain_id=whois_obj.registry_domain_id,+ registrar=whois_obj.registrar,+ city=whois_obj.city,+ country=whois_obj.country,+ creation_date=whois_obj.creation_date,+ expiration_date=whois_obj.expiration_date,)+ self.create_relationship(whois_node2, whois_obj.email, "REGISTERED_BY")self.log_graph_message(- f"WHOIS for {whois_obj.domain.domain} -> registry_id: {whois_obj.registry_domain_id} registrar: {whois_obj.registrar} org: {whois_obj.org} city: {whois_obj.city} country: {whois_obj.country} creation_date: {whois_obj.creation_date} expiration_date: {whois_obj.expiration_date}"+ f"WHOIS for {whois_obj.domain.domain} -> registry_id: {whois_obj.registry_domain_id} registrar: {whois_obj.registrar} org: {whois_obj.organization.name if whois_obj.organization else None} city: {whois_obj.city} country: {whois_obj.country} creation_date: {whois_obj.creation_date} expiration_date: {whois_obj.expiration_date}")return results---# Make types available at module level for easy access-InputType = WhoisTransform.InputType-OutputType = WhoisTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/email/to_domains.py b/flowsint-transforms/src/flowsint_transforms/email/to_domains.pyindex 33c7a561..5a66a2c3 100644--- a/flowsint-transforms/src/flowsint_transforms/email/to_domains.py+++ b/flowsint-transforms/src/flowsint_transforms/email/to_domains.py@@ -5,6 +5,7 @@ from flowsint_core.core.transform_base import Transformfrom flowsint_types.domain import Domainfrom flowsint_types.individual import Individualfrom flowsint_types.email import Email+from flowsint_types.phone import Phonefrom flowsint_types.address import Locationfrom flowsint_core.core.logger import Loggerfrom flowsint_core.core.graph_db import Neo4jConnection@@ -238,34 +239,15 @@ class EmailToDomainsTransform(Transform):processed_domains.add(domain_name)# Create email node- self.create_node(- "email",- "email",- email.email,- caption=email.email,- type="email",- )+ self.create_node(email)# Create domain node- self.create_node(- "domain",- "domain",- domain_name,- label=domain_name,- caption=domain_name,- type="domain",- )+ domain_obj = Domain(domain=domain_name)+ self.create_node(domain_obj)# Create relationship between email and domain- self.create_relationship(- "email",- "email",- email.email,- "domain",- "domain",- domain_name,- "HAS_REGISTERED_DOMAIN",- )+ domain_obj_email = Domain(domain=domain_name)+ self.create_relationship(email, domain_obj_email, "HAS_REGISTERED_DOMAIN")# Process all contact typesfor contact_type, contact in contacts.items():@@ -314,35 +296,15 @@ class EmailToDomainsTransform(Transform):processed_individuals.add(individual_id)# Create individual node- self.create_node(- "individual",- "full_name",- individual.full_name,- caption=individual.full_name,- type="individual",- )+ self.create_node(individual)# Create relationship between individual and domain- self.create_relationship(- "individual",- "full_name",- individual.full_name,- "domain",- "domain",- domain_name,- f"IS_{contact_type}_CONTACT",- )+ domain_obj_ind = Domain(domain=domain_name)+ self.create_relationship(individual, domain_obj_ind, f"IS_{contact_type}_CONTACT")# Create relationship between individual and email- self.create_relationship(- "individual",- "full_name",- individual.full_name,- "email",- "email",- email_address,- f"WORKS_FOR",- )+ email_obj_ind = Email(email=email_address)+ self.create_relationship(individual, email_obj_ind, "WORKS_FOR")# Process email addressesif individual.email_addresses:@@ -352,24 +314,11 @@ class EmailToDomainsTransform(Transform):processed_emails.add(email_str)# Create email node- self.create_node(- "email",- "email",- email_str,- caption=email_str,- type="email",- )+ email_node = Email(email=email_str)+ self.create_node(email_node)# Create relationship between individual and email- self.create_relationship(- "individual",- "full_name",- individual.full_name,- "email",- "email",- email_str,- "HAS_EMAIL",- )+ self.create_relationship(individual, email_node, "HAS_EMAIL")# Process phone numbersif individual.phone_numbers:@@ -379,24 +328,11 @@ class EmailToDomainsTransform(Transform):processed_phones.add(phone_str)# Create phone node- self.create_node(- "phone",- "number",- phone_str,- caption=phone_str,- type="phone",- )+ phone_node = Phone(number=phone_str)+ self.create_node(phone_node)# Create relationship between individual and phone- self.create_relationship(- "individual",- "full_name",- individual.full_name,- "phone",- "number",- phone_str,- "HAS_PHONE",- )+ self.create_relationship(individual, phone_node, "HAS_PHONE")# Process physical addressaddress = self.__extract_physical_address(contact)@@ -408,24 +344,10 @@ class EmailToDomainsTransform(Transform):processed_addresses.add(address_id)# Create address node- self.create_node(- "location",- "address",- address.address,- caption=f"{address.address}, {address.city}",- type="location",- )+ self.create_node(address)# Create relationship between individual and address- self.create_relationship(- "individual",- "full_name",- individual.full_name,- "location",- "address",- address.address,- "LIVES_AT",- )+ self.create_relationship(individual, address, "LIVES_AT")InputType = EmailToDomainsTransform.InputTypediff --git a/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py b/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.pyindex b6864fd7..2b9299a3 100644--- a/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py+++ b/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py@@ -25,20 +25,6 @@ class EmailToGravatarTransform(Transform):def key(cls) -> str:return "email"- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- email_obj = None- if isinstance(item, str):- email_obj = Email(email=item)- elif isinstance(item, dict) and "email" in item:- email_obj = Email(email=item["email"])- elif isinstance(item, Email):- email_obj = item- if email_obj:- cleaned.append(email_obj)- return cleaned-async def scan(self, data: InputType) -> OutputType:results: OutputType = []@@ -100,26 +86,13 @@ class EmailToGravatarTransform(Transform):for email_obj, gravatar_obj in zip(original_input, results):if not self.neo4j_conn:continue-# Create email node- self.create_node("email", "email", email_obj.email, **email_obj.__dict__)-+ self.create_node(email_obj)# Create gravatar nodegravatar_key = f"{email_obj.email}_{self.sketch_id}"- self.create_node(- "gravatar", "gravatar_id", gravatar_key, **gravatar_obj.__dict__- )-+ self.create_node(gravatar_obj)# Create relationship between email and gravatar- self.create_relationship(- "email",- "email",- email_obj.email,- "gravatar",- "gravatar_id",- gravatar_key,- "HAS_GRAVATAR",- )+ self.create_relationship(email_obj, gravatar_obj, "HAS_GRAVATAR")self.log_graph_message(f"Gravatar found for email {email_obj.email} -> hash: {gravatar_obj.hash}"diff --git a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.pyindex 610ccffe..acd3b7f0 100644--- a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py+++ b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py@@ -73,20 +73,6 @@ class EmailToBreachesTransform(Transform):},]- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- email_obj = None- if isinstance(item, str):- email_obj = Email(email=item)- elif isinstance(item, dict) and "email" in item:- email_obj = Email(email=item["email"])- elif isinstance(item, Email):- email_obj = item- if email_obj:- cleaned.append(email_obj)- return cleaned-async def scan(self, data: InputType) -> OutputType:results: OutputType = []api_key = self.get_secret("HIBP_API_KEY", os.getenv("HIBP_API_KEY"))@@ -150,7 +136,7 @@ class EmailToBreachesTransform(Transform):for email_obj in original_input:if not self.neo4j_conn:continue- self.create_node("email", "email", email_obj.email, **email_obj.__dict__)+ self.create_node(email_obj)# Process all breachesfor email_address, breach_obj in results:@@ -159,25 +145,11 @@ class EmailToBreachesTransform(Transform):# Create breach nodebreach_key = f"{breach_obj.name}_{self.sketch_id}"- self.create_node(- "breach",- "breach_id",- breach_key,- **breach_obj.dict(),- label=breach_obj.name,- type="breach",- )+ self.create_node(breach_obj)# Create relationship between the specific email and this breach- self.create_relationship(- "email",- "email",- email_address,- "breach",- "breach_id",- breach_key,- "FOUND_IN_BREACH",- )+ email_obj = Email(email=email_address)+ self.create_relationship(email_obj, breach_obj, "FOUND_IN_BREACH")self.log_graph_message(f"Breach found for email {email_address} -> {breach_obj.name} ({breach_obj.title})")diff --git a/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py b/flowsint-transforms/src/flowsint_transforms/individual/to_domains.pyindex c04b310b..235a140b 100644--- a/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py+++ b/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py@@ -139,7 +139,9 @@ class IndividualToDomainsTransform(Transform):)return domains- def __get_infos_from_whoxy(self, individual_name: str, api_key: str) -> Dict[str, Any]:+ def __get_infos_from_whoxy(+ self, individual_name: str, api_key: str+ ) -> Dict[str, Any]:infos: Dict[str, Any] = {}whoxy = WhoxyTool()try:@@ -243,9 +245,7 @@ class IndividualToDomainsTransform(Transform):if not all([address, city, zip_code, country]):return None- return Location(- address=address, city=city, zip=zip_code, country=country- )+ return Location(address=address, city=city, zip=zip_code, country=country)def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:"""Create Neo4j nodes and relationships from extracted data."""@@ -269,33 +269,14 @@ class IndividualToDomainsTransform(Transform):if domain_name in processed_domains:continueprocessed_domains.add(domain_name)-# Create individual node- self.create_node(- "individual",- "full_name",- individual.full_name,- **individual.__dict__,- )-+ self.create_node(individual)# Create domain node- self.create_node(- "domain",- "domain",- domain_name,- **domain.__dict__,- )+ self.create_node(domain)# Create relationship between individual and domain- self.create_relationship(- "individual",- "full_name",- individual.full_name,- "domain",- "domain",- domain_name,- "HAS_REGISTERED_DOMAIN",- )+ domain_obj_indiv = Domain(domain=domain_name)+ self.create_relationship(individual, domain_obj_indiv, "HAS_REGISTERED_DOMAIN")# Process all contact typesfor contact_type, contact in contacts.items():@@ -342,36 +323,16 @@ class IndividualToDomainsTransform(Transform):returnprocessed_individuals.add(individual_id)-# Create individual node- self.create_node(- "individual",- "full_name",- contact_individual.full_name,- **contact_individual.__dict__,- )+ self.create_node(contact_individual)# Create relationship between individual and domain- self.create_relationship(- "individual",- "full_name",- contact_individual.full_name,- "domain",- "domain",- domain_name,- f"IS_{contact_type}_CONTACT",- )+ domain_obj_contact = Domain(domain=domain_name)+ self.create_relationship(contact_individual, domain_obj_contact, f"IS_{contact_type}_CONTACT")# Create relationship between contact individual and main individual- self.create_relationship(- "individual",- "full_name",- contact_individual.full_name,- "individual",- "full_name",- individual_name,- f"WORKS_FOR",- )+ main_individual = Individual(first_name="", last_name="", full_name=individual_name)+ self.create_relationship(contact_individual, main_individual, "WORKS_FOR")# Process email addressesif contact_individual.email_addresses:@@ -381,23 +342,11 @@ class IndividualToDomainsTransform(Transform):processed_emails.add(email_str)# Create email node- self.create_node(- "email",- "email",- email_str,- email=email_str,- )+ email_obj = Email(email=email_str)+ self.create_node(email_obj)# Create relationship between individual and email- self.create_relationship(- "individual",- "full_name",- contact_individual.full_name,- "email",- "email",- email_str,- "HAS_EMAIL",- )+ self.create_relationship(contact_individual, email_obj, "HAS_EMAIL")# Process phone numbersif contact_individual.phone_numbers:@@ -407,23 +356,11 @@ class IndividualToDomainsTransform(Transform):processed_phones.add(phone_str)# Create phone node- self.create_node(- "phone",- "number",- phone_str,- number=phone_str,- )+ phone_obj = Phone(number=phone_str)+ self.create_node(phone_obj)# Create relationship between individual and phone- self.create_relationship(- "individual",- "full_name",- contact_individual.full_name,- "phone",- "number",- phone_str,- "HAS_PHONE",- )+ self.create_relationship(contact_individual, phone_obj, "HAS_PHONE")# Process physical addressaddress = self.__extract_physical_address(contact)@@ -435,23 +372,10 @@ class IndividualToDomainsTransform(Transform):processed_addresses.add(address_id)# Create address node- self.create_node(- "location",- "address",- address.address,- address=address,- )+ self.create_node(address)# Create relationship between individual and address- self.create_relationship(- "individual",- "full_name",- contact_individual.full_name,- "location",- "address",- address.address,- "LIVES_AT",- )+ self.create_relationship(contact_individual, address, "LIVES_AT")# Make types available at module level for easy accessdiff --git a/flowsint-transforms/src/flowsint_transforms/individual/to_org.py b/flowsint-transforms/src/flowsint_transforms/individual/to_org.pyindex adb65499..c25c88de 100644--- a/flowsint-transforms/src/flowsint_transforms/individual/to_org.py+++ b/flowsint-transforms/src/flowsint_transforms/individual/to_org.py@@ -293,12 +293,7 @@ class IndividualToOrgTransform(Transform):continueprocessed_organizations.add(org_key)- self.create_node(- "Organization",- "org_id",- org_key,- **org.__dict__,- )+ self.create_node(org)# Then, create all individual nodesfor individual in original_input:@@ -308,12 +303,7 @@ class IndividualToOrgTransform(Transform):processed_individuals.add(individual_id)# Create individual node- self.create_node(- "individual",- "full_name",- individual_id,- **individual.__dict__,- )+ self.create_node(individual)# Finally, create relationships between all individuals and all organizationsfor individual in original_input:@@ -322,15 +312,7 @@ class IndividualToOrgTransform(Transform):org_key = f"{org.name}_FR"# Create relationship between individual and organization- self.create_relationship(- "individual",- "full_name",- individual_id,- "Organization",- "org_id",- org_key,- "WORKS_FOR",- )+ self.create_relationship(individual, org, "WORKS_FOR")self.log_graph_message(f"Created {len(results)} organizations and {len(original_input)} individuals with relationships"diff --git a/flowsint-transforms/src/flowsint_transforms/ip/to_asn.py b/flowsint-transforms/src/flowsint_transforms/ip/to_asn.pyindex 90214072..4024b3e7 100644--- a/flowsint-transforms/src/flowsint_transforms/ip/to_asn.py+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_asn.py@@ -62,22 +62,6 @@ class IpToAsnTransform(Transform):def key(cls) -> str:return "address"- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- ip_obj = None- if isinstance(item, str):- if is_valid_ip(item):- ip_obj = Ip(address=item)- elif isinstance(item, dict) and "address" in item:- if is_valid_ip(item["address"]):- ip_obj = Ip(address=item["address"])- elif isinstance(item, Ip):- ip_obj = item- if ip_obj:- cleaned.append(ip_obj)- return cleaned-async def scan(self, data: InputType) -> OutputType:results: OutputType = []asnmap = AsnmapTool()@@ -130,19 +114,11 @@ class IpToAsnTransform(Transform):if input_data and self.neo4j_conn:for ip, asn in zip(input_data, results):# Create IP node- self.create_node("ip", "address", ip.address, label=ip.address, type="ip", **ip.__dict__)+ self.create_node(ip)# Create ASN node- self.create_node("asn", "number", asn.number, label=f"AS{asn.number}", type="asn", **asn.__dict__)+ self.create_node(asn)# Create relationship- self.create_relationship(- "ip",- "address",- ip.address,- "asn",- "number",- asn.number,- "BELONGS_TO",- )+ self.create_relationship(ip, asn, "BELONGS_TO")self.log_graph_message(f"IP {ip.address} belongs to AS{asn.number} ({asn.name})")diff --git a/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py b/flowsint-transforms/src/flowsint_transforms/ip/to_domain.pyindex 2cecd107..3de35ab4 100644--- a/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py@@ -30,22 +30,6 @@ class ReverseResolveTransform(Transform):def key(cls) -> str:return "address"- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- ip_obj = None- if isinstance(item, str):- if is_valid_ip(item):- ip_obj = Ip(address=item)- elif isinstance(item, dict) and "address" in item:- if is_valid_ip(item["address"]):- ip_obj = Ip(address=item["address"])- elif isinstance(item, Ip):- ip_obj = item- if ip_obj:- cleaned.append(ip_obj)- return cleaned-async def scan(self, data: InputType) -> OutputType:results: OutputType = []@@ -89,29 +73,11 @@ class ReverseResolveTransform(Transform):# Create nodes and relationships for each resolved domainfor ip_obj in original_input:# Create IP node- self.create_node("ip", "address", ip_obj.address, **ip_obj.__dict__)-+ self.create_node(ip_obj)# Create domain nodes and relationships for each resolved domainfor domain_obj in results:- self.create_node(- "domain",- "domain",- domain_obj.domain,- type=(- "domain"- if "." not in domain_obj.domain.split(".")[1:]- else "subdomain"- ),- )- self.create_relationship(- "ip",- "address",- ip_obj.address,- "domain",- "domain",- domain_obj.domain,- "REVERSE_RESOLVES_TO",- )+ self.create_node(domain_obj)+ self.create_relationship(ip_obj, domain_obj, "REVERSE_RESOLVES_TO")self.log_graph_message(f"Domain found for IP {ip_obj.address} -> {domain_obj.domain}")@@ -119,6 +85,5 @@ class ReverseResolveTransform(Transform):return results-# Make types available at module level for easy accessInputType = ReverseResolveTransform.InputTypeOutputType = ReverseResolveTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/ip/to_infos.py b/flowsint-transforms/src/flowsint_transforms/ip/to_infos.pyindex c76270e0..6c5dd14a 100644--- a/flowsint-transforms/src/flowsint_transforms/ip/to_infos.py+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_infos.py@@ -1,12 +1,7 @@import requests-from typing import List, Dict, Any, TypeAlias, Union-from pydantic import TypeAdapter+from typing import List, Dict, Anyfrom flowsint_core.core.transform_base import Transform-from flowsint_types.ip import Ip, Ip-from flowsint_core.utils import resolve_type, is_valid_ip--InputType: TypeAlias = List[Ip]-OutputType: TypeAlias = List[Ip]+from flowsint_types.ip import Ipclass IpToInfosTransform(Transform):@@ -28,60 +23,18 @@ class IpToInfosTransform(Transform):def key(cls) -> str:return "address"- @classmethod- def input_schema(cls) -> Dict[str, Any]:- adapter = TypeAdapter(InputType)- schema = adapter.json_schema()- type_name, details = list(schema["$defs"].items())[0]- return {- "type": type_name,- "properties": [- {"name": prop, "type": resolve_type(info, schema)}- for prop, info in details["properties"].items()- ],- }-- @classmethod- def output_schema(cls) -> Dict[str, Any]:- adapter = TypeAdapter(OutputType)- schema = adapter.json_schema()- type_name, details = list(schema["$defs"].items())[0]- return {- "type": type_name,- "properties": [- {"name": prop, "type": resolve_type(info, schema)}- for prop, info in details["properties"].items()- ],- }-- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- ip_obj = None- if isinstance(item, str):- ip_obj = Ip(address=item)- elif isinstance(item, dict) and "address" in item:- ip_obj = Ip(address=item["address"])- elif isinstance(item, Ip):- ip_obj = item- if ip_obj and is_valid_ip(ip_obj.address):- cleaned.append(ip_obj)- return cleaned-async def scan(self, data: InputType) -> OutputType:results: OutputType = []for ip in data:try:geo_data = self.get_location_data(ip.address)- enriched_ip = Ip(- address=ip.address,- latitude=geo_data.get("latitude"),- longitude=geo_data.get("longitude"),- country=geo_data.get("country"),- city=geo_data.get("city"),- isp=geo_data.get("isp"),- )- results.append(enriched_ip)+ # Enrich the existing IP object with geo data+ ip.latitude = geo_data.get("latitude")+ ip.longitude = geo_data.get("longitude")+ ip.country = geo_data.get("country")+ ip.city = geo_data.get("city")+ ip.isp = geo_data.get("isp")+ results.append(ip)except Exception as e:print(f"Error geolocating {ip.address}: {e}")return results@@ -90,17 +43,7 @@ class IpToInfosTransform(Transform):"""Update IP nodes in Neo4j with geolocation information."""if self.neo4j_conn:for ip in results:- self.create_node(- "ip",- "address",- ip.address,- latitude=ip.latitude,- longitude=ip.longitude,- country=ip.country,- city=ip.city,- isp=ip.isp,- type="ip",- )+ self.create_node(ip)self.log_graph_message(f"Geolocated {ip.address} to {ip.city}, {ip.country} (lat: {ip.latitude}, lon: {ip.longitude})")@@ -129,7 +72,3 @@ class IpToInfosTransform(Transform):except Exception as e:print(f"Failed to geolocate {address}: {e}")return {}---InputType = IpToInfosTransform.InputType-OutputType = IpToInfosTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/ip/to_ports.py b/flowsint-transforms/src/flowsint_transforms/ip/to_ports.pyindex f97f8e07..2b9d2c33 100644--- a/flowsint-transforms/src/flowsint_transforms/ip/to_ports.py+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_ports.py@@ -107,22 +107,6 @@ class IpToPortsTransform(Transform):def key(cls) -> str:return "address"- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- ip_obj = None- if isinstance(item, str):- if is_valid_ip(item):- ip_obj = Ip(address=item)- elif isinstance(item, dict) and "address" in item:- if is_valid_ip(item["address"]):- ip_obj = Ip(**item)- elif isinstance(item, Ip):- ip_obj = item- if ip_obj:- cleaned.append(ip_obj)- return cleaned-async def scan(self, data: InputType) -> OutputType:results: OutputType = []naabu = NaabuTool()@@ -214,34 +198,13 @@ class IpToPortsTransform(Transform):ip_address = getattr(port, "_ip_address", None)if not ip_address:continue-- # Create Port node with composite key (ip:port) to handle multiple IPs+port_id = f"{ip_address}:{port.number}"- port_label = f"{port.number}/{port.protocol}"- self.create_node(- "port",- "id",- port_id,- label=port_label,- type="port",- number=port.number,- protocol=port.protocol,- state=port.state,- service=port.service,- banner=port.banner,- ip_address=ip_address,- )+ self.create_node(port)# Create relationship from IP to Port- self.create_relationship(- "ip",- "address",- ip_address,- "port",- "id",- port_id,- "HAS_PORT",- )+ ip_obj = Ip(address=ip_address)+ self.create_relationship(ip_obj, port, "HAS_PORT")service_info = f" ({port.service})" if port.service else ""self.log_graph_message(diff --git a/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.pydeleted file mode 100644index 75556468..00000000--- a/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.py+++ /dev/null@@ -1,156 +0,0 @@-from typing import Dict, Any, List, Union, Optional-import hibpwned-from flowsint_core.core.transform_base import Transform-from flowsint_core.core.logger import Logger-from flowsint_core.core.graph_db import Neo4jConnection-import os-from dotenv import load_dotenv--# Load environment variables-load_dotenv()--HIBP_API_KEY = os.getenv("HIBP_API_KEY")---class HibpTransform(Transform):- """Queries HaveIBeenPwned for potential leaks."""-- # Define types as class attributes - base class handles schema generation automatically- InputType = List[str] # Email addresses as strings- OutputType = List[Dict[str, Any]] # Breach results as dictionaries-- def __init__(- self,- sketch_id: Optional[str] = None,- scan_id: Optional[str] = None,- neo4j_conn: Optional[Neo4jConnection] = None,- vault=None,- params: Optional[Dict[str, Any]] = None,- ):- super().__init__(- sketch_id=sketch_id,- scan_id=scan_id,- neo4j_conn=neo4j_conn,- params_schema=self.get_params_schema(),- vault=vault,- params=params,- )-- @classmethod- def required_params(cls) -> bool:- return True-- @classmethod- def get_params_schema(cls) -> List[Dict[str, Any]]:- """Declare required parameters for this transform"""- return [- {- "name": "HIBP_API_KEY",- "type": "vaultSecret",- "description": "The HIBP API key to use for breach lookups.",- "required": True,- },- ]-- @classmethod- def name(cls) -> str:- return "to_hibp_leaks"-- @classmethod- def category(cls) -> str:- return "leaks"-- @classmethod- def key(cls) -> str:- return "email"-- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- if isinstance(item, str):- cleaned.append(item)- elif isinstance(item, dict) and "email" in item:- cleaned.append(item["email"])- return cleaned-- async def scan(self, data: InputType) -> OutputType:- """Performs a search on HaveIBeenPwned for a list of emails."""- results: OutputType = []- api_key = self.get_secret("HIBP_API_KEY", os.getenv("HIBP_API_KEY"))-- for email in data:- try:- result = hibpwned.Pwned(email, "MyHIBPChecker", api_key)-- # Clear data structure for results- breaches = result.search_all_breaches()- pastes = result.search_pastes()- password = result.search_password("BadPassword")- hashes = result.search_hashes("21BD1")-- email_result = {- "email": email,- "breaches": breaches if breaches else [],- "adobe": result.single_breach("adobe") or {},- "data": result.data_classes() or [],- "pastes": pastes if pastes else [],- "password": password if password else {},- "hashes": hashes if hashes else [],- }- results.append(email_result)- except Exception as e:- results.append(- {- "email": email,- "error": f"Error during scan: {str(e)}",- }- )- Logger.error(- self.sketch_id,- {"message": f"Error scanning email {email}: {str(e)}"},- )-- return results-- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:- """Create Neo4j relationships for found breaches."""- if not self.neo4j_conn:- return results-- for result in results:- if "error" not in result:- email = result["email"]-- # Create email node- self.create_node("email", "address", email, email=email)-- # Create breach relationships- for breach in result.get("breaches", []):- if breach and isinstance(breach, dict):- breach_name = breach.get("Name", "Unknown")- self.create_node(- "breach",- "name",- breach_name,- caption=breach_name,- type="breach",- )- self.create_relationship(- "email",- "address",- email,- "breach",- "name",- breach_name,- "FOUND_IN_BREACH",- )- self.log_graph_message(- f"Email {email} found in breach: {breach_name}"- )-- return results---# Make types available at module level for easy access-InputType = HibpTransform.InputType-OutputType = HibpTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/organization/__init__.py b/flowsint-transforms/src/flowsint_transforms/organization/__init__.pydeleted file mode 100644index 8b137891..00000000--- a/flowsint-transforms/src/flowsint_transforms/organization/__init__.py+++ /dev/null@@ -1 +0,0 @@-diff --git a/flowsint-transforms/src/flowsint_transforms/organization/to_asn.py b/flowsint-transforms/src/flowsint_transforms/organization/to_asn.pyindex e68486f3..645f4ebb 100644--- a/flowsint-transforms/src/flowsint_transforms/organization/to_asn.py+++ b/flowsint-transforms/src/flowsint_transforms/organization/to_asn.py@@ -60,20 +60,6 @@ class OrgToAsnTransform(Transform):def key(cls) -> str:return "name"- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- org_obj = None- if isinstance(item, str):- org_obj = Organization(name=item)- elif isinstance(item, dict) and "name" in item:- org_obj = Organization(name=item["name"])- elif isinstance(item, Organization):- org_obj = item- if org_obj:- cleaned.append(org_obj)- return cleaned-async def scan(self, data: InputType) -> OutputType:"""Find ASN information for organizations using asnmap."""results: OutputType = []@@ -129,36 +115,13 @@ class OrgToAsnTransform(Transform):if self.neo4j_conn:# Create organization node- self.create_node(- "organization",- "name",- input_org.name,- caption=input_org.name,- type="organization",- )+ self.create_node(input_org)# Create ASN node- self.create_node(- "asn",- "number",- result_asn.number,- name=result_asn.name,- country=result_asn.country,- label=f"AS{result_asn.number}",- caption=f"AS{result_asn.number} - {result_asn.name}",- type="asn",- )+ self.create_node(result_asn)# Create relationship- self.create_relationship(- "organization",- "name",- input_org.name,- "asn",- "number",- result_asn.number,- "BELONGS_TO",- )+ self.create_relationship(input_org, result_asn, "BELONGS_TO")self.log_graph_message(f"Found for {input_org.name} -> ASN {result_asn.number}"diff --git a/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py b/flowsint-transforms/src/flowsint_transforms/organization/to_domains.pyindex 67aa62b6..b46bfd53 100644--- a/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py+++ b/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py@@ -2,6 +2,7 @@ import osimport refrom typing import Any, List, Dict, Set, Optionalfrom flowsint_core.core.transform_base import Transform+from flowsint_types import Emailfrom flowsint_types.domain import Domainfrom flowsint_types.organization import Organizationfrom flowsint_types.individual import Individual@@ -433,13 +434,8 @@ class OrgToDomainsTransform(Transform):self.sketch_id,{"message": f"[WHOXY] Creating organization node: {org_name}"},)- self.create_node(- "organization",- "name",- org_name,- caption=org_name,- type="organization",- )+ org_obj = Organization(name=org_name)+ self.create_node(org_obj)# Create domain node if not already processedif domain_name not in processed_domains:@@ -448,25 +444,13 @@ class OrgToDomainsTransform(Transform):self.sketch_id,{"message": f"[WHOXY] Creating domain node: {domain_name}"},)- self.create_node(- "domain",- "domain",- domain_name,- label=domain_name,- caption=domain_name,- type="domain",- )+ domain_obj = Domain(domain=domain_name)+ self.create_node(domain_obj)# Create relationship between organization and domain- self.create_relationship(- "organization",- "name",- org_name,- "domain",- "domain",- domain_name,- "HAS_REGISTERED_DOMAIN",- )+ org_obj_domain = Organization(name=org_name)+ domain_obj_org = Domain(domain=domain_name)+ self.create_relationship(org_obj_domain, domain_obj_org, "HAS_REGISTERED_DOMAIN")# Create individual node if not already processedindividual_id = (@@ -480,35 +464,15 @@ class OrgToDomainsTransform(Transform):"message": f"[WHOXY] Creating individual node: {individual.full_name}"},)- self.create_node(- "individual",- "full_name",- individual.full_name,- caption=individual.full_name,- type="individual",- )+ self.create_node(individual)# Create relationship between individual and domain- self.create_relationship(- "individual",- "full_name",- individual.full_name,- "domain",- "domain",- domain_name,- f"IS_{contact_type.upper()}_CONTACT",- )+ domain_obj_ind = Domain(domain=domain_name)+ self.create_relationship(individual, domain_obj_ind, f"IS_{contact_type.upper()}_CONTACT")# Create relationship between individual and organization- self.create_relationship(- "individual",- "full_name",- individual.full_name,- "organization",- "name",- org_name,- "WORKS_FOR",- )+ org_obj_ind = Organization(name=org_name)+ self.create_relationship(individual, org_obj_ind, "WORKS_FOR")# Process email addressesif individual.email_addresses:@@ -520,22 +484,9 @@ class OrgToDomainsTransform(Transform):self.sketch_id,{"message": f"[WHOXY] Creating email node: {email_str}"},)- self.create_node(- "email",- "email",- email_str,- caption=email_str,- type="email",- )- self.create_relationship(- "individual",- "full_name",- individual.full_name,- "email",- "email",- email_str,- "HAS_EMAIL",- )+ email_obj = Email(email=email_str)+ self.create_node(email_obj)+ self.create_relationship(individual, email_obj, "HAS_EMAIL")# Process phone numbersif individual.phone_numbers:@@ -547,22 +498,9 @@ class OrgToDomainsTransform(Transform):self.sketch_id,{"message": f"[WHOXY] Creating phone node: {phone_str}"},)- self.create_node(- "phone",- "number",- phone_str,- caption=phone_str,- type="phone",- )- self.create_relationship(- "individual",- "full_name",- individual.full_name,- "phone",- "number",- phone_str,- "HAS_PHONE",- )+ phone_obj = Phone(number=phone_str)+ self.create_node(phone_obj)+ self.create_relationship(individual, phone_obj, "HAS_PHONE")# Process physical address from contact datacontact_data = individual_info["contact_data"]@@ -579,22 +517,8 @@ class OrgToDomainsTransform(Transform):"message": f"[WHOXY] Creating address node: {address.address}"},)- self.create_node(- "location",- "address",- address.address,- caption=f"{address.address}, {address.city}",- type="location",- )- self.create_relationship(- "individual",- "full_name",- individual.full_name,- "location",- "address",- address.address,- "LIVES_AT",- )+ self.create_node(address)+ self.create_relationship(individual, address, "LIVES_AT")self.log_graph_message(f"Processed individual {individual.full_name} ({contact_type}) for domain {domain_name}"@@ -621,13 +545,8 @@ class OrgToDomainsTransform(Transform):self.sketch_id,{"message": f"[WHOXY] Creating organization node: {org_name}"},)- self.create_node(- "organization",- "name",- org_name,- caption=org_name,- type="organization",- )+ org_obj = Organization(name=org_name)+ self.create_node(org_obj)# Create domain node if not already processedif domain_name not in processed_domains:@@ -636,25 +555,13 @@ class OrgToDomainsTransform(Transform):self.sketch_id,{"message": f"[WHOXY] Creating domain node: {domain_name}"},)- self.create_node(- "domain",- "domain",- domain_name,- label=domain_name,- caption=domain_name,- type="domain",- )+ domain_obj = Domain(domain=domain_name)+ self.create_node(domain_obj)# Create relationship between input organization and domain- self.create_relationship(- "organization",- "name",- org_name,- "domain",- "domain",- domain_name,- "HAS_REGISTERED_DOMAIN",- )+ org_obj_domain2 = Organization(name=org_name)+ domain_obj_org2 = Domain(domain=domain_name)+ self.create_relationship(org_obj_domain2, domain_obj_org2, "HAS_REGISTERED_DOMAIN")# Create extracted organization node if not already processedif organization.name not in processed_organizations:@@ -665,24 +572,11 @@ class OrgToDomainsTransform(Transform):"message": f"[WHOXY] Creating organization node: {organization.name}"},)- self.create_node(- "organization",- "name",- organization.name,- caption=organization.name,- type="organization",- )+ self.create_node(organization)# Create relationship between extracted organization and domain- self.create_relationship(- "organization",- "name",- organization.name,- "domain",- "domain",- domain_name,- f"IS_{contact_type.upper()}_CONTACT",- )+ domain_obj_extracted = Domain(domain=domain_name)+ self.create_relationship(organization, domain_obj_extracted, f"IS_{contact_type.upper()}_CONTACT")self.log_graph_message(f"Processed organization {organization.name} ({contact_type}) for domain {domain_name}"diff --git a/flowsint-transforms/src/flowsint_transforms/organization/to_infos.py b/flowsint-transforms/src/flowsint_transforms/organization/to_infos.pyindex 3f786914..bdf72d46 100644--- a/flowsint-transforms/src/flowsint_transforms/organization/to_infos.py+++ b/flowsint-transforms/src/flowsint_transforms/organization/to_infos.py@@ -24,19 +24,6 @@ class OrgToInfosTransform(Transform):def key(cls) -> str:return "name"- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- if not isinstance(data, list):- raise ValueError(f"Expected list input, got {type(data).__name__}")- cleaned: InputType = []- for item in data:- if isinstance(item, str) and item != "":- cleaned.append(Organization(name=item))- elif isinstance(item, dict) and "name" in item and item["name"] != "":- cleaned.append(Organization(**item))- elif isinstance(item, Organization):- cleaned.append(item)- return cleaned-async def scan(self, data: InputType) -> OutputType:results: OutputType = []@@ -266,35 +253,7 @@ class OrgToInfosTransform(Transform):for org in results:# Create or update the organization node with all SIRENE propertiesorg_key = f"{org.name}_FR"- self.create_node(- "Organization",- "org_id",- org_key,- name=org.name,- country="FR",- siren=org.siren,- siege_siret=org.siege_siret,- nom_complet=org.nom_complet,- nom_raison_sociale=org.nom_raison_sociale,- sigle=org.sigle,- nombre_etablissements=org.nombre_etablissements,- nombre_etablissements_ouverts=org.nombre_etablissements_ouverts,- activite_principale=org.activite_principale,- section_activite_principale=org.section_activite_principale,- categorie_entreprise=org.categorie_entreprise,- annee_categorie_entreprise=org.annee_categorie_entreprise,- caractere_employeur=org.caractere_employeur,- tranche_effectif_salarie=org.tranche_effectif_salarie,- annee_tranche_effectif_salarie=org.annee_tranche_effectif_salarie,- date_creation=org.date_creation,- date_fermeture=org.date_fermeture,- date_mise_a_jour=org.date_mise_a_jour,- date_mise_a_jour_insee=org.date_mise_a_jour_insee,- date_mise_a_jour_rne=org.date_mise_a_jour_rne,- nature_juridique=org.nature_juridique,- statut_diffusion=org.statut_diffusion,- type="organization",- )+ self.create_node(org)if org.siren:self.log_graph_message(f"{org.name}: SIREN {org.siren} -> {org.name}")@@ -308,27 +267,9 @@ class OrgToInfosTransform(Transform):# Add dirigeants (leaders) as Individual nodes with relationshipsif org.dirigeants:for dirigeant in org.dirigeants:- self.create_node(- "individual",- "full_name",- dirigeant.full_name,- first_name=dirigeant.first_name,- last_name=dirigeant.last_name,- birth_date=dirigeant.birth_date,- gender=dirigeant.gender,- caption=dirigeant.full_name,- type="individual",- )+ self.create_node(dirigeant)- self.create_relationship(- "organization",- "org_id",- org_key,- "individual",- "full_name",- dirigeant.full_name,- "HAS_LEADER",- )+ self.create_relationship(org, dirigeant, "HAS_LEADER")self.log_graph_message(f"{org.name}: HAS_LEADER -> {dirigeant.full_name}")@@ -337,30 +278,9 @@ class OrgToInfosTransform(Transform):if org.siege_geo_adresse:address = org.siege_geo_adresseaddress_key = f"{address.address}_{address.city}_{address.country}"- self.create_node(- "location",- "address_id",- address_key,- address=address.address,- city=address.city,- country=address.country,- zip=address.zip,- latitude=address.latitude,- longitude=address.longitude,- label=f"{address.address}, {address.city}",- caption=f"{address.address}, {address.city}",- type="location",- )+ self.create_node(address)- self.create_relationship(- "organization",- "org_id",- org_key,- "location",- "address_id",- address_key,- "HAS_ADDRESS",- )+ self.create_relationship(org, address, "HAS_ADDRESS")self.log_graph_message(f"{org.name}: HAS_ADDRESS -> {address.address}, {address.city}")@@ -383,15 +303,15 @@ class OrgToInfosTransform(Transform):type="location",)- self.create_relationship(- "organization",- "org_id",- org_key,- "Location",- "location_id",- location_key,- "LOCATED_AT",+ location_obj = Location(+ latitude=float(org.siege_latitude),+ longitude=float(org.siege_longitude),+ address=org.siege_adresse,+ city=org.siege_libelle_commune,+ country="FR",+ zip=org.siege_code_postal,)+ self.create_relationship(org, location_obj, "LOCATED_AT")self.log_graph_message(f"{org.name}: LOCATED_AT -> {org.siege_libelle_commune or 'Unknown'}")diff --git a/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py b/flowsint-transforms/src/flowsint_transforms/phone/to_infos.pyindex ca49d60b..202b4552 100644--- a/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py+++ b/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py@@ -10,7 +10,7 @@ import httpxclass IgnorantTransform(Transform):# Define types as class attributes - base class handles schema generation automatically- InputType = List[str] # Phone numbers as strings+ InputType = List[Phone] # Phone objectsOutputType = List[Dict[str, Any]] # Results as dictionaries@classmethod@@ -25,43 +25,29 @@ class IgnorantTransform(Transform):def key(cls) -> str:return "number"- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for number in data:- phone_obj = None- if isinstance(number, str):- phone_obj = Phone(number=number)- elif isinstance(number, dict) and "number" in number:- phone_obj = Phone(number=number["number"])- elif isinstance(number, Phone):- phone_obj = number- if phone_obj:- cleaned.append(phone_obj)- return cleaned-async def scan(self, data: InputType) -> OutputType:"""Performs the Ignorant search for each specified phone number."""results: OutputType = []- for phone in data:+ for phone_obj in data:try:- cleaned_phone = is_valid_number(phone)+ cleaned_phone = is_valid_number(phone_obj.number)if cleaned_phone:result = await self._perform_ignorant_research(cleaned_phone)results.append(result)else:- results.append({"number": phone, "error": "Invalid phone number"})+ results.append({"number": phone_obj.number, "error": "Invalid phone number"})except Exception as e:results.append({- "number": phone,+ "number": phone_obj.number,"error": f"Unexpected error in Ignorant scan: {str(e)}",})Logger.error(self.sketch_id,- {"message": f"Error scanning phone {phone}: {str(e)}"},+ {"message": f"Error scanning phone {phone_obj.number}: {str(e)}"},)return results@@ -100,13 +86,8 @@ class IgnorantTransform(Transform):for result in results:if "error" not in result and "platforms" in result:- self.create_node(- "phone",- "number",- result["number"],- caption=result["number"],- type="phone",- )+ phone_obj = Phone(number=result["number"])+ self.create_node(phone_obj)# Create platform relationshipsfor platform_result in result["platforms"]:diff --git a/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.pydeleted file mode 100644index a75a8560..00000000--- a/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.py+++ /dev/null@@ -1,228 +0,0 @@-import os-from typing import Any, Dict, List, Optional, Union-import requests-from urllib.parse import urljoin-from flowsint_core.core.transform_base import Transform-from flowsint_core.core.logger import Logger-from flowsint_types.phone import Phone-from flowsint_types.breach import Breach-from dotenv import load_dotenv-from flowsint_core.core.graph_db import Neo4jConnection--# Load environment variables-load_dotenv()--HIBP_API_KEY = os.getenv("HIBP_API_KEY")---class PhoneToBreachesTransform(Transform):- """[HIBPWNED] Get the breaches the phone number might be invovled in."""-- InputType = List[Phone]- OutputType = List[tuple] # List of (phone, breach) tuples-- def __init__(- self,- sketch_id: Optional[str] = None,- scan_id: Optional[str] = None,- neo4j_conn: Optional[Neo4jConnection] = None,- vault=None,- params: Optional[Dict[str, Any]] = None,- ):- super().__init__(- sketch_id=sketch_id,- scan_id=scan_id,- neo4j_conn=neo4j_conn,- params_schema=self.get_params_schema(),- vault=vault,- params=params,- )-- @classmethod- def name(cls) -> str:- return "phone_to_breaches"-- @classmethod- def category(cls) -> str:- return "Email"-- @classmethod- def key(cls) -> str:- return "phone"-- @classmethod- def required_params(cls) -> bool:- return True-- @classmethod- def get_params_schema(cls) -> List[Dict[str, Any]]:- """Declare required parameters for this transform"""- return [- {- "name": "HIBP_API_KEY",- "type": "vaultSecret",- "description": "The HIBP API key to use for breaches lookup.",- "required": True,- },- {- "name": "HIBP_API_URL",- "type": "url",- "description": "The HIBP API URL to use for breaches lookup.",- "required": False,- "default": "https://haveibeenpwned.com/api/v3/breachedaccount/",- },- ]-- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for number in data:- phone_obj = None- if isinstance(number, str):- phone_obj = Phone(number=number)- elif isinstance(number, dict) and "number" in number:- phone_obj = Phone(number=number["number"])- elif isinstance(number, Phone):- phone_obj = number- if phone_obj:- cleaned.append(phone_obj)- return cleaned-- async def scan(self, data: InputType) -> OutputType:- results: OutputType = []- api_key = self.get_secret("HIBP_API_KEY", os.getenv("HIBP_API_KEY"))- api_url = self.get_params().get("HIBP_API_URL", "https://haveibeenpwned.com/api/v3/breachedaccount/")- headers = {"hibp-api-key": api_key, "User-Agent": "FlowsInt-Transform"}- Logger.info(self.sketch_id, {"message": f"HIBP API URL: {api_url}"})- for phone in data:- try:- # Query Have I Been Pwned API- full_url = urljoin(api_url, f"{phone.number}?truncateResponse=false")- Logger.error(self.sketch_id, {"message": f"full url: {full_url}"})- response = requests.get(full_url, headers=headers, timeout=10)- Logger.info(- self.sketch_id, {"message": f"HIBP API response: {response.json()}"}- )- if response.status_code == 200:- breaches_data = response.json()- Logger.info(- self.sketch_id,- {- "message": f"Found {len(breaches_data)} breaches for {phone.number}"- },- )- for breach_data in breaches_data:- breach = Breach(- name=breach_data.get("Name", ""),- title=breach_data.get("Title", ""),- domain=breach_data.get("Domain", ""),- breachdate=breach_data.get("BreachDate", ""),- addeddate=breach_data.get("AddedDate", ""),- modifieddate=breach_data.get("ModifiedDate", ""),- pwncount=breach_data.get("PwnCount", 0),- description=breach_data.get("Description", ""),- dataclasses=breach_data.get("DataClasses", []),- isverified=breach_data.get("IsVerified", False),- isfabricated=breach_data.get("IsFabricated", False),- issensitive=breach_data.get("IsSensitive", False),- isretired=breach_data.get("IsRetired", False),- isspamlist=breach_data.get("IsSpamList", False),- logopath=breach_data.get("LogoPath", ""),- )- # Store phone and breach as a tuple- results.append((phone.number, breach))- Logger.info(- self.sketch_id,- {- "message": f"Added breach: {breach.name} for phone: {phone.number}"- },- )-- elif response.status_code == 404:- # No breaches found for this phone- Logger.info(- self.sketch_id,- {"message": f"No breaches found for phone {phone.number}"},- )- continue-- else:- Logger.error(- self.sketch_id,- {- "message": f"HIBP API error for {phone.number}: {response.status_code}"- },- )- continue-- except Exception as e:- Logger.error(- self.sketch_id,- {- "message": f"Error checking breaches for phone {phone.number}: {e}"- },- )- continue-- Logger.info(- self.sketch_id,- {"message": f"Scan completed. Total results: {len(results)}"},- )- return results-- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:- Logger.info(- self.sketch_id,- {- "message": f"Postprocess started. Results count: {len(results)}, Original input count: {len(original_input)}"- },- )-- # Create phone nodes first- for phone_obj in original_input:- if not self.neo4j_conn:- continue- # Create phone node- self.create_node("phone", "phone", phone_obj.number, **phone_obj.__dict__)- Logger.info(- self.sketch_id, {"message": f"Created phone node: {phone_obj.number}"}- )-- # Process all breaches- for phone, breach_obj in results:- if not self.neo4j_conn:- continue-- # Create breach node- breach_key = f"{breach_obj.name}_{self.sketch_id}"- self.create_node(- "breach",- "breach_id",- breach_key,- **breach_obj.dict(),- label=breach_obj.name,- type="breach",- )- Logger.info(- self.sketch_id, {"message": f"Created breach node: {breach_key}"}- )-- # Create relationship between the specific phone and this breach- self.create_relationship(- "phone",- "number",- phone,- "breach",- "breach_id",- breach_key,- "FOUND_IN_BREACH",- )- self.log_graph_message(- f"Breach found for phone {phone} -> {breach_obj.name} ({breach_obj.title})"- )-- return results---# Make types available at module level for easy access-InputType = PhoneToBreachesTransform.InputType-OutputType = PhoneToBreachesTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py b/flowsint-transforms/src/flowsint_transforms/social/to_maigret.pyindex e288aab8..67af127a 100644--- a/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py+++ b/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py@@ -1,8 +1,7 @@import jsonimport subprocessfrom pathlib import Path-from typing import List, Union-from flowsint_core.utils import is_valid_username+from typing import Listfrom flowsint_core.core.transform_base import Transformfrom flowsint_types import Usernamefrom flowsint_types.social_account import SocialAccount@@ -30,25 +29,6 @@ class MaigretTransform(Transform):def key(cls) -> str:return "username"- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- obj = None- if isinstance(item, str):- obj = Username(value=item)- elif isinstance(item, dict) and "username" in item:- obj = Username(value=item["username"])- elif isinstance(item, dict) and "value" in item:- obj = Username(value=item["value"])- elif isinstance(item, SocialAccount):- obj = Username(value=item.username.value)- obj = item.username.value- elif isinstance(item, Username):- obj = item- if obj and obj.value and is_valid_username(obj.value):- cleaned.append(obj)- return cleaned-def run_maigret(self, username: str) -> Path:output_file = Path(f"/tmp/report_{username}_simple.json")try:@@ -144,43 +124,17 @@ class MaigretTransform(Transform):for profile in results:# Create social profile node- self.create_node(- "social_account",- "platform",- profile.profile_url,- username=profile.username.value,- platform=profile.platform,- profile_picture_url=profile.profile_picture_url,- bio=profile.bio,- followers_count=profile.followers_count,- following_count=profile.following_count,- posts_count=profile.posts_count,- label=f"{profile.username.value}",- type="social_account",- )+ self.create_node(profile.username)+ self.create_node(profile)# Create username node- self.create_node(- "username", "value", profile.username.value, **profile.username.__dict__- )# Create relationship- self.create_relationship(- "username",- "value",- profile.username.value,- "social_account",- "platform",- profile.platform,- "HAS_SOCIAL_ACCOUNT",- )-+ self.create_relationship(profile.username, profile, "HAS_SOCIAL_ACCOUNT")self.log_graph_message(f"{profile.username.value} -> account found on {profile.platform}")-return results-# Make types available at module level for easy accessInputType = MaigretTransform.InputTypeOutputType = MaigretTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py b/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.pyindex e90930d3..a2b4aa8b 100644--- a/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py+++ b/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py@@ -26,25 +26,6 @@ class SherlockTransform(Transform):def key(cls) -> str:return "username"- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- obj = None- if isinstance(item, str):- obj = Username(value=item)- elif isinstance(item, dict) and "username" in item:- obj = Username(value=item["username"])- elif isinstance(item, dict) and "value" in item:- obj = Username(value=item["value"])- elif isinstance(item, SocialAccount):- obj = Username(value=item.username.value)- obj = item.username.value- elif isinstance(item, Username):- obj = item- if obj and obj.value and is_valid_username(obj.value):- cleaned.append(obj)- return cleaned-async def scan(self, data: InputType) -> OutputType:"""Performs the scan using Sherlock on the list of usernames."""results: OutputType = []@@ -113,15 +94,7 @@ class SherlockTransform(Transform):return resultsfor social_account in results:- self.create_node(- "social",- "username",- social_account.username.value,- platform=social_account.platform,- url=social_account.url,- caption=social_account.platform,- type="social",- )+ self.create_node(social_account)self.log_graph_message(f"Found social account: {social_account.username.value} on {social_account.platform}")diff --git a/flowsint-transforms/src/flowsint_transforms/website/to_crawler.py b/flowsint-transforms/src/flowsint_transforms/website/to_crawler.pyindex 67e86044..72ffab1f 100644--- a/flowsint-transforms/src/flowsint_transforms/website/to_crawler.py+++ b/flowsint-transforms/src/flowsint_transforms/website/to_crawler.py@@ -43,20 +43,6 @@ class WebsiteToCrawler(Transform):except Exception:return False- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- website_obj = None- if isinstance(item, str):- website_obj = Website(url=item)- elif isinstance(item, dict) and "url" in item:- website_obj = Website(url=item["url"])- elif isinstance(item, Website):- website_obj = item- if website_obj:- cleaned.append(website_obj)- return cleaned-async def scan(self, data: InputType) -> OutputType:"""Crawl websites to extract emails and phone numbers."""results = []@@ -155,46 +141,22 @@ class WebsiteToCrawler(Transform):# Create website nodeif self.neo4j_conn:- self.create_node(- "website", "url", website_url, caption=website_url, type="website"- )+ self.create_node(input_website)# Create email nodes and relationshipsfor email in result["emails"]:- self.create_node(- "email", "email", email.email, caption=email.email, type="email"- )- self.create_relationship(- "website",- "url",- website_url,- "email",- "email",- email.email,- "HAS_EMAIL",- )+ self.create_node(email)+ website_obj = Website(url=website_url)+ self.create_relationship(website_obj, email, "HAS_EMAIL")self.log_graph_message(f"Found email {email.email} for website {website_url}")# Create phone nodes and relationshipsfor phone in result["phones"]:- self.create_node(- "phone",- "number",- phone.number,- caption=phone.number,- type="phone",- )- self.create_relationship(- "website",- "url",- website_url,- "phone",- "number",- phone.number,- "HAS_PHONE",- )+ self.create_node(phone)+ website_obj2 = Website(url=website_url)+ self.create_relationship(website_obj2, phone, "HAS_PHONE")self.log_graph_message(f"Found phone {phone.number} for website {website_url}")diff --git a/flowsint-transforms/src/flowsint_transforms/website/to_domain.py b/flowsint-transforms/src/flowsint_transforms/website/to_domain.pyindex cb400323..ff0eda28 100644--- a/flowsint-transforms/src/flowsint_transforms/website/to_domain.py+++ b/flowsint-transforms/src/flowsint_transforms/website/to_domain.py@@ -25,20 +25,6 @@ class WebsiteToDomainTransform(Transform):def key(cls) -> str:return "website"- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- website_obj = None- if isinstance(item, str):- website_obj = Website(url=item)- elif isinstance(item, dict) and "url" in item:- website_obj = Website(url=item["url"])- elif isinstance(item, Website):- website_obj = item- if website_obj:- cleaned.append(website_obj)- return cleaned-async def scan(self, data: InputType) -> OutputType:results: OutputType = []for website in data:@@ -78,31 +64,11 @@ class WebsiteToDomainTransform(Transform):website_url = str(input_website.url)domain_name = result.domain- self.create_node(- "website",- "url",- website_url,- caption=website_url,- type="website",- )-+ self.create_node(input_website)+# Create relationship with the specific domain for this website- self.create_node(- "domain",- "domain",- domain_name,- caption=domain_name,- type="domain",- )- self.create_relationship(- "website",- "url",- website_url,- "domain",- "domain",- domain_name,- "HAS_DOMAIN",- )+ self.create_node(result)+ self.create_relationship(input_website, result, "HAS_DOMAIN")self.log_graph_message(f"Extracted domain {domain_name} from website {website_url}.")diff --git a/flowsint-transforms/src/flowsint_transforms/website/to_links.py b/flowsint-transforms/src/flowsint_transforms/website/to_links.pyindex f31e3f37..77164d7e 100644--- a/flowsint-transforms/src/flowsint_transforms/website/to_links.py+++ b/flowsint-transforms/src/flowsint_transforms/website/to_links.py@@ -2,6 +2,7 @@ from typing import List, Unionfrom urllib.parse import urlparsefrom flowsint_core.core.transform_base import Transformfrom flowsint_types.website import Website+from flowsint_types.domain import Domainfrom flowsint_core.core.logger import Loggerfrom reconspread import Crawler@@ -33,20 +34,6 @@ class WebsiteToLinks(Transform):except Exception:return ""- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- website_obj = None- if isinstance(item, str):- website_obj = Website(url=item)- elif isinstance(item, dict) and "url" in item:- website_obj = Website(url=item["url"])- elif isinstance(item, Website):- website_obj = item- if website_obj:- cleaned.append(website_obj)- return cleaned-async def scan(self, data: InputType) -> OutputType:"""Crawl websites using reconspread to extract internal and external links."""results = []@@ -63,30 +50,11 @@ class WebsiteToLinks(Transform):# Create main website and domain nodes upfrontif self.neo4j_conn:- self.create_node(- "website",- "url",- str(website.url),- caption=str(website.url),- type="website",- )+ self.create_node(website)if main_domain:- self.create_node(- "domain",- "name",- main_domain,- caption=main_domain,- type="domain",- )- self.create_relationship(- "website",- "url",- str(website.url),- "domain",- "name",- main_domain,- "BELONGS_TO_DOMAIN",- )+ domain_obj = Domain(domain=main_domain)+ self.create_node(domain_obj)+ self.create_relationship(website, domain_obj, "BELONGS_TO_DOMAIN")self.log_graph_message(f"Website {str(website.url)} belongs to domain {main_domain}")@@ -105,49 +73,20 @@ class WebsiteToLinks(Transform):external_domains.add(domain)# Create external website node immediatelyif self.neo4j_conn:- self.create_node(- "website", "url", url, caption=url, type="website"- )- self.create_relationship(- "website",- "url",- str(website.url),- "website",- "url",- url,- "LINKS_TO",- )+ url_obj = Website(url=url)+ self.create_node(url_obj)+ self.create_relationship(website, url_obj, "LINKS_TO")self.log_graph_message(f"Website {str(website.url)} links to external website {url}")# Create external domain node and link external website to its domainif domain != main_domain:- self.create_node(- "domain",- "name",- domain,- caption=domain,- type="domain",- )- self.create_relationship(- "website",- "url",- url,- "domain",- "name",- domain,- "BELONGS_TO_DOMAIN",- )- self.create_relationship(- "website",- "url",- str(website.url),- "domain",- "name",- domain,- "LINKS_TO_DOMAIN",- )+ domain_obj_ext = Domain(domain=domain)+ self.create_node(domain_obj_ext)+ self.create_relationship(url_obj, domain_obj_ext, "BELONGS_TO_DOMAIN")+ domain_obj_main = Domain(domain=main_domain)+ self.create_relationship(domain_obj_main, domain_obj_ext, "LINKS_TO")self.log_graph_message(f"External website {url} belongs to domain {domain}")@@ -164,33 +103,17 @@ class WebsiteToLinks(Transform):if self.neo4j_conn and url != str(website.url): # Don't create duplicate of main website- self.create_node(- "website", "url", url, caption=url, type="website"- )- self.create_relationship(- "website",- "url",- str(website.url),- "website",- "url",- url,- "LINKS_TO",- )+ internal_website = Website(url=url)+ self.create_node(internal_website)+ self.create_relationship(website, internal_website, "LINKS_TO")self.log_graph_message(f"Website {str(website.url)} links to internal website {url}")# Also link internal websites to main domainif main_domain:- self.create_relationship(- "website",- "url",- url,- "domain",- "name",- main_domain,- "BELONGS_TO_DOMAIN",- )+ domain_obj_int = Domain(domain=main_domain)+ self.create_relationship(internal_website, domain_obj_int, "BELONGS_TO_DOMAIN")Logger.info(self.sketch_id, {"message": f"[INTERNAL] Found: {url}"})@@ -256,30 +179,11 @@ class WebsiteToLinks(Transform):# Still create main website and domain nodes even on errormain_domain = self.extract_domain(str(website.url))if self.neo4j_conn:- self.create_node(- "website",- "url",- str(website.url),- caption=str(website.url),- type="website",- )+ self.create_node(website)if main_domain:- self.create_node(- "domain",- "name",- main_domain,- caption=main_domain,- type="domain",- )- self.create_relationship(- "website",- "url",- str(website.url),- "domain",- "name",- main_domain,- "BELONGS_TO_DOMAIN",- )+ domain_obj_err = Domain(domain=main_domain)+ self.create_node(domain_obj_err)+ self.create_relationship(website, domain_obj_err, "BELONGS_TO_DOMAIN")self.log_graph_message(f"Website {str(website.url)} belongs to domain {main_domain}")diff --git a/flowsint-transforms/src/flowsint_transforms/website/to_text.py b/flowsint-transforms/src/flowsint_transforms/website/to_text.pyindex a23757a1..ae4918ab 100644--- a/flowsint-transforms/src/flowsint_transforms/website/to_text.py+++ b/flowsint-transforms/src/flowsint_transforms/website/to_text.py@@ -24,20 +24,6 @@ class WebsiteToText(Transform):def key(cls) -> str:return "website"- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- website_obj = None- if isinstance(item, str):- website_obj = Website(url=item)- elif isinstance(item, dict) and "url" in item:- website_obj = Website(url=item["url"])- elif isinstance(item, Website):- website_obj = item- if website_obj:- cleaned.append(website_obj)- return cleaned-async def scan(self, data: InputType) -> OutputType:results: OutputType = []for website in data:@@ -67,31 +53,11 @@ class WebsiteToText(Transform):website_url = str(input_website.url)if self.neo4j_conn:- self.create_node(- "website",- "url",- str(website_url),- caption=str(website_url),- type="website",- )+ self.create_node(input_website)# Create relationship with the specific phrase for this website- self.create_node(- "phrase",- "text",- result.text,- caption=result.text,- type="phrase",- )- self.create_relationship(- "website",- "url",- website_url,- "phrase",- "text",- result.text,- "HAS_INNER_TEXT",- )+ self.create_node(result)+ self.create_relationship(input_website, result, "HAS_INNER_TEXT")self.log_graph_message(f"Extracted some text from the website {website_url}.")diff --git a/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py b/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.pyindex 4ddefaac..338b4072 100644--- a/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py+++ b/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py@@ -39,20 +39,6 @@ class WebsiteToWebtrackersTransform(Transform):def key(cls) -> str:return "website"- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:- cleaned: InputType = []- for item in data:- website_obj = None- if isinstance(item, str):- website_obj = Website(url=item)- elif isinstance(item, dict) and "url" in item:- website_obj = Website(url=item["url"])- elif isinstance(item, Website):- website_obj = item- if website_obj:- cleaned.append(website_obj)- return cleaned-async def scan(self, data: InputType) -> OutputType:results: OutputType = []@@ -97,29 +83,14 @@ class WebsiteToWebtrackersTransform(Transform):# Create nodes and relationships for each website and its trackersfor website_url, trackers in website_trackers.items():- # Create website node- self.create_node(- "website", "url", website_url, caption=website_url, type="website"- )+ # Create website node (we don't have the website object here, so keep minimal)+ self.create_node(Website(url=website_url))# Create tracker nodes and relationshipsfor tracker in trackers:- self.create_node(- "webtracker",- "tracker_id",- tracker.tracker_id,- caption=tracker.name,- type="webtracker"- )- self.create_relationship(- "website",- "url",- website_url,- "webtracker",- "tracker_id",- tracker.tracker_id,- "HAS_TRACKER",- )+ self.create_node(tracker)+ website_obj = Website(url=website_url)+ self.create_relationship(website_obj, tracker, "HAS_TRACKER")self.log_graph_message(f"Found tracker {tracker.name} ({tracker.tracker_id}) for website {website_url}")diff --git a/flowsint-transforms/tests/README.md b/flowsint-transforms/tests/README.mdindex de70f0f8..ab473239 100644--- a/flowsint-transforms/tests/README.md+++ b/flowsint-transforms/tests/README.md@@ -1,6 +1,6 @@-# flowsint-api tests+# flowsint-transforms tests-Run the tests.+Run the tests (make sure you are in `flowsint_tranforms` folder).```bashpython -m pytest tests/ -v --tb=shortdiff --git a/flowsint-transforms/tests/insert_command.py b/flowsint-transforms/tests/insert_command.pydeleted file mode 100644index 4df5a035..00000000--- a/flowsint-transforms/tests/insert_command.py+++ /dev/null@@ -1,28 +0,0 @@-import sys-import os-import asyncio--if __name__ == "__main__":- sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))--from flowsint_types.domain import Domain-from flowsint_types.ip import Ip-from flowsint_transforms.domains.resolve import ResolveTransform---async def main():- # Create test data- domains = [Domain(domain="adaltas.com")]- ips = [Ip(address="12.23.34.45"), Ip(address="56.67.78.89")]-- # Test the transform- transform = ResolveTransform("sketch_123", "scan_123")-- # Test the new KISS postprocess method- transform.postprocess(ips[:1], domains) # Only use first IP to match domains length-- print("Postprocess test completed successfully!")---if __name__ == "__main__":- asyncio.run(main())diff --git a/flowsint-transforms/tests/transforms/domain/resolve.py b/flowsint-transforms/tests/transforms/domain/resolve.pyindex bccf0101..f7f1f3a2 100644--- a/flowsint-transforms/tests/transforms/domain/resolve.py+++ b/flowsint-transforms/tests/transforms/domain/resolve.py@@ -1,4 +1,4 @@-from flowsint_transforms.domains.resolve import ResolveTransform+from flowsint_transforms.domain.to_ip import ResolveTransformfrom flowsint_types.domain import Domainfrom flowsint_types.ip import Ipfrom typing import Listdiff --git a/flowsint-transforms/tests/transforms/domain/test_to_asn_preprocess.py b/flowsint-transforms/tests/transforms/domain/test_to_asn_preprocess.pynew file mode 100644index 00000000..7e89511b--- /dev/null+++ b/flowsint-transforms/tests/transforms/domain/test_to_asn_preprocess.py@@ -0,0 +1,98 @@+import pytest+from flowsint_transforms.domain.to_asn import DomainToAsnTransform+from flowsint_types.domain import Domain+++def transform():+ """Create transform instance for testing."""+ return DomainToAsnTransform(sketch_id="test_sketch", scan_id="test_scan")+++def test_preprocess_valid_objects(transform):+ """Test preprocess with valid Domain objects."""+ inputs = [+ Domain(domain="example.com"),+ Domain(domain="google.com"),+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Domain) for item in result)+ assert result[0].domain == "example.com"+ assert result[1].domain == "google.com"+++def test_preprocess_strings(transform):+ """Test preprocess with string inputs (converted via primary field)."""+ inputs = ["example.com", "google.com"]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Domain) for item in result)+ assert result[0].domain == "example.com"+ assert result[1].domain == "google.com"+++def test_preprocess_dicts(transform):+ """Test preprocess with dict inputs."""+ inputs = [+ {"domain": "example.com"},+ {"domain": "google.com"},+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Domain) for item in result)+ assert result[0].domain == "example.com"+ assert result[1].domain == "google.com"+++def test_preprocess_invalid_filtered(transform):+ """Test that invalid items are filtered out."""+ inputs = [+ Domain(domain="example.com"),+ "not a domain", # Invalid+ Domain(domain="google.com"),+ ]+ result = transform.preprocess(inputs)++ # Only valid items should remain+ assert len(result) == 2+ assert result[0].domain == "example.com"+ assert result[1].domain == "google.com"+++def test_preprocess_mixed_formats(transform):+ """Test preprocess with mixed input formats."""+ inputs = [+ Domain(domain="example.com"), # Object+ "google.com", # String+ {"domain": "github.com"}, # Dict+ {"invalid_key": "wrong.com"}, # Invalid dict (wrong key)+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 3 # Should have 3 valid items+ assert all(isinstance(item, Domain) for item in result)+ assert result[0].domain == "example.com"+ assert result[1].domain == "google.com"+ assert result[2].domain == "github.com"+++def test_preprocess_empty_list(transform):+ """Test preprocess with empty input."""+ result = transform.preprocess([])+ assert result == []+++def test_preprocess_all_invalid(transform):+ """Test preprocess when all items are invalid."""+ inputs = [+ "not a domain",+ "domain with spaces",+ "",+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 0diff --git a/flowsint-transforms/tests/transforms/domain/test_to_ip_preprocess.py b/flowsint-transforms/tests/transforms/domain/test_to_ip_preprocess.pynew file mode 100644index 00000000..55b342ed--- /dev/null+++ b/flowsint-transforms/tests/transforms/domain/test_to_ip_preprocess.py@@ -0,0 +1,83 @@+import pytest+from flowsint_transforms.domain.to_ip import ResolveTransform+from flowsint_types.domain import Domain+++def transform():+ """Create transform instance for testing."""+ return ResolveTransform(sketch_id="test_sketch", scan_id="test_scan")+++def test_preprocess_valid_objects(transform):+ """Test preprocess with valid Domain objects."""+ inputs = [+ Domain(domain="example.com"),+ Domain(domain="google.com"),+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Domain) for item in result)+ assert result[0].domain == "example.com"+ assert result[1].domain == "google.com"+++def test_preprocess_strings(transform):+ """Test preprocess with string inputs (converted via primary field)."""+ inputs = ["example.com", "google.com"]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Domain) for item in result)+ assert result[0].domain == "example.com"+ assert result[1].domain == "google.com"+++def test_preprocess_dicts(transform):+ """Test preprocess with dict inputs."""+ inputs = [+ {"domain": "example.com"},+ {"domain": "google.com"},+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Domain) for item in result)+ assert result[0].domain == "example.com"+ assert result[1].domain == "google.com"+++def test_preprocess_invalid_filtered(transform):+ """Test that invalid items are filtered out."""+ inputs = [+ Domain(domain="example.com"),+ "invalid domain with spaces", # Invalid+ Domain(domain="google.com"),+ ]+ result = transform.preprocess(inputs)++ # Only valid items should remain+ assert len(result) == 2+ assert result[0].domain == "example.com"+ assert result[1].domain == "google.com"+++def test_preprocess_mixed_formats(transform):+ """Test preprocess with mixed input formats."""+ inputs = [+ Domain(domain="example.com"), # Object+ "google.com", # String+ {"domain": "github.com"}, # Dict+ {"invalid_key": "wrong.com"}, # Invalid dict (wrong key)+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 3 # Should have 3 valid items+ assert all(isinstance(item, Domain) for item in result)+++def test_preprocess_empty_list(transform):+ """Test preprocess with empty input."""+ result = transform.preprocess([])+ assert result == []diff --git a/flowsint-transforms/tests/transforms/domain/test_to_root_domain_preprocess.py b/flowsint-transforms/tests/transforms/domain/test_to_root_domain_preprocess.pynew file mode 100644index 00000000..a55678ee--- /dev/null+++ b/flowsint-transforms/tests/transforms/domain/test_to_root_domain_preprocess.py@@ -0,0 +1,83 @@+import pytest+from flowsint_transforms.domain.to_root_domain import DomainToRootDomain+from flowsint_types.domain import Domain+++def transform():+ """Create transform instance for testing."""+ return DomainToRootDomain(sketch_id="test_sketch", scan_id="test_scan")+++def test_preprocess_valid_objects(transform):+ """Test preprocess with valid Domain objects."""+ inputs = [+ Domain(domain="subdomain.example.com"),+ Domain(domain="www.google.com"),+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Domain) for item in result)+ assert result[0].domain == "subdomain.example.com"+ assert result[1].domain == "www.google.com"+++def test_preprocess_strings(transform):+ """Test preprocess with string inputs (converted via primary field)."""+ inputs = ["subdomain.example.com", "www.google.com"]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Domain) for item in result)+ assert result[0].domain == "subdomain.example.com"+ assert result[1].domain == "www.google.com"+++def test_preprocess_dicts(transform):+ """Test preprocess with dict inputs."""+ inputs = [+ {"domain": "subdomain.example.com"},+ {"domain": "www.google.com"},+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Domain) for item in result)+ assert result[0].domain == "subdomain.example.com"+ assert result[1].domain == "www.google.com"+++def test_preprocess_invalid_filtered(transform):+ """Test that invalid items are filtered out."""+ inputs = [+ Domain(domain="example.com"),+ "not a domain", # Invalid+ Domain(domain="google.com"),+ ]+ result = transform.preprocess(inputs)++ # Only valid items should remain+ assert len(result) == 2+ assert result[0].domain == "example.com"+ assert result[1].domain == "google.com"+++def test_preprocess_mixed_formats(transform):+ """Test preprocess with mixed input formats."""+ inputs = [+ Domain(domain="subdomain.example.com"), # Object+ "www.google.com", # String+ {"domain": "api.github.com"}, # Dict+ {"invalid_key": "wrong.com"}, # Invalid dict (wrong key)+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 3 # Should have 3 valid items+ assert all(isinstance(item, Domain) for item in result)+++def test_preprocess_empty_list(transform):+ """Test preprocess with empty input."""+ result = transform.preprocess([])+ assert result == []diff --git a/flowsint-transforms/tests/transforms/email/test_to_gravatar_preprocess.py b/flowsint-transforms/tests/transforms/email/test_to_gravatar_preprocess.pynew file mode 100644index 00000000..c214f39e--- /dev/null+++ b/flowsint-transforms/tests/transforms/email/test_to_gravatar_preprocess.py@@ -0,0 +1,98 @@+import pytest+from flowsint_transforms.email.to_gravatar import EmailToGravatarTransform+from flowsint_types.email import Email+++def transform():+ """Create transform instance for testing."""+ return EmailToGravatarTransform(sketch_id="test_sketch", scan_id="test_scan")+++def test_preprocess_valid_objects(transform):+ """Test preprocess with valid Email objects."""+ inputs = [+ Email(email="[email protected]"),+ Email(email="[email protected]"),+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Email) for item in result)+ assert result[0].email == "[email protected]"+ assert result[1].email == "[email protected]"+++def test_preprocess_strings(transform):+ """Test preprocess with string inputs (converted via primary field)."""+ inputs = ["[email protected]", "[email protected]"]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Email) for item in result)+ assert result[0].email == "[email protected]"+ assert result[1].email == "[email protected]"+++def test_preprocess_dicts(transform):+ """Test preprocess with dict inputs."""+ inputs = [+ {"email": "[email protected]"},+ {"email": "[email protected]"},+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Email) for item in result)+ assert result[0].email == "[email protected]"+ assert result[1].email == "[email protected]"+++def test_preprocess_invalid_filtered(transform):+ """Test that invalid items are filtered out."""+ inputs = [+ Email(email="[email protected]"),+ "not-an-email", # Invalid+ Email(email="[email protected]"),+ ]+ result = transform.preprocess(inputs)++ # Only valid items should remain+ assert len(result) == 2+ assert result[0].email == "[email protected]"+ assert result[1].email == "[email protected]"+++def test_preprocess_mixed_formats(transform):+ """Test preprocess with mixed input formats."""+ inputs = [+ Email(email="[email protected]"), # Object+ "[email protected]", # String+ {"email": "[email protected]"}, # Dict+ {"invalid_key": "[email protected]"}, # Invalid dict (wrong key)+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 3 # Should have 3 valid items+ assert all(isinstance(item, Email) for item in result)+ assert result[0].email == "[email protected]"+ assert result[1].email == "[email protected]"+ assert result[2].email == "[email protected]"+++def test_preprocess_empty_list(transform):+ """Test preprocess with empty input."""+ result = transform.preprocess([])+ assert result == []+++def test_preprocess_all_invalid(transform):+ """Test preprocess when all items are invalid."""+ inputs = [+ "not-an-email",+ "missing@",+ "invalid",+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 0diff --git a/flowsint-transforms/tests/transforms/ip/test_to_asn_preprocess.py b/flowsint-transforms/tests/transforms/ip/test_to_asn_preprocess.pynew file mode 100644index 00000000..2577e4b7--- /dev/null+++ b/flowsint-transforms/tests/transforms/ip/test_to_asn_preprocess.py@@ -0,0 +1,83 @@+import pytest+from flowsint_transforms.ip.to_asn import IpToAsnTransform+from flowsint_types.ip import Ip+++def transform():+ """Create transform instance for testing."""+ return IpToAsnTransform(sketch_id="test_sketch", scan_id="test_scan")+++def test_preprocess_valid_objects(transform):+ """Test preprocess with valid Ip objects."""+ inputs = [+ Ip(address="8.8.8.8"),+ Ip(address="1.1.1.1"),+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Ip) for item in result)+ assert result[0].address == "8.8.8.8"+ assert result[1].address == "1.1.1.1"+++def test_preprocess_strings(transform):+ """Test preprocess with string inputs (converted via primary field)."""+ inputs = ["8.8.8.8", "1.1.1.1"]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Ip) for item in result)+ assert result[0].address == "8.8.8.8"+ assert result[1].address == "1.1.1.1"+++def test_preprocess_dicts(transform):+ """Test preprocess with dict inputs."""+ inputs = [+ {"address": "8.8.8.8"},+ {"address": "1.1.1.1"},+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Ip) for item in result)+ assert result[0].address == "8.8.8.8"+ assert result[1].address == "1.1.1.1"+++def test_preprocess_invalid_filtered(transform):+ """Test that invalid items are filtered out."""+ inputs = [+ Ip(address="8.8.8.8"),+ "not-an-ip", # Invalid+ Ip(address="1.1.1.1"),+ ]+ result = transform.preprocess(inputs)++ # Only valid items should remain+ assert len(result) == 2+ assert result[0].address == "8.8.8.8"+ assert result[1].address == "1.1.1.1"+++def test_preprocess_mixed_formats(transform):+ """Test preprocess with mixed input formats."""+ inputs = [+ Ip(address="8.8.8.8"), # Object+ "1.1.1.1", # String+ {"address": "192.168.1.1"}, # Dict+ {"invalid_key": "10.0.0.1"}, # Invalid dict (wrong key)+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 3 # Should have 3 valid items+ assert all(isinstance(item, Ip) for item in result)+++def test_preprocess_empty_list(transform):+ """Test preprocess with empty input."""+ result = transform.preprocess([])+ assert result == []diff --git a/flowsint-transforms/tests/transforms/ip/test_to_infos_preprocess.py b/flowsint-transforms/tests/transforms/ip/test_to_infos_preprocess.pynew file mode 100644index 00000000..db43ad60--- /dev/null+++ b/flowsint-transforms/tests/transforms/ip/test_to_infos_preprocess.py@@ -0,0 +1,98 @@+import pytest+from flowsint_transforms.ip.to_infos import IpToInfosTransform+from flowsint_types.ip import Ip+++def transform():+ """Create transform instance for testing."""+ return IpToInfosTransform(sketch_id="test_sketch", scan_id="test_scan")+++def test_preprocess_valid_objects(transform):+ """Test preprocess with valid Ip objects."""+ inputs = [+ Ip(address="8.8.8.8"),+ Ip(address="1.1.1.1"),+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Ip) for item in result)+ assert result[0].address == "8.8.8.8"+ assert result[1].address == "1.1.1.1"+++def test_preprocess_strings(transform):+ """Test preprocess with string inputs (converted via primary field)."""+ inputs = ["8.8.8.8", "1.1.1.1"]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Ip) for item in result)+ assert result[0].address == "8.8.8.8"+ assert result[1].address == "1.1.1.1"+++def test_preprocess_dicts(transform):+ """Test preprocess with dict inputs."""+ inputs = [+ {"address": "8.8.8.8"},+ {"address": "1.1.1.1"},+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Ip) for item in result)+ assert result[0].address == "8.8.8.8"+ assert result[1].address == "1.1.1.1"+++def test_preprocess_invalid_filtered(transform):+ """Test that invalid items are filtered out."""+ inputs = [+ Ip(address="8.8.8.8"),+ "999.999.999.999", # Invalid+ Ip(address="1.1.1.1"),+ ]+ result = transform.preprocess(inputs)++ # Only valid items should remain+ assert len(result) == 2+ assert result[0].address == "8.8.8.8"+ assert result[1].address == "1.1.1.1"+++def test_preprocess_mixed_formats(transform):+ """Test preprocess with mixed input formats."""+ inputs = [+ Ip(address="8.8.8.8"), # Object+ "1.1.1.1", # String+ {"address": "192.168.1.1"}, # Dict+ {"invalid_key": "10.0.0.1"}, # Invalid dict (wrong key)+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 3 # Should have 3 valid items+ assert all(isinstance(item, Ip) for item in result)+ assert result[0].address == "8.8.8.8"+ assert result[1].address == "1.1.1.1"+ assert result[2].address == "192.168.1.1"+++def test_preprocess_empty_list(transform):+ """Test preprocess with empty input."""+ result = transform.preprocess([])+ assert result == []+++def test_preprocess_all_invalid(transform):+ """Test preprocess when all items are invalid."""+ inputs = [+ "not-an-ip",+ "999.999.999.999",+ "invalid",+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 0diff --git a/flowsint-transforms/tests/transforms/organization/test_to_asn_preprocess.py b/flowsint-transforms/tests/transforms/organization/test_to_asn_preprocess.pynew file mode 100644index 00000000..6b103af8--- /dev/null+++ b/flowsint-transforms/tests/transforms/organization/test_to_asn_preprocess.py@@ -0,0 +1,87 @@+import pytest+from flowsint_transforms.organization.to_asn import OrgToAsnTransform+from flowsint_types.organization import Organization+++def transform():+ """Create transform instance for testing."""+ return OrgToAsnTransform(sketch_id="test_sketch", scan_id="test_scan")+++def test_preprocess_valid_objects(transform):+ """Test preprocess with valid Organization objects."""+ inputs = [+ Organization(name="Acme Corp"),+ Organization(name="Google LLC"),+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Organization) for item in result)+ assert result[0].name == "Acme Corp"+ assert result[1].name == "Google LLC"+++def test_preprocess_strings(transform):+ """Test preprocess with string inputs (converted via primary field)."""+ inputs = ["Acme Corp", "Google LLC"]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Organization) for item in result)+ assert result[0].name == "Acme Corp"+ assert result[1].name == "Google LLC"+++def test_preprocess_dicts(transform):+ """Test preprocess with dict inputs."""+ inputs = [+ {"name": "Acme Corp"},+ {"name": "Google LLC"},+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Organization) for item in result)+ assert result[0].name == "Acme Corp"+ assert result[1].name == "Google LLC"+++def test_preprocess_mixed_formats(transform):+ """Test preprocess with mixed input formats."""+ inputs = [+ Organization(name="Acme Corp"), # Object+ "Google LLC", # String+ {"name": "Microsoft Corporation"}, # Dict+ {"invalid_key": "Apple Inc"}, # Invalid dict (wrong key)+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 3 # Should have 3 valid items+ assert all(isinstance(item, Organization) for item in result)+ assert result[0].name == "Acme Corp"+ assert result[1].name == "Google LLC"+ assert result[2].name == "Microsoft Corporation"+++def test_preprocess_empty_list(transform):+ """Test preprocess with empty input."""+ result = transform.preprocess([])+ assert result == []+++def test_preprocess_with_additional_fields(transform):+ """Test preprocess with dicts containing additional fields."""+ inputs = [+ {"name": "Acme Corp", "nom_complet": "Acme Corporation"},+ {"name": "Google LLC", "siren": "123456789"},+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Organization) for item in result)+ assert result[0].name == "Acme Corp"+ assert result[0].nom_complet == "Acme Corporation"+ assert result[1].name == "Google LLC"+ assert result[1].siren == "123456789"diff --git a/flowsint-transforms/tests/transforms/phone/test_to_infos_preprocess.py b/flowsint-transforms/tests/transforms/phone/test_to_infos_preprocess.pynew file mode 100644index 00000000..43b085d1--- /dev/null+++ b/flowsint-transforms/tests/transforms/phone/test_to_infos_preprocess.py@@ -0,0 +1,95 @@+import pytest+from flowsint_transforms.phone.to_infos import IgnorantTransform+from flowsint_types.phone import Phone+++def transform():+ """Create transform instance for testing."""+ return IgnorantTransform(sketch_id="test_sketch", scan_id="test_scan")+++def test_preprocess_valid_objects(transform):+ """Test preprocess with valid Phone objects."""+ inputs = [+ Phone(number="+33612345678"),+ Phone(number="+14155552671"),+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Phone) for item in result)+ assert result[0].number == "+33612345678"+ assert result[1].number == "+14155552671"+++def test_preprocess_strings(transform):+ """Test preprocess with string inputs (converted via primary field)."""+ inputs = ["+33612345678", "+14155552671"]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Phone) for item in result)+ assert result[0].number == "+33612345678"+ assert result[1].number == "+14155552671"+++def test_preprocess_dicts(transform):+ """Test preprocess with dict inputs."""+ inputs = [+ {"number": "+33612345678"},+ {"number": "+14155552671"},+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Phone) for item in result)+ assert result[0].number == "+33612345678"+ assert result[1].number == "+14155552671"+++def test_preprocess_invalid_filtered(transform):+ """Test that invalid items are filtered out."""+ inputs = [+ Phone(number="+33612345678"),+ "123", # Invalid (too short)+ Phone(number="+14155552671"),+ ]+ result = transform.preprocess(inputs)++ # Only valid items should remain+ assert len(result) == 2+ assert result[0].number == "+33612345678"+ assert result[1].number == "+14155552671"+++def test_preprocess_mixed_formats(transform):+ """Test preprocess with mixed input formats."""+ inputs = [+ Phone(number="+33612345678"), # Object+ "+14155552671", # String+ {"number": "+447911123456"}, # Dict+ {"invalid_key": "+15555555555"}, # Invalid dict (wrong key)+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 3 # Should have 3 valid items+ assert all(isinstance(item, Phone) for item in result)+++def test_preprocess_empty_list(transform):+ """Test preprocess with empty input."""+ result = transform.preprocess([])+ assert result == []+++def test_preprocess_all_invalid(transform):+ """Test preprocess when all items are invalid."""+ inputs = [+ "123",+ "not-a-phone",+ "",+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 0diff --git a/flowsint-transforms/tests/transforms/social/test_to_maigret_preprocess.py b/flowsint-transforms/tests/transforms/social/test_to_maigret_preprocess.pynew file mode 100644index 00000000..8f349d64--- /dev/null+++ b/flowsint-transforms/tests/transforms/social/test_to_maigret_preprocess.py@@ -0,0 +1,111 @@+import pytest+from flowsint_transforms.social.to_maigret import MaigretTransform+from flowsint_types.username import Username+++def transform():+ """Create transform instance for testing."""+ return MaigretTransform(sketch_id="test_sketch", scan_id="test_scan")+++def test_preprocess_valid_objects(transform):+ """Test preprocess with valid Username objects."""+ inputs = [+ Username(value="john_doe"),+ Username(value="user123"),+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Username) for item in result)+ assert result[0].value == "john_doe"+ assert result[1].value == "user123"+++def test_preprocess_strings(transform):+ """Test preprocess with string inputs (converted via primary field)."""+ inputs = ["john_doe", "user123"]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Username) for item in result)+ assert result[0].value == "john_doe"+ assert result[1].value == "user123"+++def test_preprocess_dicts(transform):+ """Test preprocess with dict inputs."""+ inputs = [+ {"value": "john_doe"},+ {"value": "user123"},+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Username) for item in result)+ assert result[0].value == "john_doe"+ assert result[1].value == "user123"+++def test_preprocess_invalid_filtered(transform):+ """Test that invalid items are filtered out."""+ inputs = [+ Username(value="john_doe"),+ "ab", # Invalid (too short, must be 3-30 chars)+ Username(value="user123"),+ ]+ result = transform.preprocess(inputs)++ # Only valid items should remain+ assert len(result) == 2+ assert result[0].value == "john_doe"+ assert result[1].value == "user123"+++def test_preprocess_mixed_formats(transform):+ """Test preprocess with mixed input formats."""+ inputs = [+ Username(value="john_doe"), # Object+ "user123", # String+ {"value": "alice_2023"}, # Dict+ {"invalid_key": "bob_smith"}, # Invalid dict (wrong key)+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 3 # Should have 3 valid items+ assert all(isinstance(item, Username) for item in result)+++def test_preprocess_empty_list(transform):+ """Test preprocess with empty input."""+ result = transform.preprocess([])+ assert result == []+++def test_preprocess_all_invalid(transform):+ """Test preprocess when all items are invalid."""+ inputs = [+ "ab", # Too short+ "user with spaces", # Invalid characters+ "", # Empty+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 0+++def test_preprocess_with_platform(transform):+ """Test preprocess with username including platform."""+ inputs = [+ {"value": "john_doe", "platform": "twitter"},+ {"value": "user123", "platform": "github"},+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Username) for item in result)+ assert result[0].value == "john_doe"+ assert result[0].platform == "twitter"+ assert result[1].value == "user123"+ assert result[1].platform == "github"diff --git a/flowsint-transforms/tests/transforms/social/test_to_sherlock_preprocess.py b/flowsint-transforms/tests/transforms/social/test_to_sherlock_preprocess.pynew file mode 100644index 00000000..a63f6856--- /dev/null+++ b/flowsint-transforms/tests/transforms/social/test_to_sherlock_preprocess.py@@ -0,0 +1,95 @@+import pytest+from flowsint_transforms.social.to_sherlock import SherlockTransform+from flowsint_types.username import Username+++def transform():+ """Create transform instance for testing."""+ return SherlockTransform(sketch_id="test_sketch", scan_id="test_scan")+++def test_preprocess_valid_objects(transform):+ """Test preprocess with valid Username objects."""+ inputs = [+ Username(value="john_doe"),+ Username(value="user123"),+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Username) for item in result)+ assert result[0].value == "john_doe"+ assert result[1].value == "user123"+++def test_preprocess_strings(transform):+ """Test preprocess with string inputs (converted via primary field)."""+ inputs = ["john_doe", "user123"]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Username) for item in result)+ assert result[0].value == "john_doe"+ assert result[1].value == "user123"+++def test_preprocess_dicts(transform):+ """Test preprocess with dict inputs."""+ inputs = [+ {"value": "john_doe"},+ {"value": "user123"},+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Username) for item in result)+ assert result[0].value == "john_doe"+ assert result[1].value == "user123"+++def test_preprocess_invalid_filtered(transform):+ """Test that invalid items are filtered out."""+ inputs = [+ Username(value="john_doe"),+ "ab", # Invalid (too short, must be 3-30 chars)+ Username(value="user123"),+ ]+ result = transform.preprocess(inputs)++ # Only valid items should remain+ assert len(result) == 2+ assert result[0].value == "john_doe"+ assert result[1].value == "user123"+++def test_preprocess_mixed_formats(transform):+ """Test preprocess with mixed input formats."""+ inputs = [+ Username(value="john_doe"), # Object+ "user123", # String+ {"value": "alice_2023"}, # Dict+ {"invalid_key": "bob_smith"}, # Invalid dict (wrong key)+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 3 # Should have 3 valid items+ assert all(isinstance(item, Username) for item in result)+++def test_preprocess_empty_list(transform):+ """Test preprocess with empty input."""+ result = transform.preprocess([])+ assert result == []+++def test_preprocess_all_invalid(transform):+ """Test preprocess when all items are invalid."""+ inputs = [+ "ab", # Too short (min 3 chars)+ "user with spaces", # Invalid characters+ "a" * 81, # Too long (max 80 chars)+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 0diff --git a/flowsint-transforms/tests/transforms/website/test_to_domain_preprocess.py b/flowsint-transforms/tests/transforms/website/test_to_domain_preprocess.pynew file mode 100644index 00000000..1dfbf072--- /dev/null+++ b/flowsint-transforms/tests/transforms/website/test_to_domain_preprocess.py@@ -0,0 +1,95 @@+import pytest+from flowsint_transforms.website.to_domain import WebsiteToDomainTransform+from flowsint_types.website import Website+++def transform():+ """Create transform instance for testing."""+ return WebsiteToDomainTransform(sketch_id="test_sketch", scan_id="test_scan")+++def test_preprocess_valid_objects(transform):+ """Test preprocess with valid Website objects."""+ inputs = [+ Website(url="https://example.com"),+ Website(url="https://google.com"),+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Website) for item in result)+ assert str(result[0].url) == "https://example.com/"+ assert str(result[1].url) == "https://google.com/"+++def test_preprocess_strings(transform):+ """Test preprocess with string inputs (converted via primary field)."""+ inputs = ["https://example.com", "https://google.com"]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Website) for item in result)+ assert str(result[0].url) == "https://example.com/"+ assert str(result[1].url) == "https://google.com/"+++def test_preprocess_dicts(transform):+ """Test preprocess with dict inputs."""+ inputs = [+ {"url": "https://example.com"},+ {"url": "https://google.com"},+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Website) for item in result)+ assert str(result[0].url) == "https://example.com/"+ assert str(result[1].url) == "https://google.com/"+++def test_preprocess_invalid_filtered(transform):+ """Test that invalid items are filtered out."""+ inputs = [+ Website(url="https://example.com"),+ "not-a-url", # Invalid+ Website(url="https://google.com"),+ ]+ result = transform.preprocess(inputs)++ # Only valid items should remain+ assert len(result) == 2+ assert str(result[0].url) == "https://example.com/"+ assert str(result[1].url) == "https://google.com/"+++def test_preprocess_mixed_formats(transform):+ """Test preprocess with mixed input formats."""+ inputs = [+ Website(url="https://example.com"), # Object+ "https://google.com", # String+ {"url": "https://github.com"}, # Dict+ {"invalid_key": "https://wrong.com"}, # Invalid dict (wrong key)+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 3 # Should have 3 valid items+ assert all(isinstance(item, Website) for item in result)+++def test_preprocess_empty_list(transform):+ """Test preprocess with empty input."""+ result = transform.preprocess([])+ assert result == []+++def test_preprocess_all_invalid(transform):+ """Test preprocess when all items are invalid."""+ inputs = [+ "not-a-url",+ "ht tp://broken.com",+ "",+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 0diff --git a/flowsint-transforms/tests/transforms/website/test_to_text_preprocess.py b/flowsint-transforms/tests/transforms/website/test_to_text_preprocess.pynew file mode 100644index 00000000..1142b7b6--- /dev/null+++ b/flowsint-transforms/tests/transforms/website/test_to_text_preprocess.py@@ -0,0 +1,83 @@+import pytest+from flowsint_transforms.website.to_text import WebsiteToText+from flowsint_types.website import Website+++def transform():+ """Create transform instance for testing."""+ return WebsiteToText(sketch_id="test_sketch", scan_id="test_scan")+++def test_preprocess_valid_objects(transform):+ """Test preprocess with valid Website objects."""+ inputs = [+ Website(url="https://example.com"),+ Website(url="https://google.com"),+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Website) for item in result)+ assert str(result[0].url) == "https://example.com/"+ assert str(result[1].url) == "https://google.com/"+++def test_preprocess_strings(transform):+ """Test preprocess with string inputs (converted via primary field)."""+ inputs = ["https://example.com", "https://google.com"]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Website) for item in result)+ assert str(result[0].url) == "https://example.com/"+ assert str(result[1].url) == "https://google.com/"+++def test_preprocess_dicts(transform):+ """Test preprocess with dict inputs."""+ inputs = [+ {"url": "https://example.com"},+ {"url": "https://google.com"},+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 2+ assert all(isinstance(item, Website) for item in result)+ assert str(result[0].url) == "https://example.com/"+ assert str(result[1].url) == "https://google.com/"+++def test_preprocess_invalid_filtered(transform):+ """Test that invalid items are filtered out."""+ inputs = [+ Website(url="https://example.com"),+ "not-a-url", # Invalid+ Website(url="https://google.com"),+ ]+ result = transform.preprocess(inputs)++ # Only valid items should remain+ assert len(result) == 2+ assert str(result[0].url) == "https://example.com/"+ assert str(result[1].url) == "https://google.com/"+++def test_preprocess_mixed_formats(transform):+ """Test preprocess with mixed input formats."""+ inputs = [+ Website(url="https://example.com"), # Object+ "https://google.com", # String+ {"url": "https://github.com"}, # Dict+ {"invalid_key": "https://wrong.com"}, # Invalid dict (wrong key)+ ]+ result = transform.preprocess(inputs)++ assert len(result) == 3 # Should have 3 valid items+ assert all(isinstance(item, Website) for item in result)+++def test_preprocess_empty_list(transform):+ """Test preprocess with empty input."""+ result = transform.preprocess([])+ assert result == []diff --git a/flowsint-types/src/flowsint_types/address.py b/flowsint-types/src/flowsint_types/address.pyindex 8c78534d..dd79ab53 100644--- a/flowsint-types/src/flowsint_types/address.py+++ b/flowsint-types/src/flowsint_types/address.py@@ -1,8 +1,9 @@-from pydantic import BaseModel, Field-from typing import Optional+from pydantic import Field, model_validator+from typing import Optional, Self+from .flowsint_base import FlowsintType-class Location(BaseModel):+class Location(FlowsintType):"""Represents a physical address with geographical coordinates."""address: str = Field(..., description="Street address", title="Street Address")@@ -15,3 +16,8 @@ class Location(BaseModel):longitude: Optional[float] = Field(None, description="Longitude coordinate of the address", title="Longitude")++ @model_validator(mode="after")+ def compute_label(self) -> Self:+ self.label = f"{self.address} {self.city}, {self.country}"+ return selfdiff --git a/flowsint-types/src/flowsint_types/affiliation.py b/flowsint-types/src/flowsint_types/affiliation.pyindex 9b594f11..f11d8317 100644--- a/flowsint-types/src/flowsint_types/affiliation.py+++ b/flowsint-types/src/flowsint_types/affiliation.py@@ -1,8 +1,10 @@-from pydantic import BaseModel, Field-from typing import Optional, List+from pydantic import Field, model_validator+from typing import Optional, List, Self+from .flowsint_base import FlowsintType-class Affiliation(BaseModel):++class Affiliation(FlowsintType):"""Represents an organizational affiliation or employment relationship."""organization: str = Field(@@ -53,3 +55,11 @@ class Affiliation(BaseModel):description="Hierarchical level within organization",title="Hierarchy Level",)++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ if self.role:+ self.label = f"{self.role} at {self.organization}"+ else:+ self.label = self.organization+ return selfdiff --git a/flowsint-types/src/flowsint_types/alias.py b/flowsint-types/src/flowsint_types/alias.pyindex 2e766125..274fb507 100644--- a/flowsint-types/src/flowsint_types/alias.py+++ b/flowsint-types/src/flowsint_types/alias.py@@ -1,8 +1,10 @@-from pydantic import BaseModel, Field-from typing import Optional, List+from pydantic import Field, model_validator+from typing import Optional, List, Self+from .flowsint_base import FlowsintType-class Alias(BaseModel):++class Alias(FlowsintType):"""Represents an alias or alternative name used by an entity."""alias: str = Field(..., description="Alias or alternative name", title="Alias")@@ -43,3 +45,8 @@ class Alias(BaseModel):region: Optional[str] = Field(None, description="Geographic region where alias is used", title="Region")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ self.label = self.alias+ return selfdiff --git a/flowsint-types/src/flowsint_types/asn.py b/flowsint-types/src/flowsint_types/asn.pyindex 5977fbd4..e4334bc4 100644--- a/flowsint-types/src/flowsint_types/asn.py+++ b/flowsint-types/src/flowsint_types/asn.py@@ -1,9 +1,10 @@-from typing import List, Optional, Union-from pydantic import BaseModel, Field, field_validator, model_validator+from typing import List, Optional, Union, Self+from pydantic import Field, field_validator, model_validatorimport re+from .flowsint_base import FlowsintType-class ASN(BaseModel):+class ASN(FlowsintType):"""Represents an Autonomous System Number with associated network information."""number: int = Field(@@ -59,12 +60,21 @@ class ASN(BaseModel):return v@model_validator(mode='after')- def populate_asn_str(self) -> 'ASN':+ def populate_asn_str(self) -> Self:"""Automatically populate asn_str from number if not provided."""# Always set asn_str based on number to ensure consistencyself.asn_str = f"AS{self.number}"return self+ @model_validator(mode='after')+ def compute_label(self) -> Self:+ # Use name and ASN string if available+ if self.name:+ self.label = f"{self.asn_str} - {self.name}"+ else:+ self.label = self.asn_str+ return self+# Import CIDR here to avoid circular importfrom .cidr import CIDRdiff --git a/flowsint-types/src/flowsint_types/bank_account.py b/flowsint-types/src/flowsint_types/bank_account.pyindex bbafc745..3ef88642 100644--- a/flowsint-types/src/flowsint_types/bank_account.py+++ b/flowsint-types/src/flowsint_types/bank_account.py@@ -1,8 +1,10 @@-from pydantic import BaseModel, Field-from typing import Optional, List+from pydantic import Field, model_validator+from typing import Optional, List, Self+from .flowsint_base import FlowsintType-class BankAccount(BaseModel):++class BankAccount(FlowsintType):"""Represents a bank account with financial and security information."""account_number: str = Field(@@ -61,3 +63,12 @@ class BankAccount(BaseModel):breach_source: Optional[str] = Field(None, description="Source of breach if compromised", title="Breach Source")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ parts = []+ if self.bank_name:+ parts.append(self.bank_name)+ parts.append(f"****{self.account_number[-4:]}" if len(self.account_number) > 4 else self.account_number)+ self.label = " - ".join(parts)+ return selfdiff --git a/flowsint-types/src/flowsint_types/breach.py b/flowsint-types/src/flowsint_types/breach.pyindex e3b5b050..784619fb 100644--- a/flowsint-types/src/flowsint_types/breach.py+++ b/flowsint-types/src/flowsint_types/breach.py@@ -1,8 +1,9 @@-from typing import List, Optional, Dict-from pydantic import BaseModel, Field+from typing import List, Optional, Dict, Self+from pydantic import Field, model_validator+from .flowsint_base import FlowsintType-class Breach(BaseModel):+class Breach(FlowsintType):"""Represents a data breach incident with affected accounts and details."""name: str = Field(@@ -66,3 +67,12 @@ class Breach(BaseModel):description="Full breach data as returned by the API",title="Full Breach Data",)++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ # Use title if available, otherwise name+ if self.title:+ self.label = self.title+ else:+ self.label = self.name+ return selfdiff --git a/flowsint-types/src/flowsint_types/cidr.py b/flowsint-types/src/flowsint_types/cidr.pyindex 5cbc24bb..ed414c43 100644--- a/flowsint-types/src/flowsint_types/cidr.py+++ b/flowsint-types/src/flowsint_types/cidr.py@@ -1,9 +1,16 @@-from pydantic import BaseModel, IPvAnyNetwork, Field+from pydantic import IPvAnyNetwork, Field, model_validator+from typing import Self+from .flowsint_base import FlowsintType-class CIDR(BaseModel):+class CIDR(FlowsintType):"""Represents a CIDR (Classless Inter-Domain Routing) network block."""network: IPvAnyNetwork = Field(..., description="CIDR block (e.g., 8.8.8.0/24)", title="Network Block")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ self.label = str(self.network)+ return selfdiff --git a/flowsint-types/src/flowsint_types/credential.py b/flowsint-types/src/flowsint_types/credential.pyindex a7c75e9c..93a2d9b6 100644--- a/flowsint-types/src/flowsint_types/credential.py+++ b/flowsint-types/src/flowsint_types/credential.py@@ -1,8 +1,10 @@-from pydantic import BaseModel, Field-from typing import Optional, List+from pydantic import Field, model_validator+from typing import Optional, List, Self+from .flowsint_base import FlowsintType-class Credential(BaseModel):++class Credential(FlowsintType):"""Represents user credentials with compromise and usage information."""username: str = Field(..., description="Username or identifier", title="Username")@@ -62,3 +64,11 @@ class Credential(BaseModel):source: Optional[str] = Field(None, description="Source of credential information", title="Source")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ if self.service:+ self.label = f"{self.username}@{self.service}"+ else:+ self.label = self.username+ return selfdiff --git a/flowsint-types/src/flowsint_types/credit_card.py b/flowsint-types/src/flowsint_types/credit_card.pyindex f6ef1fc8..17d901e5 100644--- a/flowsint-types/src/flowsint_types/credit_card.py+++ b/flowsint-types/src/flowsint_types/credit_card.py@@ -1,8 +1,10 @@-from pydantic import BaseModel, Field-from typing import Optional, List+from pydantic import Field, model_validator+from typing import Optional, List, Self+from .flowsint_base import FlowsintType-class CreditCard(BaseModel):++class CreditCard(FlowsintType):"""Represents a credit card with financial details and security status."""card_number: str = Field(..., description="Credit card number", title="Card Number")@@ -56,3 +58,12 @@ class CreditCard(BaseModel):last_used: Optional[str] = Field(None, description="Last time card was used", title="Last Used")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ parts = []+ if self.card_type:+ parts.append(self.card_type)+ parts.append(f"****{self.card_number[-4:]}" if len(self.card_number) > 4 else self.card_number)+ self.label = " ".join(parts)+ return selfdiff --git a/flowsint-types/src/flowsint_types/device.py b/flowsint-types/src/flowsint_types/device.pyindex 829c7c15..6f8995f2 100644--- a/flowsint-types/src/flowsint_types/device.py+++ b/flowsint-types/src/flowsint_types/device.py@@ -1,8 +1,10 @@-from pydantic import BaseModel, Field-from typing import Optional, List+from pydantic import Field, model_validator+from typing import Optional, List, Self+from .flowsint_base import FlowsintType-class Device(BaseModel):++class Device(FlowsintType):"""Represents a device with hardware, software, and network information."""device_id: str = Field(@@ -63,3 +65,15 @@ class Device(BaseModel):source: Optional[str] = Field(None, description="Source of device information", title="Source")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ parts = []+ if self.manufacturer:+ parts.append(self.manufacturer)+ if self.model:+ parts.append(self.model)+ if not parts:+ parts.append(self.device_id)+ self.label = " ".join(parts)+ return selfdiff --git a/flowsint-types/src/flowsint_types/dns_record.py b/flowsint-types/src/flowsint_types/dns_record.pyindex aa20a02c..9cca5d11 100644--- a/flowsint-types/src/flowsint_types/dns_record.py+++ b/flowsint-types/src/flowsint_types/dns_record.py@@ -1,8 +1,10 @@-from pydantic import BaseModel, Field-from typing import Optional, List+from pydantic import Field, model_validator+from typing import Optional, List, Self+from .flowsint_base import FlowsintType-class DNSRecord(BaseModel):++class DNSRecord(FlowsintType):"""Represents a DNS record with type, value, and security information."""record_type: str = Field(@@ -48,3 +50,8 @@ class DNSRecord(BaseModel):threat_level: Optional[str] = Field(None, description="Threat level assessment", title="Threat Level")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ self.label = f"{self.name} ({self.record_type})"+ return selfdiff --git a/flowsint-types/src/flowsint_types/document.py b/flowsint-types/src/flowsint_types/document.pyindex 7be38dee..2c6f09c7 100644--- a/flowsint-types/src/flowsint_types/document.py+++ b/flowsint-types/src/flowsint_types/document.py@@ -1,8 +1,10 @@-from pydantic import BaseModel, Field-from typing import Optional, List+from pydantic import Field, model_validator+from typing import Optional, List, Self+from .flowsint_base import FlowsintType-class Document(BaseModel):++class Document(FlowsintType):"""Represents a document with metadata, security, and content information."""title: str = Field(..., description="Document title", title="Title")@@ -56,3 +58,8 @@ class Document(BaseModel):source: Optional[str] = Field(None, description="Source of document information", title="Source")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ self.label = self.title+ return selfdiff --git a/flowsint-types/src/flowsint_types/domain.py b/flowsint-types/src/flowsint_types/domain.pyindex 00db94f6..a46037b5 100644--- a/flowsint-types/src/flowsint_types/domain.py+++ b/flowsint-types/src/flowsint_types/domain.py@@ -1,46 +1,44 @@-from typing import Optional, Union, Any-from pydantic import BaseModel, Field, field_validator, model_validator+from typing import Optional, Self+from pydantic import Field, field_validator, model_validatorfrom urllib.parse import urlparseimport re+from .flowsint_base import FlowsintType-class Domain(BaseModel):+class Domain(FlowsintType):"""Represents a domain name and its properties."""- domain: str = Field(..., description="Domain name", title="Domain Name")+ domain: str = Field(+ ...,+ description="Domain name",+ title="Domain name",+ json_schema_extra={"primary": True},+ )root: Optional[bool] = Field(True, description="Is root or not", title="Is Root Domain")- @model_validator(mode='before')- @classmethod- def convert_string_to_dict(cls, data: Any) -> Any:- """Allow creating Domain from a string directly."""- if isinstance(data, str):- return {'domain': data}- return data-- @field_validator('domain')+ @field_validator("domain")@classmethoddef validate_domain(cls, v: str) -> str:- """Validate that the domain is valid."""try:- # Parse URL to extract hostnameparsed = urlparse(v if "://" in v else "http://" + v)hostname = parsed.hostname or v-- # Check that domain has at least one dotif not hostname or "." not in hostname:- raise ValueError(f"Invalid domain format: {v}")-- # Validate domain format with regex+ raise ValueErrorif not re.match(r"^[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$", hostname):- raise ValueError(f"Invalid domain format: {v}")-- # Return the cleaned hostname (without protocol)+ raise ValueErrorreturn hostname- except Exception as e:- raise ValueError(f"Invalid domain: {v}") from e---Domain.model_rebuild()+ except Exception:+ raise ValueError(f"Invalid domain: {v}")++ @model_validator(mode="after")+ def check_root(self) -> Self:+ parts = self.domain.split(".")+ self.root = len(parts) == 2+ return self++ @model_validator(mode="after")+ def compute_label(self) -> Self:+ self.label = self.domain+ return selfdiff --git a/flowsint-types/src/flowsint_types/email.py b/flowsint-types/src/flowsint_types/email.pyindex f5405574..3630460d 100644--- a/flowsint-types/src/flowsint_types/email.py+++ b/flowsint-types/src/flowsint_types/email.py@@ -1,16 +1,14 @@-from pydantic import BaseModel, Field, EmailStr, model_validator-from typing import Any+from pydantic import Field, EmailStr, model_validator+from typing import Any, Self+from .flowsint_base import FlowsintType-class Email(BaseModel):+class Email(FlowsintType):"""Represents an email address."""email: EmailStr = Field(..., description="Email address", title="Email Address")- @model_validator(mode='before')- @classmethod- def convert_string_to_dict(cls, data: Any) -> Any:- """Allow creating Email from a string directly."""- if isinstance(data, str):- return {'email': data}- return data+ @model_validator(mode='after')+ def compute_label(self) -> Self:+ self.label = self.email+ return selfdiff --git a/flowsint-types/src/flowsint_types/file.py b/flowsint-types/src/flowsint_types/file.pyindex 608ec3a3..fea549b9 100644--- a/flowsint-types/src/flowsint_types/file.py+++ b/flowsint-types/src/flowsint_types/file.py@@ -1,8 +1,10 @@-from pydantic import BaseModel, Field-from typing import Optional, List+from pydantic import Field, model_validator+from typing import Optional, List, Self+from .flowsint_base import FlowsintType-class File(BaseModel):++class File(FlowsintType):"""Represents a file with metadata, type information, and security assessment."""filename: str = Field(..., description="File name", title="Filename")@@ -61,3 +63,8 @@ class File(BaseModel):threat_level: Optional[str] = Field(None, description="Threat level assessment", title="Threat Level")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ self.label = self.filename+ return selfdiff --git a/flowsint-types/src/flowsint_types/flowsint_base.py b/flowsint-types/src/flowsint_types/flowsint_base.pynew file mode 100644index 00000000..9cc9312c--- /dev/null+++ b/flowsint-types/src/flowsint_types/flowsint_base.py@@ -0,0 +1,11 @@+from pydantic import BaseModel, Field+from typing import Optional+++class FlowsintType(BaseModel):+ """Base class for all Flowsint entity types with label support.+ Label is optional but computed at definition time.+ """+ label: Optional[str] = Field(+ None, description="UI-readable label for this entity, the one used on the graph.", title="Label"+ )diff --git a/flowsint-types/src/flowsint_types/gravatar.py b/flowsint-types/src/flowsint_types/gravatar.pyindex cc1cb059..cf1c5554 100644--- a/flowsint-types/src/flowsint_types/gravatar.py+++ b/flowsint-types/src/flowsint_types/gravatar.py@@ -1,8 +1,9 @@-from pydantic import BaseModel, Field, HttpUrl-from typing import Optional, List+from pydantic import Field, HttpUrl, model_validator+from typing import Optional, List, Self+from .flowsint_base import FlowsintType-class Gravatar(BaseModel):+class Gravatar(FlowsintType):"""Represents a Gravatar profile with image and user information."""src: HttpUrl = Field(..., description="Gravatar image URL", title="Image URL")@@ -42,3 +43,12 @@ class Gravatar(BaseModel):large_url: Optional[HttpUrl] = Field(None, description="Larger version of the image", title="Large Image URL")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ # Use display name if available, otherwise hash+ if self.display_name:+ self.label = f"{self.display_name}"+ else:+ self.label = f"{self.hash}"+ return selfdiff --git a/flowsint-types/src/flowsint_types/individual.py b/flowsint-types/src/flowsint_types/individual.pyindex dc670474..4fcdb8c6 100644--- a/flowsint-types/src/flowsint_types/individual.py+++ b/flowsint-types/src/flowsint_types/individual.py@@ -1,12 +1,13 @@-from pydantic import BaseModel, Field, field_validator-from typing import Optional, Literal, List, Union+from pydantic import Field, field_validator, model_validator+from typing import Optional, Literal, List, Union, Selffrom .address import Locationfrom .email import Emailfrom .phone import Phonefrom .ip import Ip+from .flowsint_base import FlowsintType-class Individual(BaseModel):+class Individual(FlowsintType):"""Represents an individual person with comprehensive personal information."""# Basic Information@@ -16,8 +17,8 @@ class Individual(BaseModel):last_name: str = Field(..., description="Last name of the individual", title="Last Name")- full_name: str = Field(- ..., description="Full name of the individual", title="Full Name"+ full_name: Optional[str] = Field(+ None, description="Full name of the individual", title="Full Name")middle_name: Optional[str] = Field(None, description="Middle name or initial", title="Middle Name"@@ -321,7 +322,6 @@ class Individual(BaseModel):for email in v:if not email:continue-try:# If already an Email object, keep itif isinstance(email, Email):@@ -344,12 +344,10 @@ class Individual(BaseModel):"""Validate phone numbers in the list and convert to Phone objects."""if v is None:return None-validated_phones = []for phone in v:if not phone:continue-try:# If already a Phone object, keep itif isinstance(phone, Phone):@@ -363,7 +361,6 @@ class Individual(BaseModel):except Exception:# Skip invalid phone numberscontinue-return validated_phones if validated_phones else None@field_validator('ip_addresses', mode='before')@@ -372,12 +369,10 @@ class Individual(BaseModel):"""Validate that all IP addresses in the list are valid and convert to Ip objects."""if v is None:return None-validated_ips = []for ip in v:if not ip:continue-try:# If already an Ip object, keep itif isinstance(ip, Ip):@@ -393,3 +388,16 @@ class Individual(BaseModel):continuereturn validated_ips if validated_ips else None++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ # Use full_name if available, otherwise concatenate first and last name+ if self.full_name:+ self.label = self.full_name+ elif self.first_name and self.last_name:+ self.label = f"{self.first_name} {self.last_name}"+ elif self.first_name:+ self.label = self.first_name+ elif self.last_name:+ self.label = self.last_name+ return selfdiff --git a/flowsint-types/src/flowsint_types/ip.py b/flowsint-types/src/flowsint_types/ip.pyindex a329df47..d4fdb688 100644--- a/flowsint-types/src/flowsint_types/ip.py+++ b/flowsint-types/src/flowsint_types/ip.py@@ -1,12 +1,18 @@-from pydantic import BaseModel, Field, field_validator, model_validator-from typing import Optional, Any+from pydantic import Field, field_validator, model_validator+from typing import Optional, Any, Selfimport ipaddress+from .flowsint_base import FlowsintType-class Ip(BaseModel):+class Ip(FlowsintType):"""Represents an IP address with geolocation and ISP information."""- address: str = Field(..., description="IP address", title="IP Address")+ address: str = Field(+ ...,+ description="IP address",+ title="IP Address",+ json_schema_extra={"primary": True},+ )latitude: Optional[float] = Field(None, description="Latitude coordinate of the IP location", title="Latitude")@@ -23,15 +29,7 @@ class Ip(BaseModel):None, description="Internet Service Provider", title="ISP")- @model_validator(mode='before')- @classmethod- def convert_string_to_dict(cls, data: Any) -> Any:- """Allow creating Ip from a string directly."""- if isinstance(data, str):- return {'address': data}- return data-- @field_validator('address')+ @field_validator("address")@classmethoddef validate_ip_address(cls, v: str) -> str:"""Validate that the address is a valid IP address."""@@ -40,3 +38,8 @@ class Ip(BaseModel):return vexcept ValueError:raise ValueError(f"Invalid IP address: {v}")++ @model_validator(mode="after")+ def compute_label(self) -> Self:+ self.label = self.address+ return selfdiff --git a/flowsint-types/src/flowsint_types/leak.py b/flowsint-types/src/flowsint_types/leak.pyindex ec392304..b8cd252d 100644--- a/flowsint-types/src/flowsint_types/leak.py+++ b/flowsint-types/src/flowsint_types/leak.py@@ -1,8 +1,10 @@-from typing import List, Dict, Optional-from pydantic import BaseModel, Field+from typing import List, Dict, Optional, Self+from pydantic import Field, model_validator+from .flowsint_base import FlowsintType-class Leak(BaseModel):++class Leak(FlowsintType):"""Represents a data leak or breach with associated data."""name: str = Field(@@ -11,3 +13,8 @@ class Leak(BaseModel):leak: Optional[List[Dict]] = Field(None, description="List of data leaks found", title="Leak Data")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ self.label = self.name+ return selfdiff --git a/flowsint-types/src/flowsint_types/malware.py b/flowsint-types/src/flowsint_types/malware.pyindex b6c3c019..cacfab37 100644--- a/flowsint-types/src/flowsint_types/malware.py+++ b/flowsint-types/src/flowsint_types/malware.py@@ -1,8 +1,10 @@-from pydantic import BaseModel, Field-from typing import Optional, List+from pydantic import Field, model_validator+from typing import Optional, List, Self+from .flowsint_base import FlowsintType-class Malware(BaseModel):++class Malware(FlowsintType):"""Represents malware with family, capabilities, and threat intelligence."""name: str = Field(..., description="Malware name or identifier", title="Name")@@ -58,3 +60,11 @@ class Malware(BaseModel):sample_hashes: Optional[List[str]] = Field(None, description="Sample file hashes", title="Sample Hashes")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ if self.family:+ self.label = f"{self.name} ({self.family})"+ else:+ self.label = self.name+ return selfdiff --git a/flowsint-types/src/flowsint_types/message.py b/flowsint-types/src/flowsint_types/message.pyindex a16875e3..16ac231b 100644--- a/flowsint-types/src/flowsint_types/message.py+++ b/flowsint-types/src/flowsint_types/message.py@@ -1,8 +1,10 @@-from pydantic import BaseModel, Field-from typing import Optional, List+from pydantic import Field, model_validator+from typing import Optional, List, Self+from .flowsint_base import FlowsintType-class Message(BaseModel):++class Message(FlowsintType):"""Represents a message with content, metadata, and security analysis."""message_id: str = Field(@@ -60,3 +62,13 @@ class Message(BaseModel):threat_level: Optional[str] = Field(None, description="Threat level assessment", title="Threat Level")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ if self.subject:+ self.label = self.subject+ else:+ # Truncate content to first 50 characters+ content_preview = self.content[:50] + "..." if len(self.content) > 50 else self.content+ self.label = content_preview+ return selfdiff --git a/flowsint-types/src/flowsint_types/organization.py b/flowsint-types/src/flowsint_types/organization.pyindex 3f84dc9c..e1b538ba 100644--- a/flowsint-types/src/flowsint_types/organization.py+++ b/flowsint-types/src/flowsint_types/organization.py@@ -1,10 +1,11 @@-from pydantic import BaseModel, Field, model_validator-from typing import Any, Optional, List+from pydantic import Field, model_validator+from typing import Any, Optional, List, Selffrom .individual import Individualfrom .address import Location+from .flowsint_base import FlowsintType-class Organization(BaseModel):+class Organization(FlowsintType):"""Represents an organization with detailed business and administrative information."""# Basic information@@ -369,3 +370,14 @@ class Organization(BaseModel):complements_type_siae: Optional[Any] = Field(None, description="Complements SIAE type", title="SIAE Type")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ # Use the full name if available, otherwise use name+ if self.nom_complet:+ self.label = str(self.nom_complet)+ elif self.nom_raison_sociale:+ self.label = str(self.nom_raison_sociale)+ elif self.name:+ self.label = str(self.name)+ return selfdiff --git a/flowsint-types/src/flowsint_types/phone.py b/flowsint-types/src/flowsint_types/phone.pyindex 6e503be3..3b3aaef9 100644--- a/flowsint-types/src/flowsint_types/phone.py+++ b/flowsint-types/src/flowsint_types/phone.py@@ -1,10 +1,11 @@-from pydantic import BaseModel, Field, field_validator, model_validator-from typing import Optional, Any+from pydantic import Field, field_validator, model_validator+from typing import Optional, Any, Selfimport phonenumbersfrom phonenumbers import NumberParseException+from .flowsint_base import FlowsintType-class Phone(BaseModel):+class Phone(FlowsintType):"""Represents a phone number with country and carrier information."""number: str = Field(..., description="Phone number", title="Phone Number")@@ -54,3 +55,8 @@ class Phone(BaseModel):# If all attempts fail, raise an errorraise ValueError(f"Invalid phone number: {v}. Must be in international format (+...) or a valid format for common regions.")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ self.label = self.number+ return selfdiff --git a/flowsint-types/src/flowsint_types/phrase.py b/flowsint-types/src/flowsint_types/phrase.pyindex 678832c8..d9b4dd54 100644--- a/flowsint-types/src/flowsint_types/phrase.py+++ b/flowsint-types/src/flowsint_types/phrase.py@@ -1,10 +1,19 @@-from pydantic import BaseModel, Field-from typing import Any+from pydantic import Field, model_validator+from typing import Any, Self+from .flowsint_base import FlowsintType-class Phrase(BaseModel):++class Phrase(FlowsintType):"""Represents a phrase or text content."""text: Any = Field(..., description="The content of the phrase.", title="Phrase text value.")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ text_str = str(self.text)+ # Truncate to 100 characters for display+ self.label = text_str[:100] + "..." if len(text_str) > 100 else text_str+ return selfdiff --git a/flowsint-types/src/flowsint_types/port.py b/flowsint-types/src/flowsint_types/port.pyindex 384bede9..aec16091 100644--- a/flowsint-types/src/flowsint_types/port.py+++ b/flowsint-types/src/flowsint_types/port.py@@ -1,8 +1,9 @@-from pydantic import BaseModel, Field, field_validator-from typing import Optional+from pydantic import Field, field_validator, model_validator+from typing import Optional, Self+from .flowsint_base import FlowsintType-class Port(BaseModel):+class Port(FlowsintType):"""Represents an open network port related to an IP address."""number: int = Field(..., description="Port number", title="Port Number")@@ -26,3 +27,14 @@ class Port(BaseModel):if not (0 <= v <= 65535):raise ValueError(f"Port number must be between 0 and 65535, got {v}")return v++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ # Include service and protocol if available+ parts = [str(self.number)]+ if self.service:+ parts.append(self.service)+ if self.protocol:+ parts.append(f"({self.protocol})")+ self.label = " ".join(parts)+ return selfdiff --git a/flowsint-types/src/flowsint_types/reputation_score.py b/flowsint-types/src/flowsint_types/reputation_score.pyindex c08693e9..93a2e6fa 100644--- a/flowsint-types/src/flowsint_types/reputation_score.py+++ b/flowsint-types/src/flowsint_types/reputation_score.py@@ -1,8 +1,10 @@-from pydantic import BaseModel, Field-from typing import Optional, List+from pydantic import Field, model_validator+from typing import Optional, List, Self+from .flowsint_base import FlowsintType-class ReputationScore(BaseModel):++class ReputationScore(FlowsintType):"""Represents a reputation score for an entity with historical data and trends."""entity_id: str = Field(..., description="Entity identifier", title="Entity ID")@@ -54,3 +56,11 @@ class ReputationScore(BaseModel):recommendations: Optional[List[str]] = Field(None, description="Recommendations based on score", title="Recommendations")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ parts = [self.entity_id]+ if self.score is not None:+ parts.append(f"Score: {self.score}")+ self.label = " - ".join(parts)+ return selfdiff --git a/flowsint-types/src/flowsint_types/risk_profile.py b/flowsint-types/src/flowsint_types/risk_profile.pyindex 38a52326..ac227cdb 100644--- a/flowsint-types/src/flowsint_types/risk_profile.py+++ b/flowsint-types/src/flowsint_types/risk_profile.py@@ -1,8 +1,10 @@-from pydantic import BaseModel, Field-from typing import Optional, List+from pydantic import Field, model_validator+from typing import Optional, List, Self+from .flowsint_base import FlowsintType-class RiskProfile(BaseModel):++class RiskProfile(FlowsintType):"""Represents a comprehensive risk assessment profile for an entity."""entity_id: str = Field(..., description="Entity identifier", title="Entity ID")@@ -71,3 +73,11 @@ class RiskProfile(BaseModel):next_review_date: Optional[str] = Field(None, description="Next review date", title="Next Review Date")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ parts = [self.entity_id]+ if self.risk_level:+ parts.append(f"Risk: {self.risk_level}")+ self.label = " - ".join(parts)+ return selfdiff --git a/flowsint-types/src/flowsint_types/script.py b/flowsint-types/src/flowsint_types/script.pyindex 7d564fe9..c7bed277 100644--- a/flowsint-types/src/flowsint_types/script.py+++ b/flowsint-types/src/flowsint_types/script.py@@ -1,8 +1,10 @@-from pydantic import BaseModel, Field-from typing import Optional, List+from pydantic import Field, model_validator+from typing import Optional, List, Self+from .flowsint_base import FlowsintType-class Script(BaseModel):++class Script(FlowsintType):"""Represents a script or code file with analysis and security information."""script_id: str = Field(@@ -63,3 +65,11 @@ class Script(BaseModel):minified: Optional[bool] = Field(None, description="Whether script is minified", title="Minified")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ if self.name:+ self.label = self.name+ else:+ self.label = self.script_id+ return selfdiff --git a/flowsint-types/src/flowsint_types/session.py b/flowsint-types/src/flowsint_types/session.pyindex 9d025e4e..bd3692f8 100644--- a/flowsint-types/src/flowsint_types/session.py+++ b/flowsint-types/src/flowsint_types/session.py@@ -1,8 +1,10 @@-from pydantic import BaseModel, Field-from typing import Optional, List+from pydantic import Field, model_validator+from typing import Optional, List, Self+from .flowsint_base import FlowsintType-class Session(BaseModel):++class Session(FlowsintType):"""Represents a user session with device and activity information."""session_id: str = Field(@@ -49,3 +51,15 @@ class Session(BaseModel):source: Optional[str] = Field(None, description="Source of session information", title="Source")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ parts = []+ if self.user_id:+ parts.append(self.user_id)+ if self.service:+ parts.append(self.service)+ if not parts:+ parts.append(self.session_id)+ self.label = " - ".join(parts)+ return selfdiff --git a/flowsint-types/src/flowsint_types/social_account.py b/flowsint-types/src/flowsint_types/social_account.pyindex efa4b37f..a8954fdd 100644--- a/flowsint-types/src/flowsint_types/social_account.py+++ b/flowsint-types/src/flowsint_types/social_account.py@@ -1,9 +1,10 @@-from pydantic import BaseModel, Field-from typing import Optional, List+from pydantic import Field, model_validator+from typing import Optional, List, Selffrom .username import Username+from .flowsint_base import FlowsintType-class SocialAccount(BaseModel):+class SocialAccount(FlowsintType):"""Represents a social media account (the 'home' of a username)."""username: Username = Field(..., description="Username associated with this account", title="Username")@@ -22,3 +23,12 @@ class SocialAccount(BaseModel):is_suspended: Optional[bool] = Field(None, description="Whether the account is suspended/banned", title="Is suspended")associated_emails: Optional[List[str]] = Field(None, description="Email addresses associated with the account", title="Associated emails")associated_phones: Optional[List[str]] = Field(None, description="Phone numbers associated with the account", title="Associated phones")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ # Use display name if available, otherwise username+ if self.display_name:+ self.label = f"{self.display_name} (@{self.username.value})"+ else:+ self.label = f"@{self.username.value}"+ return selfdiff --git a/flowsint-types/src/flowsint_types/ssl_certificate.py b/flowsint-types/src/flowsint_types/ssl_certificate.pyindex 3e33c652..8abc92f4 100644--- a/flowsint-types/src/flowsint_types/ssl_certificate.py+++ b/flowsint-types/src/flowsint_types/ssl_certificate.py@@ -1,8 +1,10 @@-from pydantic import BaseModel, Field-from typing import Optional, List+from pydantic import Field, model_validator+from typing import Optional, List, Self+from .flowsint_base import FlowsintType-class SSLCertificate(BaseModel):++class SSLCertificate(FlowsintType):"""Represents an SSL/TLS certificate with validation and security details."""subject: str = Field(@@ -69,3 +71,8 @@ class SSLCertificate(BaseModel):fingerprint_sha256: Optional[str] = Field(None, description="SHA256 fingerprint", title="SHA256 Fingerprint")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ self.label = self.subject+ return selfdiff --git a/flowsint-types/src/flowsint_types/username.py b/flowsint-types/src/flowsint_types/username.pyindex 732d3b0e..14c2b51c 100644--- a/flowsint-types/src/flowsint_types/username.py+++ b/flowsint-types/src/flowsint_types/username.py@@ -1,9 +1,10 @@-from pydantic import BaseModel, Field, field_validator, model_validator-from typing import Optional, Any+from pydantic import Field, field_validator, model_validator+from typing import Optional, Any, Selfimport re+from .flowsint_base import FlowsintType-class Username(BaseModel):+class Username(FlowsintType):"""Represents a username or handle on any platform."""value: str = Field(..., description="Username or handle string", title="Username value")@@ -12,27 +13,27 @@ class Username(BaseModel):None, description="Last time this username was observed", title="Last seen at")- @model_validator(mode='before')- @classmethod- def convert_string_to_dict(cls, data: Any) -> Any:- """Allow creating Username from a string directly."""- if isinstance(data, str):- return {'value': data}- return data-@field_validator('value')@classmethoddef validate_username(cls, v: str) -> str:"""Validate username format.- Username must be 3-30 characters long and contain only:+ Username must be 3-80 characters long and contain only:- Letters (a-z, A-Z)- Numbers (0-9)- Underscores (_)- Hyphens (-)"""- if not re.match(r"^[a-zA-Z0-9_-]{3,30}$", v):+ if not re.match(r"^[a-zA-Z0-9_-]{3,80}$", v):raise ValueError(- f"Invalid username: {v}. Must be 3-30 characters and contain only letters, numbers, underscores, and hyphens."+ f"Invalid username: {v}. Must be 3-80 characters and contain only letters, numbers, underscores, and hyphens.")return v++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ if self.platform:+ self.label = f"@{self.value} ({self.platform})"+ else:+ self.label = f"@{self.value}"+ return selfdiff --git a/flowsint-types/src/flowsint_types/wallet.py b/flowsint-types/src/flowsint_types/wallet.pyindex 6fd47b66..95f04a40 100644--- a/flowsint-types/src/flowsint_types/wallet.py+++ b/flowsint-types/src/flowsint_types/wallet.py@@ -1,8 +1,10 @@-from typing import Optional-from pydantic import BaseModel, Field, HttpUrl+from typing import Optional, Self+from pydantic import Field, HttpUrl, field_validator, model_validator+import re+from .flowsint_base import FlowsintType-class CryptoWallet(BaseModel):+class CryptoWallet(FlowsintType):"""Represents a cryptocurrency wallet."""address: str = Field(..., description="Wallet address", title="Wallet Address")@@ -10,8 +12,41 @@ class CryptoWallet(BaseModel):None, description="Wallet Explorer node ID", title="Node ID")+ @field_validator('address')+ @classmethod+ def validate_address(cls, v: str) -> str:+ """Validate that the wallet address is not empty and has a valid format."""+ if not v or not v.strip():+ raise ValueError("Wallet address cannot be empty")-class CryptoWalletTransaction(BaseModel):+ # Strip whitespace+ v = v.strip()++ # Basic validation: check if it looks like a valid crypto address+ # Ethereum addresses start with 0x and are 42 characters (0x + 40 hex chars)+ # Bitcoin addresses vary but are typically 26-35 characters+ # We'll do a permissive check for common formats+ if len(v) < 26:+ raise ValueError("Wallet address is too short to be valid")++ # Check for common patterns+ ethereum_pattern = r'^0x[a-fA-F0-9]{40}$'+ bitcoin_pattern = r'^[13][a-km-zA-HJ-NP-Z1-9]{25,34}$|^bc1[a-z0-9]{39,59}$'++ # If it matches Ethereum pattern, validate it+ if v.startswith('0x'):+ if not re.match(ethereum_pattern, v):+ raise ValueError("Invalid Ethereum address format")++ return v++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ self.label = self.address+ return self+++class CryptoWalletTransaction(FlowsintType):"""Represents a cryptocurrency transaction."""source: CryptoWallet = Field(@@ -82,8 +117,35 @@ class CryptoWalletTransaction(BaseModel):None, description="Error message if transaction failed", title="Error Message")+ @field_validator('value', 'amount', 'amount_usd')+ @classmethod+ def validate_positive_amounts(cls, v: Optional[float]) -> Optional[float]:+ """Validate that monetary amounts are non-negative."""+ if v is not None and v < 0:+ raise ValueError("Monetary amounts must be non-negative")+ return v++ @field_validator('gas', 'gas_price', 'gas_used', 'cumulative_gas_used', 'block_number', 'nonce', 'transaction_index', 'confirmations', 'hop')+ @classmethod+ def validate_non_negative_integers(cls, v: Optional[int]) -> Optional[int]:+ """Validate that integer fields are non-negative."""+ if v is not None and v < 0:+ raise ValueError("Integer values must be non-negative")+ return v-class CryptoNFT(BaseModel):+ @model_validator(mode='after')+ def compute_label(self) -> Self:+ # Use hash if available, otherwise create a descriptive label+ if self.hash:+ self.label = self.hash+ elif self.source and self.target:+ self.label = f"Transaction from {self.source.address[:8]}... to {self.target.address[:8]}..."+ elif self.source:+ self.label = f"Transaction from {self.source.address[:8]}..."+ return self+++class CryptoNFT(FlowsintType):"""Represents a Non-Fungible Token (NFT) held or minted by a wallet."""wallet: CryptoWallet = Field(..., description="Source wallet", title="Wallet")@@ -135,6 +197,41 @@ class CryptoNFT(BaseModel):def uid(self):return f"{self.contract_address}:{self.token_id}"+ @field_validator('contract_address')+ @classmethod+ def validate_contract_address(cls, v: str) -> str:+ """Validate that the NFT contract address has a valid format."""+ if not v or not v.strip():+ raise ValueError("Contract address cannot be empty")++ v = v.strip()++ # NFT contracts are typically on Ethereum, so validate as Ethereum address+ ethereum_pattern = r'^0x[a-fA-F0-9]{40}$'+ if not re.match(ethereum_pattern, v):+ raise ValueError("Invalid contract address format (expected Ethereum address: 0x followed by 40 hex characters)")++ return v++ @field_validator('token_id')+ @classmethod+ def validate_token_id(cls, v: str) -> str:+ """Validate that the token ID is not empty."""+ if not v or not v.strip():+ raise ValueError("Token ID cannot be empty")+ return v.strip()++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ # Prefer name, then collection_name with token_id, fallback to uid+ if self.name:+ self.label = self.name+ elif self.collection_name:+ self.label = f"{self.collection_name} #{self.token_id}"+ else:+ self.label = self.uid+ return self+# Update forward referencesCryptoWallet.model_rebuild()diff --git a/flowsint-types/src/flowsint_types/weapon.py b/flowsint-types/src/flowsint_types/weapon.pyindex ee0f4249..e418ef29 100644--- a/flowsint-types/src/flowsint_types/weapon.py+++ b/flowsint-types/src/flowsint_types/weapon.py@@ -1,8 +1,10 @@-from pydantic import BaseModel, Field-from typing import Literal, Optional, List+from pydantic import Field, model_validator+from typing import Literal, Optional, List, Self+from .flowsint_base import FlowsintType-class Weapon(BaseModel):++class Weapon(FlowsintType):"""Represents a weapon with detailed specifications and forensic information."""name: str = Field(..., description="Weapon name or identifier", title="Name")@@ -112,3 +114,11 @@ class Weapon(BaseModel):notes: Optional[str] = Field(None, description="Additional notes or observations", title="Notes")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ parts = [self.name]+ if self.type:+ parts.append(f"({self.type})")+ self.label = " ".join(parts)+ return selfdiff --git a/flowsint-types/src/flowsint_types/web_tracker.py b/flowsint-types/src/flowsint_types/web_tracker.pyindex 36ba0450..c9eb5e83 100644--- a/flowsint-types/src/flowsint_types/web_tracker.py+++ b/flowsint-types/src/flowsint_types/web_tracker.py@@ -1,8 +1,10 @@-from pydantic import BaseModel, Field-from typing import Optional, List+from pydantic import Field, model_validator+from typing import Optional, List, Self+from .flowsint_base import FlowsintType-class WebTracker(BaseModel):++class WebTracker(FlowsintType):"""Represents a web tracking technology with privacy and compliance information."""tracker_id: str = Field(@@ -57,3 +59,11 @@ class WebTracker(BaseModel):risk_level: Optional[str] = Field(None, description="Privacy risk level", title="Risk Level")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ if self.name:+ self.label = self.name+ else:+ self.label = self.tracker_id+ return selfdiff --git a/flowsint-types/src/flowsint_types/website.py b/flowsint-types/src/flowsint_types/website.pyindex 3e986360..bf3d9ef0 100644--- a/flowsint-types/src/flowsint_types/website.py+++ b/flowsint-types/src/flowsint_types/website.py@@ -1,9 +1,10 @@-from typing import List, Optional-from pydantic import BaseModel, Field, HttpUrl+from typing import List, Optional, Self+from pydantic import Field, HttpUrl, model_validatorfrom .domain import Domain+from .flowsint_base import FlowsintType-class Website(BaseModel):+class Website(FlowsintType):"""Represents a website with its URL, domain, and redirect information."""url: HttpUrl = Field(@@ -18,3 +19,8 @@ class Website(BaseModel):active: Optional[bool] = Field(False, description="Whether the website is active", title="Is Active")++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ self.label = str(self.url)+ return selfdiff --git a/flowsint-types/src/flowsint_types/whois.py b/flowsint-types/src/flowsint_types/whois.pyindex 67335369..7d2516d6 100644--- a/flowsint-types/src/flowsint_types/whois.py+++ b/flowsint-types/src/flowsint_types/whois.py@@ -1,10 +1,12 @@-from typing import Optional, Union-from pydantic import BaseModel, Field, field_validator+from typing import Optional, Union, Self+from pydantic import Field, field_validator, model_validatorfrom .email import Emailfrom .domain import Domain+from .organization import Organization+from .flowsint_base import FlowsintType-class Whois(BaseModel):+class Whois(FlowsintType):"""Represents WHOIS domain registration information."""domain: Domain = Field(..., description="Domain information", title="Domain")@@ -14,9 +16,9 @@ class Whois(BaseModel):registrar: Optional[str] = Field(None, description="Domain registrar name", title="Registrar")- org: Optional[str] = Field(+ organization: Optional[Organization] = Field(None,- description="Organization name associated with the domain",+ description="Organization associated with the domain",title="Organization",)city: Optional[str] = Field(@@ -46,3 +48,26 @@ class Whois(BaseModel):elif isinstance(v, dict):return Domain(**v)return v++ @field_validator('organization', mode='before')+ @classmethod+ def convert_organization(cls, v: Union[str, dict, Organization, None]) -> Optional[Organization]:+ """Convert string or dict to Organization object if needed."""+ if v is None:+ return None+ if isinstance(v, Organization):+ return v+ elif isinstance(v, str):+ return Organization(name=v)+ elif isinstance(v, dict):+ return Organization(**v)+ return v++ @model_validator(mode='after')+ def compute_label(self) -> Self:+ # Use domain and organization if available+ if self.organization:+ self.label = f"{self.domain.domain} - {self.organization.name}"+ else:+ self.label = self.domain.domain+ return selfdiff --git a/flowsint-types/tests/domain.py b/flowsint-types/tests/domain.pynew file mode 100644index 00000000..5ffbd35c--- /dev/null+++ b/flowsint-types/tests/domain.py@@ -0,0 +1,42 @@+from flowsint_types.domain import Domain+import pytest+++def test_valid_domain_from_object():+ domain = Domain(**{"domain": "mydomain.com"})+ assert domain.domain == "mydomain.com"+ assert domain.label == "mydomain.com"+ assert domain.root == True+++def test_valid_subbomain_from_object():+ domain = Domain(**{"domain": "blog.mydomain.com"})+ assert domain.domain == "blog.mydomain.com"+ assert domain.label == "blog.mydomain.com"+ assert domain.root == False+++def test_valid_domain_from_instance():+ domain = Domain(domain="mydomain.com")+ assert domain.domain == "mydomain.com"+ assert domain.label == "mydomain.com"+ assert domain.root == True+++def test_valid_subdomain_from_instance():+ domain = Domain(domain="blog.mydomain.com")+ assert domain.domain == "blog.mydomain.com"+ assert domain.label == "blog.mydomain.com"+ assert domain.root == False+++def test_invalid_domain_from_object():+ with pytest.raises(Exception) as e_info:+ Domain(**{"domain": "my_domain.com"})+ assert "Invalid domain" in str(e_info.value)+++def test_domain_type_from_none():+ with pytest.raises(Exception) as e_info:+ Domain()+ assert "1 validation error for Domain" in str(e_info.value)diff --git a/flowsint-types/tests/individual.py b/flowsint-types/tests/individual.pynew file mode 100644index 00000000..cfe81bdc--- /dev/null+++ b/flowsint-types/tests/individual.py@@ -0,0 +1,15 @@+from flowsint_types.individual import Individual+import pytest+++def test_valid_individual():+ indivudual = Individual(first_name="John", last_name="Doe")+ assert indivudual.first_name == "John"+ assert indivudual.last_name == "Doe"+ assert indivudual.label == "John Doe"+++def test_invalid_individual():+ with pytest.raises(Exception) as e_info:+ Individual(name="John Doe")+ assert "validation errors for Individual" in str(e_info.value)diff --git a/flowsint-types/tests/ip.py b/flowsint-types/tests/ip.pynew file mode 100644index 00000000..5c34151e--- /dev/null+++ b/flowsint-types/tests/ip.py@@ -0,0 +1,14 @@+from flowsint_types.ip import Ip+import pytest+++def test_valid_ip():+ ip = Ip(address="12.23.34.56")+ assert ip.address == "12.23.34.56"+ assert ip.label == "12.23.34.56"+++def test_invalid_ip():+ with pytest.raises(Exception) as e_info:+ Ip(address="12.23.34.564")+ assert "Invalid IP address" in str(e_info.value)