A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat: whoxy scanners
dextmorgn5a34f3c87a4a
diff --git a/flowsint-core/src/flowsint_core/core/registry.py b/flowsint-core/src/flowsint_core/core/registry.pyindex 9d3a4eb7..91b8dc78 100644--- a/flowsint-core/src/flowsint_core/core/registry.py+++ b/flowsint-core/src/flowsint_core/core/registry.py@@ -9,6 +9,8 @@ from flowsint_transforms.domain.to_ip import ResolveScannerfrom flowsint_transforms.domain.to_website import DomainToWebsiteScannerfrom flowsint_transforms.domain.to_root_domain import DomainToRootDomainfrom flowsint_transforms.domain.to_asn import DomainToAsnScanner+from flowsint_transforms.domain.to_history import DomainToHistoryScanner+# IP-related scannersfrom flowsint_transforms.email.to_domains import EmailToDomainsScanner@@ -121,13 +123,14 @@ class TransformRegistry:return [cls._create_scanner_metadata(scanner)for scanner in cls._scanners.values()+ if scanner.name() not in exclude]return [cls._create_scanner_metadata(scanner)for scanner in cls._scanners.values()if scanner.input_schema()["type"].lower() in ["any", input_type_lower]- and scanner.name not in exclude+ and scanner.name() not in exclude]@@ -141,6 +144,7 @@ TransformRegistry.register(WhoisScanner)TransformRegistry.register(DomainToWebsiteScanner)TransformRegistry.register(DomainToRootDomain)TransformRegistry.register(DomainToAsnScanner)+TransformRegistry.register(DomainToHistoryScanner)# IP-related scannersTransformRegistry.register(GeolocationScanner)diff --git a/flowsint-core/src/flowsint_core/core/scanner_base.py b/flowsint-core/src/flowsint_core/core/scanner_base.pyindex 013fc53e..cce02292 100644--- a/flowsint-core/src/flowsint_core/core/scanner_base.py+++ b/flowsint-core/src/flowsint_core/core/scanner_base.py@@ -134,17 +134,17 @@ class Scanner(ABC):def resolve_params(self) -> Dict[str, Any]:resolved = {}- Logger.warn(+ Logger.debug(self.sketch_id, {"message": f"Params schema: {str(self.params_schema)}"})- Logger.warn(self.sketch_id, {"message": f"Params: {str(self.params)}"})- Logger.warn(+ Logger.debug(self.sketch_id, {"message": f"Params: {str(self.params)}"})+ Logger.debug(self.sketch_id,{"message": f"Params schema length: {len(self.params_schema)}"},)i = 1for param in self.params_schema:- Logger.warn(self.sketch_id, {"message": f"Param {i}: {str(param)}"})+ Logger.debug(self.sketch_id, {"message": f"Param {i}: {str(param)}"})i += 1if param["type"] == "vaultSecret":# Check if the vault secret parameter is provided@@ -356,11 +356,26 @@ class Scanner(ABC):if not self.neo4j_conn:return- properties["sketch_id"] = self.sketch_id- properties["label"] = properties.get("label", key_value)+ # Serialize properties to handle nested Pydantic objects+ serialized_properties = self._serialize_properties(properties)- set_clauses = [f"n.{prop} = ${prop}" for prop in properties.keys()]- params = {key_prop: key_value, **properties}+ # Ensure all values are Neo4j-compatible primitive types+ final_properties = {}+ for key, value in serialized_properties.items():+ if value is None:+ final_properties[key] = ""+ elif isinstance(value, (str, int, float, bool)):+ final_properties[key] = "" if value == "None" else value+ else:+ # Convert any remaining complex types to strings+ final_properties[key] = str(value)++ final_properties["type"] = node_type.lower()+ final_properties["sketch_id"] = self.sketch_id+ final_properties["label"] = final_properties.get("label", key_value)++ set_clauses = [f"n.{prop} = ${prop}" for prop in final_properties.keys()]+ params = {key_prop: key_value, **final_properties}query = f"""MERGE (n:{node_type} {{{key_prop}: ${key_prop}}})@@ -368,6 +383,60 @@ class Scanner(ABC):"""self.neo4j_conn.query(query, params)+ def _serialize_properties(self, properties: dict) -> dict:+ """Convert properties to Neo4j-compatible values, handling nested Pydantic objects."""+ serialized = {}++ for key, value in properties.items():++ if hasattr(value, "__dict__") and not isinstance(+ value, (str, int, float, bool)+ ):+ # Handle Pydantic objects and other complex objects+ if hasattr(value, "model_dump"): # Pydantic v2+ # Flatten the Pydantic object into individual properties+ flattened = value.model_dump()+ for nested_key, nested_value in flattened.items():+ if nested_value is not None:+ serialized[f"{key}_{nested_key}"] = nested_value+ elif hasattr(value, "dict"): # Pydantic v1+ # Flatten the Pydantic object into individual properties+ flattened = value.dict()+ for nested_key, nested_value in flattened.items():+ if nested_value is not None:+ serialized[f"{key}_{nested_key}"] = nested_value+ else:+ # For other objects, try to convert to dict or string+ try:+ flattened = value.__dict__+ for nested_key, nested_value in flattened.items():+ if nested_value is not None:+ serialized[f"{key}_{nested_key}"] = nested_value+ except:+ serialized[key] = str(value)+ elif isinstance(value, list):+ # Handle lists - convert all items to primitive types+ serialized_list = []+ for item in value:+ if hasattr(item, "__dict__") and not isinstance(+ item, (str, int, float, bool)+ ):+ # Convert complex objects to strings+ serialized_list.append(str(item))+ else:+ serialized_list.append(item)+ serialized[key] = serialized_list+ elif isinstance(value, dict):+ # Handle dictionaries - flatten them+ for dict_key, dict_value in value.items():+ if dict_value is not None:+ serialized[f"{key}_{dict_key}"] = dict_value+ else:+ # Keep primitive types as-is+ serialized[key] = value++ return serialized+def create_relationship(self,from_type: str,