A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat(core): only support pydantic object insert and manipulation
dextmorgn7562da12a65c
diff --git a/flowsint-core/src/flowsint_core/core/enricher_base.py b/flowsint-core/src/flowsint_core/core/enricher_base.pyindex c222ea29..43b351c8 100644--- a/flowsint-core/src/flowsint_core/core/enricher_base.py+++ b/flowsint-core/src/flowsint_core/core/enricher_base.py@@ -8,6 +8,7 @@ from .vault import VaultProtocolfrom .graph_service import GraphService, create_graph_servicefrom ..utils import resolve_type+class InvalidEnricherParams(Exception):pass@@ -413,9 +414,7 @@ class Enricher(ABC):async def execute(self, values: List[str]) -> List[Dict[str, Any]]:if self.name() != "enricher_orchestrator":- Logger.info(- self.sketch_id, {"message": f"Enricher {self.name()} started."}- )+ Logger.info(self.sketch_id, {"message": f"Enricher {self.name()} started."})try:await self.async_init()preprocessed = self.preprocess(values)@@ -440,240 +439,50 @@ class Enricher(ABC):)return []- def create_node(- self, node_type_or_obj, key_prop=None, key_value=None, **properties- ) -> None:+ def create_node(self, node_obj, **properties) -> None:"""Create a single Neo4j node.- This method now uses the GraphService for improved performance and- better separation of concerns.-The following properties are automatically added to every node:- type: Lowercase version of node_type- sketch_id: Current sketch ID from enricher context- label: Automatically computed by FlowsintType, or defaults to key_value if not provided- created_at: ISO 8601 UTC timestamp (only on creation, not updates)- Best Practice - Use Pydantic object directly:- The simplest way is to pass a Pydantic object directly. The node type,- key property, and key value are automatically inferred:-+ Use Pydantic object directly:```python- # Best: pass the Pydantic object directlyself.create_node(ip)-- # Also good if you need to override properties- self.create_node(domain, type="subdomain")```Args:- node_type_or_obj: Either a Pydantic object (FlowsintType), or node label string (e.g., "domain", "ip")- key_prop: Property name used as unique identifier (optional if passing Pydantic object)- key_value: Value of the key property (optional if passing Pydantic object)- **properties: Additional node properties or property overrides-- Note:- Uses MERGE semantics - if a node with the same (key_prop, sketch_id)- exists, it will be updated. The created_at field is only set on creation.- """- # Check if first argument is a Pydantic object- if isinstance(node_type_or_obj, BaseModel):- obj = node_type_or_obj-- # Infer node_type from class name (e.g., Ip -> "ip", Domain -> "domain")- node_type = obj.__class__.__name__.lower()-- # Get the primary field and its value- primary_field = self._get_primary_field(obj)- key_prop = primary_field- key_value = getattr(obj, primary_field)-- # If key_value is itself a Pydantic model, extract its primary value- if isinstance(key_value, BaseModel):- key_value = self._extract_primary_value(key_value)-- # Merge object properties with any overrides, but skip nested Pydantic objects- # Use model_dump(mode="json") to properly serialize Pydantic types (e.g., HttpUrl)- obj_dict = obj.model_dump(mode="json") if hasattr(obj, "model_dump") else obj.dict()- obj_properties = {}- for k, v in obj_dict.items():- # Skip nested Pydantic objects (represented as dicts after model_dump)- if not isinstance(v, dict):- obj_properties[k] = v- obj_properties.update(properties)- properties = obj_properties- else:- # Legacy signature: node_type_or_obj is the node_type string- node_type = node_type_or_obj-- self._graph_service.create_node(- node_type=node_type, key_prop=key_prop, key_value=key_value, **properties- )-- def _serialize_properties(self, properties: dict) -> dict:+ node_obj: Either a Pydantic object or node label string+ **properties: Additional node properties or overrides"""- Convert properties to Neo4j-compatible values.-- DEPRECATED: This method is kept for backward compatibility.- New code should use GraphSerializer directly.-- Args:- properties: Dictionary of properties to serialize-- Returns:- Dictionary of serialized properties- """- from .graph_serializer import GraphSerializer-- return GraphSerializer.serialize_properties(properties)+ self._graph_service.create_node(node_obj=node_obj, **properties)def create_relationship(self,- from_type_or_obj,- from_key_or_to_obj,- from_value_or_rel_type=None,- to_type=None,- to_key=None,- to_value=None,- rel_type=None,+ from_obj,+ to_obj,+ rel_label="IS_RELATED_TO",) -> None:"""Create a relationship between two nodes.- This method now uses the GraphService for improved performance and- better separation of concerns.-Best Practice - Use Pydantic objects directly:- The simplest way is to pass two Pydantic objects and the relationship type:-```python- # Best: pass Pydantic objects directlyself.create_relationship(individual, domain, "HAS_DOMAIN")self.create_relationship(email, breach, "FOUND_IN_BREACH")```- Legacy Usage:- You can still use the explicit signature for backward compatibility:-- ```python- # Legacy: explicit signature- self.create_relationship(- "individual", "full_name", individual.full_name,- "domain", "domain", domain_name,- "HAS_DOMAIN"- )- ```-Args:- from_type_or_obj: Either a Pydantic object (source node) or source node label string- from_key_or_to_obj: Either a Pydantic object (target node) or source node key property- from_value_or_rel_type: Either relationship type string (if using objects) or source node key value- to_type: Target node label (only for legacy signature)- to_key: Target node key property (only for legacy signature)- to_value: Target node key value (only for legacy signature)- rel_type: Relationship type (only for legacy signature)- """- # Check if using new signature (Pydantic objects)- if isinstance(from_type_or_obj, BaseModel) and isinstance(from_key_or_to_obj, BaseModel):- from_obj = from_type_or_obj- to_obj = from_key_or_to_obj- relationship_type = from_value_or_rel_type-- # Extract from_node info- from_node_type = from_obj.__class__.__name__.lower()- from_primary_field = self._get_primary_field(from_obj)-- # Use model_dump to properly serialize Pydantic types (e.g., HttpUrl)- from_obj_dict = from_obj.model_dump(mode="json") if hasattr(from_obj, "model_dump") else from_obj.dict()- from_key_value = from_obj_dict.get(from_primary_field)-- # If key_value is still a dict (nested Pydantic model), extract its primary value- if isinstance(from_key_value, dict):- # Get the raw nested object to extract its primary value- nested_obj = getattr(from_obj, from_primary_field)- if isinstance(nested_obj, BaseModel):- from_key_value = self._extract_primary_value(nested_obj)-- # Extract to_node info- to_node_type = to_obj.__class__.__name__.lower()- to_primary_field = self._get_primary_field(to_obj)-- # Use model_dump to properly serialize Pydantic types (e.g., HttpUrl)- to_obj_dict = to_obj.model_dump(mode="json") if hasattr(to_obj, "model_dump") else to_obj.dict()- to_key_value = to_obj_dict.get(to_primary_field)-- # If key_value is still a dict (nested Pydantic model), extract its primary value- if isinstance(to_key_value, dict):- # Get the raw nested object to extract its primary value- nested_obj = getattr(to_obj, to_primary_field)- if isinstance(nested_obj, BaseModel):- to_key_value = self._extract_primary_value(nested_obj)-- self._graph_service.create_relationship(- from_type=from_node_type,- from_key=from_primary_field,- from_value=from_key_value,- to_type=to_node_type,- to_key=to_primary_field,- to_value=to_key_value,- rel_type=relationship_type,- )- else:- # Legacy signature- self._graph_service.create_relationship(- from_type=from_type_or_obj,- from_key=from_key_or_to_obj,- from_value=from_value_or_rel_type,- to_type=to_type,- to_key=to_key,- to_value=to_value,- rel_type=rel_type,- )-- def _get_primary_field(self, obj: BaseModel) -> str:- """Helper method to get the primary field of a Pydantic object."""- # Access model_fields from the class, not the instance- model_fields = obj.__class__.model_fields-- # Find the primary field (marked with json_schema_extra={"primary": True})- primary_field = None- for field_name, field_info in model_fields.items():- if field_info.json_schema_extra and field_info.json_schema_extra.get("primary"):- primary_field = field_name- break-- # Fallback: use first required field or first field- if primary_field is None:- for field_name, field_info in model_fields.items():- if field_info.is_required():- primary_field = field_name- break- if primary_field is None:- primary_field = next(iter(model_fields.keys()))-- return primary_field-- def _extract_primary_value(self, obj: BaseModel) -> Any:+ from_obj: Either a Pydantic object (source) or source node label+ to_obj: Either a Pydantic object (target) or source node key property+ rel_label: Either relationship type (Pydantic) or source node key value"""- Extract the primitive value from a Pydantic object recursively.- If the primary field is itself a Pydantic object, recursively extract its primary value.- Uses model_dump to properly serialize Pydantic types like HttpUrl.- """- primary_field = self._get_primary_field(obj)-- # Use model_dump to properly serialize Pydantic types (e.g., HttpUrl)- obj_dict = obj.model_dump(mode="json") if hasattr(obj, "model_dump") else obj.dict()- value = obj_dict.get(primary_field)-- # If the value is still a dict (nested Pydantic model), recursively extract- if isinstance(value, dict):- # Get the raw nested object to recursively extract- nested_obj = getattr(obj, primary_field)- if isinstance(nested_obj, BaseModel):- return self._extract_primary_value(nested_obj)-- return value+ self._graph_service.create_relationship(+ from_obj=from_obj, to_obj=to_obj, rel_label=rel_label+ )def log_graph_message(self, message: str) -> None:"""diff --git a/flowsint-core/src/flowsint_core/core/graph_db.py b/flowsint-core/src/flowsint_core/core/graph_db.pyindex 0d7e200c..8e557877 100644--- a/flowsint-core/src/flowsint_core/core/graph_db.py+++ b/flowsint-core/src/flowsint_core/core/graph_db.py@@ -91,9 +91,17 @@ class Neo4jConnection:List of result records as dictionaries"""with self._driver.session() as session:- result = session.run(query, parameters or {})+ cleaned_params = self._clean_parameters(parameters)+ result = session.run(query, cleaned_params)return result.data()+ @staticmethod+ def _clean_parameters(parameters: Dict[str, Any]) -> Dict[str, Any]:+ """Remove None keys from parameters dict to avoid Neo4j errors."""+ if not parameters:+ return {}+ return {k: v for k, v in parameters.items() if k is not None}+def execute_write(self, query: str, parameters: Dict[str, Any] = None) -> List[Dict[str, Any]]:"""Execute a write query within a write transaction.@@ -106,7 +114,8 @@ class Neo4jConnection:List of result records as dictionaries"""def _execute(tx):- result = tx.run(query, parameters or {})+ cleaned_params = self._clean_parameters(parameters)+ result = tx.run(query, cleaned_params)return result.data()with self._driver.session() as session:@@ -121,7 +130,8 @@ class Neo4jConnection:"""def _execute_batch(tx):for query, params in queries:- tx.run(query, params or {})+ cleaned_params = self._clean_parameters(params)+ tx.run(query, cleaned_params)with self._driver.session() as session:session.execute_write(_execute_batch)diff --git a/flowsint-core/src/flowsint_core/core/graph_repository.py b/flowsint-core/src/flowsint_core/core/graph_repository.pyindex e12ae02b..9bec397a 100644--- a/flowsint-core/src/flowsint_core/core/graph_repository.py+++ b/flowsint-core/src/flowsint_core/core/graph_repository.py@@ -5,11 +5,11 @@ This module provides a repository pattern implementation for Neo4j,handling node and relationship operations with batching support."""-from typing import Dict, Any, List, Optional, Tuple+from typing import Dict, Any, List, Optional, Tuple, Unionfrom datetime import datetime, timezone+from pydantic import BaseModelfrom .graph_db import Neo4jConnectionfrom .graph_serializer import GraphSerializer-from flowsint_types.registry import get_typeclass GraphRepository:@@ -33,142 +33,105 @@ class GraphRepository:self._batch_size = 100@staticmethod- def _get_primary_field_for_type(type_name: str) -> Optional[str]:- """- Get the primary field name for a given type.+ def _get_primary_field(obj: BaseModel) -> str:+ """Get the primary field of a Pydantic object."""+ model_fields = obj.__class__.model_fields- Args:- type_name: The type name (e.g., "Ip", "Domain", or lowercase variants)+ primary_field = None+ for field_name, field_info in model_fields.items():+ if field_info.json_schema_extra and field_info.json_schema_extra.get(+ "primary"+ ):+ primary_field = field_name+ break++ if primary_field is None:+ for field_name, field_info in model_fields.items():+ if field_info.is_required():+ primary_field = field_name+ break+ if primary_field is None:+ primary_field = next(iter(model_fields.keys()))++ return primary_field- Returns:- The primary field name, or None if type not found+ @staticmethod+ def _extract_primary_value(obj: BaseModel) -> Any:"""- # Get the type class from registry- type_class = get_type(type_name, case_sensitive=False)- if not type_class:- return None-- # Find the primary field (marked with json_schema_extra={"primary": True})- model_fields = type_class.model_fields- for field_name, field_info in model_fields.items():- if field_info.json_schema_extra and field_info.json_schema_extra.get("primary"):- return field_name+ Extract the primitive value from a Pydantic object recursively.+ Uses model_dump to properly serialize Pydantic types like HttpUrl.+ """+ primary_field = GraphRepository._get_primary_field(obj)+ obj_dict = (+ obj.model_dump(mode="json") if hasattr(obj, "model_dump") else obj.dict()+ )+ value = obj_dict.get(primary_field)- # Fallback: use first required field or first field- for field_name, field_info in model_fields.items():- if field_info.is_required():- return field_name+ if isinstance(value, dict):+ nested_obj = getattr(obj, primary_field)+ if isinstance(nested_obj, BaseModel):+ return GraphRepository._extract_primary_value(nested_obj)- # Last resort: first field- return next(iter(model_fields.keys())) if model_fields else None+ return valuedef create_node(self,- node_type: str,- key_prop: str,- key_value: str,- sketch_id: str,- **properties: Any- ) -> None:+ node_obj: BaseModel,+ sketch_id: Optional[str] = None,+ **properties: Any,+ ) -> Optional[str]:"""Create or update a single node in Neo4j.- Args:- node_type: Node label (e.g., "domain", "ip")- key_prop: Property name used as unique identifier- key_value: Value of the key property- sketch_id: Investigation sketch ID- **properties: Additional node properties- """- if not self._connection:- return+ Supports 1 signature:+ 1. Pydantic object: create_node(obj, sketch_id="...", **overrides)- # Serialize properties- serialized_props = GraphSerializer.serialize_properties(properties)-- # Add required properties- serialized_props["type"] = node_type.lower()- serialized_props["sketch_id"] = sketch_id- serialized_props["label"] = serialized_props.get("label", key_value)-- # Build SET clauses (exclude sketch_id as it's in MERGE)- set_clauses = [f"n.{prop} = ${prop}" for prop in serialized_props.keys() if prop != "sketch_id"]- params = {- key_prop: key_value,- "created_at": datetime.now(timezone.utc).isoformat(),- **serialized_props- }+ Args:+ node_obj: a Pydantic object+ sketch_id: Investigation sketch ID (required)+ **properties: Additional node properties or overrides- # Build and execute query - MERGE on both key_prop AND sketch_id for uniqueness- # Use ON CREATE SET to only set created_at when creating, not updating- query = f"""- MERGE (n:{node_type} {{{key_prop}: ${key_prop}, sketch_id: $sketch_id}})- ON CREATE SET n.created_at = $created_at- SET {', '.join(set_clauses)}+ Returns:+ Element ID of created/updated node"""+ if not self._connection:+ return None- self._connection.execute_write(query, params)+ query, params = self._build_node_query(node_obj, sketch_id, **properties)+ result = self._connection.query(query, params)+ return result[0]["id"] if result else Nonedef create_relationship(self,- from_type: str,- from_key: str,- from_value: str,- to_type: str,- to_key: str,- to_value: str,- rel_type: str,- sketch_id: str,- **properties: Any+ from_obj: BaseModel,+ to_obj: BaseModel,+ rel_label: Optional[str] = None,+ sketch_id: Optional[str] = None,+ **properties: Any,) -> None:"""Create a relationship between two nodes.+ Supports one signature:+ - Pydantic objects: create_relationship(obj1, obj2, "REL_TYPE", sketch_id="...")+Args:- from_type: Source node label- from_key: Source node key property- from_value: Source node key value- to_type: Target node label- to_key: Target node key property- to_value: Target node key value- rel_type: Relationship type- sketch_id: Investigation sketch ID+ from_obj: a Pydantic object (source)+ to_obj: a Pydantic object (target)+ rel_label: Relationship label (ex: "IS_CONNECTED_TO")+ sketch_id: Investigation sketch ID (required)**properties: Additional relationship properties"""if not self._connection:return- # Serialize relationship properties- serialized_props = GraphSerializer.serialize_properties(properties)- serialized_props["sketch_id"] = sketch_id-- # Build relationship properties string- if serialized_props:- props_str = ", ".join([f"{k}: ${k}" for k in serialized_props.keys()])- rel_props = f"{{{props_str}}}"- else:- rel_props = "{sketch_id: $sketch_id}"-- # MATCH nodes by both key and sketch_id to ensure we're connecting nodes from the same sketch- query = f"""- MATCH (from:{from_type} {{{from_key}: $from_value, sketch_id: $sketch_id}})- MATCH (to:{to_type} {{{to_key}: $to_value, sketch_id: $sketch_id}})- MERGE (from)-[:{rel_type} {rel_props}]->(to)- """-- params = {- "from_value": from_value,- "to_value": to_value,- **serialized_props- }+ query, params = self._build_relationship_query(+ from_obj, to_obj, rel_label, sketch_id, **properties+ )self._connection.execute_write(query, params)- def add_to_batch(- self,- operation_type: str,- **kwargs: Any- ) -> None:+ def add_to_batch(self, operation_type: str, **kwargs: Any) -> None:"""Add an operation to the batch queue.@@ -191,24 +154,64 @@ class GraphRepository:def _build_node_query(self,- node_type: str,- key_prop: str,- key_value: str,+ node_obj: BaseModel,sketch_id: str,- **properties: Any+ **properties: Any,) -> Tuple[str, Dict[str, Any]]:"""Build a node creation query."""++ node_type = node_obj.__class__.__name__.lower()+ primary_field = self._get_primary_field(node_obj)+ key_prop = primary_field+ key_value = getattr(node_obj, primary_field, None)++ # If primary key value is None or empty, fallback to label+ if key_value is None or (isinstance(key_value, str) and not key_value):+ key_prop = "label"+ key_value = properties.get("label") or getattr(node_obj, "label", "Node")++ if isinstance(key_value, BaseModel):+ key_value = self._extract_primary_value(key_value)++ obj_dict = (+ node_obj.model_dump(mode="json")+ if hasattr(node_obj, "model_dump")+ else node_obj.dict()+ )+ # Extract only non-dict values, skip None keys+ obj_properties = {+ k: (v if v is not None else "")+ for k, v in obj_dict.items()+ if k is not None and not isinstance(v, dict)+ }+ obj_properties.update(properties)+ properties = obj_properties++ if not sketch_id:+ raise ValueError("sketch_id is required")+ if not key_prop or key_prop is None:+ raise ValueError(f"key_prop cannot be None for node type {node_type}")+serialized_props = GraphSerializer.serialize_properties(properties)serialized_props["type"] = node_type.lower()serialized_props["sketch_id"] = sketch_id- serialized_props["label"] = serialized_props.get("label", key_value)+ label = serialized_props.get("label", key_value)+ serialized_props["label"] = label++ # Clean None keys from serialized_props+ serialized_props = {k: v for k, v in serialized_props.items() if k is not None}++ set_clauses = [+ f"n.{prop} = ${prop}"+ for prop in serialized_props.keys()+ if prop != "sketch_id"+ ]- # Build SET clauses (exclude sketch_id as it's in MERGE)- set_clauses = [f"n.{prop} = ${prop}" for prop in serialized_props.keys() if prop != "sketch_id"]+ # Build params, ensuring no None keysparams = {key_prop: key_value,"created_at": datetime.now(timezone.utc).isoformat(),- **serialized_props+ **serialized_props,}# MERGE on both key_prop AND sketch_id for uniqueness per sketch@@ -217,44 +220,87 @@ class GraphRepository:MERGE (n:{node_type} {{{key_prop}: ${key_prop}, sketch_id: $sketch_id}})ON CREATE SET n.created_at = $created_atSET {', '.join(set_clauses)}+ RETURN elementId(n) as id"""return query, paramsdef _build_relationship_query(self,- from_type: str,- from_key: str,- from_value: str,- to_type: str,- to_key: str,- to_value: str,- rel_type: str,- sketch_id: str,- **properties: Any+ from_obj: BaseModel,+ to_obj: BaseModel,+ rel_label: Optional[str] = None,+ sketch_id: Optional[str] = None,+ **properties: Any,) -> Tuple[str, Dict[str, Any]]:"""Build a relationship creation query."""+ # From object+ from_node_type = from_obj.__class__.__name__.lower()+ from_primary_field = self._get_primary_field(from_obj)+ from_obj_dict = (+ from_obj.model_dump(mode="json")+ if hasattr(from_obj, "model_dump")+ else from_obj.dict()+ )+ from_key_value = from_obj_dict.get(from_primary_field)++ if isinstance(from_key_value, dict):+ nested_obj = getattr(from_obj, from_primary_field)+ if isinstance(nested_obj, BaseModel):+ from_key_value = self._extract_primary_value(nested_obj)+ # To object+ to_node_type = to_obj.__class__.__name__.lower()+ to_primary_field = self._get_primary_field(to_obj)+ to_obj_dict = (+ to_obj.model_dump(mode="json")+ if hasattr(to_obj, "model_dump")+ else to_obj.dict()+ )+ to_key_value = to_obj_dict.get(to_primary_field)++ if isinstance(to_key_value, dict):+ nested_obj = getattr(to_obj, to_primary_field)+ if isinstance(nested_obj, BaseModel):+ to_key_value = self._extract_primary_value(nested_obj)++ from_type = from_node_type+ from_key = from_primary_field+ from_value = from_key_value+ to_type = to_node_type+ to_key = to_primary_field+ to_value = to_key_value+ rel_type = rel_label++ if not sketch_id:+ raise ValueError("sketch_id is required")+ if not from_key or from_key is None:+ raise ValueError(+ f"from_key cannot be None for relationship type {rel_type}"+ )+ if not to_key or to_key is None:+ raise ValueError(f"to_key cannot be None for relationship type {rel_type}")+serialized_props = GraphSerializer.serialize_properties(properties)serialized_props["sketch_id"] = sketch_id+ # Clean None keys+ serialized_props = {k: v for k, v in serialized_props.items() if k is not None}+if serialized_props:props_str = ", ".join([f"{k}: ${k}" for k in serialized_props.keys()])rel_props = f"{{{props_str}}}"else:rel_props = "{sketch_id: $sketch_id}"- # MATCH nodes by both key and sketch_id to ensure we're connecting nodes from the same sketchquery = f"""MATCH (from:{from_type} {{{from_key}: $from_value, sketch_id: $sketch_id}})MATCH (to:{to_type} {{{to_key}: $to_value, sketch_id: $sketch_id}})MERGE (from)-[:{rel_type} {rel_props}]->(to)"""- params = {- "from_value": from_value,- "to_value": to_value,- **serialized_props- }+ params = {"from_value": from_value, "to_value": to_value, **serialized_props}+ # Clean None keys from params+ params = {k: v for k, v in params.items() if k is not None}return query, params@@ -287,121 +333,122 @@ class GraphRepository:raise ValueError("Batch size must be at least 1")self._batch_size = size- def create_node_from_import(+ def batch_create_nodes(self,- node_type: str,- label: str,+ nodes: List[BaseModel],sketch_id: str,- **properties: Any- ) -> Optional[str]:+ ) -> Dict[str, Any]:"""- Create a node from import operation using the type's primary key.-- This method uses the same MERGE strategy as enrichers: it identifies nodes- by their primary key field (e.g., "address" for IP, "domain" for Domain)- combined with sketch_id, preventing duplicates between imports and enrichers.+ Create multiple nodes in a single batch transaction.Args:- node_type: Node label (e.g., "Domain", "Ip") - will be normalized to lowercase- label: Human-readable label for the node+ nodes: List of Pydantic model instances to createsketch_id: Investigation sketch ID- **properties: All additional node propertiesReturns:- Element ID of the created/found node or None+ Dictionary with:+ - nodes_created: Number of successfully created nodes+ - node_ids: List of created node element IDs+ - errors: List of error messages for failed nodes"""if not self._connection:- return None+ return {"nodes_created": 0, "node_ids": [], "errors": ["No database connection"]}- # Normalize node_type to lowercase (same as enrichers)- node_type = node_type.lower()+ if not nodes:+ return {"nodes_created": 0, "node_ids": [], "errors": []}- # Get the primary key field for this type- key_prop = self._get_primary_field_for_type(node_type)- if not key_prop:- # Fallback: if type not found in registry, use "label" as key- key_prop = "label"- key_value = label- else:- # Extract the primary key value from properties- key_value = properties.get(key_prop)- if not key_value:- # Fallback: if primary key not in properties, use label- key_value = label+ # Build all queries+ batch_operations = []+ errors = []- # Serialize and prepare all properties- serialized_props = GraphSerializer.serialize_properties(properties)- serialized_props["type"] = node_type.lower()- serialized_props["sketch_id"] = sketch_id- serialized_props["label"] = label- serialized_props["caption"] = label+ for idx, node_obj in enumerate(nodes):+ try:+ query, params = self._build_node_query(+ node_obj=node_obj,+ sketch_id=sketch_id,+ )+ batch_operations.append((query, params))+ except Exception as e:+ errors.append(f"Node {idx}: {str(e)}")- # Build SET clauses (exclude sketch_id as it's in MERGE, but include key_prop)- set_clauses = [f"n.{prop} = ${prop}" for prop in serialized_props.keys()- if prop != "sketch_id"]+ # Execute batch+ if not batch_operations:+ return {"nodes_created": 0, "node_ids": [], "errors": errors}- params = {- key_prop: key_value,- "created_at": datetime.now(timezone.utc).isoformat(),- **serialized_props- }+ try:+ # Execute all operations in a single transaction+ results = self._connection.execute_batch(batch_operations)++ # Extract node IDs from results+ node_ids = []+ if results:+ for result in results:+ if result and len(result) > 0 and "id" in result[0]:+ node_ids.append(result[0]["id"])++ return {+ "nodes_created": len(node_ids),+ "node_ids": node_ids,+ "errors": errors,+ }+ except Exception as e:+ errors.append(f"Batch execution failed: {str(e)}")+ return {"nodes_created": 0, "node_ids": [], "errors": errors}- # MERGE on primary key + sketch_id (same as enrichers)- query = f"""- MERGE (n:{node_type} {{{key_prop}: ${key_prop}, sketch_id: $sketch_id}})- ON CREATE SET n.created_at = $created_at- SET {', '.join(set_clauses)}- RETURN elementId(n) as id- """-- result = self._connection.query(query, params)- return result[0]["id"] if result else Nonedef update_node(self,- node_type: str,- key_prop: str,- key_value: str,+ element_id: str,+ node_obj: BaseModel,sketch_id: str,- **properties: Any- ) -> Optional[Dict[str, Any]]:+ ) -> Optional[str]:"""- Update an existing node's properties.+ Update an existing node's properties using a Pydantic model.Args:- node_type: Node label (e.g., "domain", "ip")- key_prop: Property name used as unique identifier- key_value: Value of the key property- sketch_id: Investigation sketch ID- **properties: Properties to update+ element_id: Neo4j element ID of the node to update+ node_obj: Pydantic model instance with updated data+ sketch_id: Investigation sketch ID (for safety)Returns:- Updated node properties or None if not found+ Element ID of the updated node or None if not found"""if not self._connection:return None+ # Extract properties from Pydantic object+ obj_dict = (+ node_obj.model_dump(mode="json")+ if hasattr(node_obj, "model_dump")+ else node_obj.dict()+ )++ # Extract only non-dict values, skip None keys+ properties = {+ k: (v if v is not None else "")+ for k, v in obj_dict.items()+ if k is not None and not isinstance(v, dict)+ }+# Serialize propertiesserialized_props = GraphSerializer.serialize_properties(properties)+ serialized_props["type"] = node_obj.__class__.__name__.lower()# Build SET clausesset_clauses = [f"n.{prop} = ${prop}" for prop in serialized_props.keys()]- params = {key_prop: key_value, "sketch_id": sketch_id, **serialized_props}query = f"""- MATCH (n:{node_type} {{{key_prop}: ${key_prop}, sketch_id: $sketch_id}})+ MATCH (n)+ WHERE elementId(n) = $element_id AND n.sketch_id = $sketch_idSET {', '.join(set_clauses)}- RETURN properties(n) as node+ RETURN elementId(n) as id"""+ params = {"element_id": element_id, "sketch_id": sketch_id, **serialized_props}result = self._connection.query(query, params)- return result[0]["node"] if result else None+ return result[0]["id"] if result else None- def delete_nodes(- self,- node_ids: List[str],- sketch_id: str- ) -> int:+ def delete_nodes(self, node_ids: List[str], sketch_id: str) -> int:"""Delete nodes by their element IDs.@@ -424,16 +471,11 @@ class GraphRepository:"""result = self._connection.query(- query,- {"node_ids": node_ids, "sketch_id": sketch_id}+ query, {"node_ids": node_ids, "sketch_id": sketch_id})return result[0]["deleted_count"] if result else 0- def delete_relationships(- self,- relationship_ids: List[str],- sketch_id: str- ) -> int:+ def delete_relationships(self, relationship_ids: List[str], sketch_id: str) -> int:"""Delete relationships by their element IDs.@@ -456,8 +498,7 @@ class GraphRepository:"""result = self._connection.query(- query,- {"relationship_ids": relationship_ids, "sketch_id": sketch_id}+ query, {"relationship_ids": relationship_ids, "sketch_id": sketch_id})return result[0]["deleted_count"] if result else 0@@ -484,9 +525,7 @@ class GraphRepository:return result[0]["deleted_count"] if result else 0def get_sketch_graph(- self,- sketch_id: str,- limit: int = 100000+ self, sketch_id: str, limit: int = 100000) -> Dict[str, List[Dict[str, Any]]]:"""Get all nodes and relationships for a sketch.@@ -509,8 +548,7 @@ class GraphRepository:LIMIT $limit"""nodes_result = self._connection.query(- nodes_query,- {"sketch_id": sketch_id, "limit": limit}+ nodes_query, {"sketch_id": sketch_id, "limit": limit})if not nodes_result:@@ -526,15 +564,9 @@ class GraphRepository:RETURN elementId(r) as id, type(r) as type, elementId(a) as source,elementId(b) as target, properties(r) as data"""- rels_result = self._connection.query(- rels_query,- {"node_ids": node_ids}- )+ rels_result = self._connection.query(rels_query, {"node_ids": node_ids})- return {- "nodes": nodes_result,- "relationships": rels_result or []- }+ return {"nodes": nodes_result, "relationships": rels_result or []}def create_relationship_by_element_id(self,@@ -542,7 +574,7 @@ class GraphRepository:to_element_id: str,rel_type: str,sketch_id: str,- **properties: Any+ **properties: Any,) -> Optional[Dict[str, Any]]:"""Create a relationship between two nodes using their element IDs.@@ -576,47 +608,15 @@ class GraphRepository:params = {"from_id": from_element_id,"to_id": to_element_id,- **serialized_props+ **serialized_props,}result = self._connection.query(query, params)return result[0]["rel"] if result else None- def update_node_by_element_id(- self,- element_id: str,- sketch_id: str,- **properties: Any- ) -> Optional[Dict[str, Any]]:- """- Update a node by its element ID.-- Args:- element_id: Neo4j element ID- sketch_id: Investigation sketch ID (for safety)- **properties: Properties to update-- Returns:- Updated node properties or None if not found- """- if not self._connection:- return None-- serialized_props = GraphSerializer.serialize_properties(properties)- set_clauses = [f"n.{prop} = ${prop}" for prop in serialized_props.keys()]-- query = f"""- MATCH (n)- WHERE elementId(n) = $element_id AND n.sketch_id = $sketch_id- SET {', '.join(set_clauses)}- RETURN properties(n) as node- """-- params = {"element_id": element_id, "sketch_id": sketch_id, **serialized_props}- result = self._connection.query(query, params)- return result[0]["node"] if result else None-- def query(self, cypher: str, parameters: Dict[str, Any] = None) -> List[Dict[str, Any]]:+ def query(+ self, cypher: str, parameters: Dict[str, Any] = None+ ) -> List[Dict[str, Any]]:"""Execute a custom Cypher query.@@ -633,9 +633,7 @@ class GraphRepository:return self._connection.query(cypher, parameters)def update_nodes_positions(- self,- positions: List[Dict[str, Any]],- sketch_id: str+ self, positions: List[Dict[str, Any]], sketch_id: str) -> int:"""Update positions (x, y) for multiple nodes in batch.@@ -658,18 +656,13 @@ class GraphRepository:RETURN count(n) as updated_count"""- params = {- "positions": positions,- "sketch_id": sketch_id- }+ params = {"positions": positions, "sketch_id": sketch_id}result = self._connection.query(query, params)return result[0]["updated_count"] if result else 0def get_nodes_by_ids(- self,- node_ids: List[str],- sketch_id: str+ self, node_ids: List[str], sketch_id: str) -> List[Dict[str, Any]]:"""Get nodes by their element IDs.@@ -692,12 +685,237 @@ class GraphRepository:"""result = self._connection.query(- query,- {"node_ids": node_ids, "sketch_id": sketch_id}+ query, {"node_ids": node_ids, "sketch_id": sketch_id})return [record["data"] for record in result] if result else []+ def merge_nodes(+ self,+ old_node_ids: List[str],+ new_node_data: Dict[str, Any],+ new_node_id: Optional[str],+ sketch_id: str,+ ) -> Optional[str]:+ """+ Merge multiple nodes into one, transferring all relationships.++ Args:+ old_node_ids: List of element IDs of nodes to merge+ new_node_data: Properties for the merged node+ new_node_id: Optional element ID if reusing an existing node+ sketch_id: Investigation sketch ID++ Returns:+ Element ID of the merged node+ """+ if not self._connection or not old_node_ids:+ return None++ node_type = new_node_data.get("type", "Node")+ properties = GraphSerializer.serialize_properties(new_node_data)+ properties["sketch_id"] = sketch_id++ is_reusing_node = new_node_id and new_node_id in old_node_ids++ if is_reusing_node:+ set_clause = ", ".join(f"n.{key} = ${key}" for key in properties.keys())+ create_query = f"""+ MATCH (n)+ WHERE elementId(n) = $nodeId AND n.sketch_id = $sketch_id+ SET {set_clause}+ RETURN elementId(n) as newElementId+ """+ params = {"nodeId": new_node_id, "sketch_id": sketch_id, **properties}+ else:+ properties["created_at"] = datetime.now(timezone.utc).isoformat()+ create_query = f"""+ CREATE (n:`{node_type}`)+ SET n = $properties+ RETURN elementId(n) as newElementId+ """+ params = {"properties": properties}++ result = self._connection.query(create_query, params)+ if not result:+ return None++ new_node_element_id = result[0]["newElementId"]++ copy_relationships_query = """+ MATCH (new) WHERE elementId(new) = $newElementId++ UNWIND $oldNodeIds AS oldNodeId+ MATCH (old) WHERE elementId(old) = oldNodeId AND old.sketch_id = $sketch_id++ WITH new, collect(old) as oldNodes+ UNWIND oldNodes as old+ MATCH (src)-[r]->(old)+ WHERE elementId(src) NOT IN $oldNodeIds AND elementId(src) <> $newElementId+ WITH new, src, type(r) as relType, properties(r) as relProps, r+ MERGE (src)-[newRel:RELATED_TO {sketch_id: $sketch_id}]->(new)+ SET newRel = relProps++ WITH new, $oldNodeIds as oldNodeIds+ UNWIND oldNodeIds AS oldNodeId+ MATCH (old) WHERE elementId(old) = oldNodeId AND old.sketch_id = $sketch_id++ MATCH (old)-[r]->(dst)+ WHERE elementId(dst) NOT IN oldNodeIds AND elementId(dst) <> $newElementId+ WITH new, dst, type(r) as relType, properties(r) as relProps+ MERGE (new)-[newRel:RELATED_TO {sketch_id: $sketch_id}]->(dst)+ SET newRel = relProps+ """++ self._connection.query(+ copy_relationships_query,+ {+ "newElementId": new_node_element_id,+ "oldNodeIds": old_node_ids,+ "sketch_id": sketch_id,+ },+ )++ nodes_to_delete = [nid for nid in old_node_ids if nid != new_node_element_id]+ if nodes_to_delete:+ delete_query = """+ UNWIND $nodeIds AS nodeId+ MATCH (old)+ WHERE elementId(old) = nodeId AND old.sketch_id = $sketch_id+ DETACH DELETE old+ """+ self._connection.query(+ delete_query, {"nodeIds": nodes_to_delete, "sketch_id": sketch_id}+ )++ return new_node_element_id++ def get_related_nodes(self, node_id: str, sketch_id: str) -> Dict[str, Any]:+ """+ Get a node and all its direct relationships and connected nodes.++ Args:+ node_id: Element ID of the center node+ sketch_id: Investigation sketch ID++ Returns:+ Dictionary with 'nds' (nodes) and 'rls' (relationships) lists+ """+ if not self._connection:+ return {"nds": [], "rls": []}++ center_query = """+ MATCH (n)+ WHERE elementId(n) = $node_id AND n.sketch_id = $sketch_id+ RETURN elementId(n) as id, labels(n) as labels, properties(n) as data+ """++ center_result = self._connection.query(+ center_query, {"sketch_id": sketch_id, "node_id": node_id}+ )++ if not center_result:+ return {"nds": [], "rls": []}++ relationships_query = """+ MATCH (n)+ WHERE elementId(n) = $node_id AND n.sketch_id = $sketch_id+ OPTIONAL MATCH (n)-[r]->(other)+ WHERE other.sketch_id = $sketch_id+ OPTIONAL MATCH (other)-[r2]->(n)+ WHERE other.sketch_id = $sketch_id+ RETURN+ elementId(r) as rel_id,+ type(r) as rel_type,+ properties(r) as rel_data,+ elementId(other) as other_node_id,+ labels(other) as other_node_labels,+ properties(other) as other_node_data,+ 'outgoing' as direction+ UNION+ MATCH (n)+ WHERE elementId(n) = $node_id AND n.sketch_id = $sketch_id+ OPTIONAL MATCH (other)-[r]->(n)+ WHERE other.sketch_id = $sketch_id+ RETURN+ elementId(r) as rel_id,+ type(r) as rel_type,+ properties(r) as rel_data,+ elementId(other) as other_node_id,+ labels(other) as other_node_labels,+ properties(other) as other_node_data,+ 'incoming' as direction+ """++ result = self._connection.query(+ relationships_query, {"sketch_id": sketch_id, "node_id": node_id}+ )++ center_record = center_result[0]+ center_node = {+ "id": center_record["id"],+ "labels": center_record["labels"],+ "data": center_record["data"],+ "label": center_record["data"].get("label", "Node"),+ "type": "custom",+ "caption": center_record["data"].get("label", "Node"),+ }++ related_nodes = []+ relationships = []+ seen_nodes = set()+ seen_relationships = set()++ for record in result:+ if not record["rel_id"]:+ continue++ if record["rel_id"] not in seen_relationships:+ if record["direction"] == "outgoing":+ relationships.append(+ {+ "id": record["rel_id"],+ "type": "straight",+ "source": center_node["id"],+ "target": record["other_node_id"],+ "data": record["rel_data"],+ "caption": record["rel_type"],+ }+ )+ else:+ relationships.append(+ {+ "id": record["rel_id"],+ "type": "straight",+ "source": record["other_node_id"],+ "target": center_node["id"],+ "data": record["rel_data"],+ "caption": record["rel_type"],+ }+ )+ seen_relationships.add(record["rel_id"])++ if (+ record["other_node_id"]+ and record["other_node_id"] not in seen_nodes+ and record["other_node_id"] != center_node["id"]+ ):+ related_nodes.append(+ {+ "id": record["other_node_id"],+ "labels": record["other_node_labels"],+ "data": record["other_node_data"],+ "label": record["other_node_data"].get("label", "Node"),+ "type": "custom",+ "caption": record["other_node_data"].get("label", "Node"),+ }+ )+ seen_nodes.add(record["other_node_id"])++ all_nodes = [center_node] + related_nodes++ return {"nds": all_nodes, "rls": relationships}+def __enter__(self):"""Context manager entry."""return selfdiff --git a/flowsint-core/src/flowsint_core/core/graph_serializer.py b/flowsint-core/src/flowsint_core/core/graph_serializer.pyindex 63f658b6..1546286f 100644--- a/flowsint-core/src/flowsint_core/core/graph_serializer.py+++ b/flowsint-core/src/flowsint_core/core/graph_serializer.py@@ -26,7 +26,7 @@ class GraphSerializer:- Nested objects (flattened with prefixed keys)- Lists (converted to primitive types)- Dictionaries (flattened with prefixed keys)- - None values (converted to empty strings)+ - None values (skipped)Args:properties: Dictionary of properties to serialize@@ -37,6 +37,8 @@ class GraphSerializer:serialized = {}for key, value in properties.items():+ if key is None:+ continueif value is None:serialized[key] = ""elif GraphSerializer._is_pydantic_model(value):@@ -84,21 +86,24 @@ class GraphSerializer:# Try Pydantic v2 first, then v1if hasattr(model, "model_dump"):- data = model.model_dump()+ data = model.model_dump(mode="json")elif hasattr(model, "dict"):data = model.dict()else:# Fallback to __dict__- data = model.__dict__+ data = {k: v for k, v in model.__dict__.items() if k is not None}for nested_key, nested_value in data.items():- if nested_value is not None:- new_key = f"{prefix}_{nested_key}"- if isinstance(nested_value, (str, int, float, bool)):- flattened[new_key] = nested_value- else:- # Recursively handle nested complex types- flattened[new_key] = str(nested_value)+ if nested_key is None:+ continue+ new_key = f"{prefix}_{nested_key}"+ if nested_value is None:+ flattened[new_key] = ""+ elif isinstance(nested_value, (str, int, float, bool)):+ flattened[new_key] = nested_value+ else:+ # Recursively handle nested complex types+ flattened[new_key] = str(nested_value)return flattened@@ -117,12 +122,15 @@ class GraphSerializer:flattened = {}for dict_key, dict_value in data.items():- if dict_value is not None:- new_key = f"{prefix}_{dict_key}"- if isinstance(dict_value, (str, int, float, bool)):- flattened[new_key] = dict_value- else:- flattened[new_key] = str(dict_value)+ if dict_key is None:+ continue+ new_key = f"{prefix}_{dict_key}"+ if dict_value is None:+ flattened[new_key] = ""+ elif isinstance(dict_value, (str, int, float, bool)):+ flattened[new_key] = dict_value+ else:+ flattened[new_key] = str(dict_value)return flatteneddiff --git a/flowsint-core/src/flowsint_core/core/graph_service.py b/flowsint-core/src/flowsint_core/core/graph_service.pyindex 3862fd3a..d622fa7c 100644--- a/flowsint-core/src/flowsint_core/core/graph_service.py+++ b/flowsint-core/src/flowsint_core/core/graph_service.py@@ -5,8 +5,9 @@ This module provides a service layer for graph operations,integrating repository and logging functionality."""-from typing import Dict, Any, Optional, Protocol+from typing import Dict, Any, Optional, Protocol, Unionfrom uuid import UUID+from pydantic import BaseModelfrom .graph_repository import GraphRepositoryfrom .graph_db import Neo4jConnection@@ -33,7 +34,7 @@ class GraphService:sketch_id: str,neo4j_connection: Optional[Neo4jConnection] = None,logger: Optional[LoggerProtocol] = None,- enable_batching: bool = True+ enable_batching: bool = True,):"""Initialize the graph service.@@ -59,94 +60,66 @@ class GraphService:"""Get the underlying repository."""return self._repository- def create_node(- self,- node_type: str,- key_prop: str,- key_value: str,- **properties: Any- ) -> None:+ def create_node(self, node_obj: BaseModel, **properties: Any) -> None:"""Create or update a node in the graph.- Automatically adds the following properties:- - type: Lowercase version of node_type- - sketch_id: Current sketch ID- - label: Defaults to key_value if not provided- - created_at: ISO 8601 UTC timestamp (only on creation via ON CREATE SET)+ Supports one signatures:+ - Pydantic object: create_node(obj, **overrides)Args:- node_type: Node label (e.g., "domain", "ip")- key_prop: Property name used as unique identifier- key_value: Value of the key property- **properties: Additional node properties+ from_obj: a Pydantic object+ **properties: Additional node properties or overrides"""if self._enable_batching:self._repository.add_to_batch("node",- node_type=node_type,- key_prop=key_prop,- key_value=key_value,+ node_obj=node_obj,sketch_id=self._sketch_id,- **properties+ **properties,)else:self._repository.create_node(- node_type=node_type,- key_prop=key_prop,- key_value=key_value,+ node_obj=node_obj,sketch_id=self._sketch_id,- **properties+ **properties,)def create_relationship(self,- from_type: str,- from_key: str,- from_value: str,- to_type: str,- to_key: str,- to_value: str,- rel_type: str,- **properties: Any+ from_obj: BaseModel,+ to_obj: BaseModel,+ rel_label: Optional[str] = None,+ **properties: Any,) -> None:"""Create a relationship between two nodes.+ Supports 1 signature:+ - Pydantic objects: create_relationship(obj1, obj2, "REL_TYPE")+Args:- from_type: Source node label- from_key: Source node key property- from_value: Source node key value- to_type: Target node label- to_key: Target node key property- to_value: Target node key value- rel_type: Relationship type+ from_obj: Either a Pydantic object (source)+ to_obj: Either a Pydantic object (target)+ rel_label: Relationship label (ex: "IS_CONNECTED_TO")**properties: Additional relationship properties"""if self._enable_batching:self._repository.add_to_batch("relationship",- from_type=from_type,- from_key=from_key,- from_value=from_value,- to_type=to_type,- to_key=to_key,- to_value=to_value,- rel_type=rel_type,+ from_obj=from_obj,+ to_obj=to_obj,+ rel_label=rel_label,sketch_id=self._sketch_id,- **properties+ **properties,)else:self._repository.create_relationship(- from_type=from_type,- from_key=from_key,- from_value=from_value,- to_type=to_type,- to_key=to_key,- to_value=to_value,- rel_type=rel_type,+ from_obj=from_obj,+ to_obj=to_obj,+ rel_label=rel_label,sketch_id=self._sketch_id,- **properties+ **properties,)def log_graph_message(self, message: str) -> None:@@ -199,7 +172,7 @@ class GraphService:def create_graph_service(sketch_id: str,neo4j_connection: Optional[Neo4jConnection] = None,- enable_batching: bool = True+ enable_batching: bool = True,) -> GraphService:"""Factory function to create a GraphService instance.@@ -219,5 +192,5 @@ def create_graph_service(sketch_id=sketch_id,neo4j_connection=neo4j_connection,logger=Logger,- enable_batching=enable_batching+ enable_batching=enable_batching,)diff --git a/flowsint-core/tests/enrichers/base.py b/flowsint-core/tests/core/base.pysimilarity index 100%rename from flowsint-core/tests/enrichers/base.pyrename to flowsint-core/tests/core/base.pydiff --git a/flowsint-core/tests/core/test_enricher_base_simplified_api.py b/flowsint-core/tests/core/test_enricher_base_simplified_api.pynew file mode 100644index 00000000..9b741eca--- /dev/null+++ b/flowsint-core/tests/core/test_enricher_base_simplified_api.py@@ -0,0 +1,52 @@+"""Test simplified API for create_node and create_relationship."""++import pytest+from flowsint_core.core.enricher_base import Enricher+from flowsint_types.domain import Domain+from flowsint_types.email import Email+from flowsint_types.individual import Individual+from typing import List+++class MockEnricher(Enricher):+ """Simple enricher for testing."""++ InputType = Domain+ OutputType = Domain++ @classmethod+ def name(cls) -> str:+ return "test_enricher"++ @classmethod+ def category(cls) -> str:+ return "Test"++ @classmethod+ def key(cls) -> str:+ return "domain"++ async def scan(self, data: List[InputType]) -> List[OutputType]:+ return data+++def test_create_relationship_with_pydantic_objects():+ """Test that create_relationship works with Pydantic objects."""+ enricher = MockEnricher(sketch_id="test", scan_id="test")++ # Create objects+ individual = Individual(first_name="John", last_name="Doe", full_name="John Doe")+ domain = Domain(domain="example.com")++ # This should not raise an error+ enricher.create_relationship(individual, domain, "HAS_DOMAIN")+++def test_create_node_with_property_override():+ """Test that property overrides work with Pydantic objects."""+ enricher = MockEnricher(sketch_id="test", scan_id="test")++ domain = Domain(domain="example.com")++ # Should be able to override properties+ enricher.create_node(domain, type="subdomain")diff --git a/flowsint-core/tests/core/test_graph_repository.py b/flowsint-core/tests/core/test_graph_repository.pynew file mode 100644index 00000000..a603071b--- /dev/null+++ b/flowsint-core/tests/core/test_graph_repository.py@@ -0,0 +1,260 @@+"""Test GraphRepository batch operations."""++import pytest+from unittest.mock import Mock, MagicMock+from flowsint_core.core.graph_repository import GraphRepository+from flowsint_types.domain import Domain+from flowsint_types.ip import Ip+from flowsint_types.email import Email+++def mock_connection():+ """Create a mock Neo4j connection."""+ conn = Mock()+ conn.execute_batch = Mock(return_value=[])+ conn.query = Mock(return_value=[])+ return conn+++def graph_repo(mock_connection):+ """Create a GraphRepository with mocked connection."""+ return GraphRepository(neo4j_connection=mock_connection)+++def test_batch_create_nodes_empty_list(graph_repo):+ """Test batch_create_nodes with empty list."""+ result = graph_repo.batch_create_nodes(nodes=[], sketch_id="test-sketch")++ assert result["nodes_created"] == 0+ assert result["node_ids"] == []+ assert result["errors"] == []+++def test_batch_create_nodes_single_node(graph_repo, mock_connection):+ """Test batch_create_nodes with a single node."""+ domain = Domain(domain="example.com")++ # Mock the batch execution to return a node ID+ mock_connection.execute_batch.return_value = [+ [{"id": "element-id-123"}]+ ]++ result = graph_repo.batch_create_nodes(+ nodes=[domain],+ sketch_id="test-sketch"+ )++ assert result["nodes_created"] == 1+ assert len(result["node_ids"]) == 1+ assert result["node_ids"][0] == "element-id-123"+ assert result["errors"] == []++ # Verify execute_batch was called once+ assert mock_connection.execute_batch.call_count == 1+++def test_batch_create_nodes_multiple_nodes(graph_repo, mock_connection):+ """Test batch_create_nodes with multiple nodes of different types."""+ domain = Domain(domain="example.com")+ ip = Ip(address="192.168.1.1")+ email = Email(email="[email protected]")++ # Mock the batch execution to return multiple node IDs+ mock_connection.execute_batch.return_value = [+ [{"id": "element-id-1"}],+ [{"id": "element-id-2"}],+ [{"id": "element-id-3"}]+ ]++ result = graph_repo.batch_create_nodes(+ nodes=[domain, ip, email],+ sketch_id="test-sketch"+ )++ assert result["nodes_created"] == 3+ assert len(result["node_ids"]) == 3+ assert result["errors"] == []++ # Verify execute_batch was called once with 3 operations+ assert mock_connection.execute_batch.call_count == 1+ batch_operations = mock_connection.execute_batch.call_args[0][0]+ assert len(batch_operations) == 3+++def test_batch_create_nodes_with_validation_errors(graph_repo, mock_connection):+ """Test batch_create_nodes when some nodes have validation errors."""+ valid_domain = Domain(domain="example.com")++ # Mock execute_batch to succeed for valid nodes+ mock_connection.execute_batch.return_value = [+ [{"id": "element-id-1"}]+ ]++ result = graph_repo.batch_create_nodes(+ nodes=[valid_domain],+ sketch_id="test-sketch"+ )++ assert result["nodes_created"] == 1+ assert len(result["errors"]) == 0+++def test_batch_create_nodes_batch_execution_failure(graph_repo, mock_connection):+ """Test batch_create_nodes when the batch execution fails."""+ domain = Domain(domain="example.com")++ # Mock execute_batch to raise an exception+ mock_connection.execute_batch.side_effect = Exception("Database connection error")++ result = graph_repo.batch_create_nodes(+ nodes=[domain],+ sketch_id="test-sketch"+ )++ assert result["nodes_created"] == 0+ assert result["node_ids"] == []+ assert len(result["errors"]) > 0+ assert "Database connection error" in result["errors"][0]+++def test_batch_create_nodes_no_connection():+ """Test batch_create_nodes when there's no database connection."""+ # Create a repository with no connection by manually setting it to None+ repo = GraphRepository(neo4j_connection=Mock())+ repo._connection = None # Force no connection+ domain = Domain(domain="example.com")++ result = repo.batch_create_nodes(+ nodes=[domain],+ sketch_id="test-sketch"+ )++ assert result["nodes_created"] == 0+ assert result["node_ids"] == []+ assert len(result["errors"]) == 1+ assert "No database connection" in result["errors"][0]+++def test_batch_create_nodes_with_label_fallback(graph_repo, mock_connection):+ """Test batch_create_nodes with nodes that need label fallback."""+ from pydantic import BaseModel++ class CustomNode(BaseModel):+ """Node type without a clear primary field."""+ name: str = ""+ label: str = "Custom Node"++ node = CustomNode(name="", label="Fallback Label")++ # Mock the batch execution+ mock_connection.execute_batch.return_value = [+ [{"id": "element-id-fallback"}]+ ]++ result = graph_repo.batch_create_nodes(+ nodes=[node],+ sketch_id="test-sketch"+ )++ # Should succeed with fallback to label+ assert result["nodes_created"] == 1+ assert len(result["node_ids"]) == 1+++def test_batch_create_nodes_large_batch(graph_repo, mock_connection):+ """Test batch_create_nodes with a large number of nodes."""+ # Create 1000 domains+ domains = [Domain(domain=f"example{i}.com") for i in range(1000)]++ # Mock the batch execution to return 1000 node IDs+ mock_connection.execute_batch.return_value = [+ [{"id": f"element-id-{i}"}] for i in range(1000)+ ]++ result = graph_repo.batch_create_nodes(+ nodes=domains,+ sketch_id="test-sketch"+ )++ assert result["nodes_created"] == 1000+ assert len(result["node_ids"]) == 1000+ assert result["errors"] == []++ # Verify execute_batch was called once (single transaction)+ assert mock_connection.execute_batch.call_count == 1+ batch_operations = mock_connection.execute_batch.call_args[0][0]+ assert len(batch_operations) == 1000+++# Tests for update_node+++def test_update_node_success(graph_repo, mock_connection):+ """Test update_node with a valid Pydantic object."""+ domain = Domain(domain="updated-example.com", label="Updated Domain")++ # Mock the query to return the element ID+ mock_connection.query.return_value = [{"id": "element-id-123"}]++ result = graph_repo.update_node(+ element_id="element-id-123",+ node_obj=domain,+ sketch_id="test-sketch"+ )++ assert result == "element-id-123"+ assert mock_connection.query.call_count == 1+++def test_update_node_not_found(graph_repo, mock_connection):+ """Test update_node when node doesn't exist."""+ domain = Domain(domain="example.com")++ # Mock the query to return empty result+ mock_connection.query.return_value = []++ result = graph_repo.update_node(+ element_id="non-existent-id",+ node_obj=domain,+ sketch_id="test-sketch"+ )++ assert result is None+++def test_update_node_no_connection():+ """Test update_node when there's no database connection."""+ repo = GraphRepository(neo4j_connection=Mock())+ repo._connection = None+ domain = Domain(domain="example.com")++ result = repo.update_node(+ element_id="element-id-123",+ node_obj=domain,+ sketch_id="test-sketch"+ )++ assert result is None+++def test_update_node_different_types(graph_repo, mock_connection):+ """Test update_node with different Pydantic types."""+ ip = Ip(address="10.0.0.1", label="Updated IP")++ # Mock the query+ mock_connection.query.return_value = [{"id": "element-id-456"}]++ result = graph_repo.update_node(+ element_id="element-id-456",+ node_obj=ip,+ sketch_id="test-sketch"+ )++ assert result == "element-id-456"++ # Verify the query contains the correct type+ query_call = mock_connection.query.call_args+ params = query_call[0][1]+ assert params["type"] == "ip"diff --git a/flowsint-core/tests/test_logger_singleton.py b/flowsint-core/tests/core/test_logger_singleton.pysimilarity index 98%rename from flowsint-core/tests/test_logger_singleton.pyrename to flowsint-core/tests/core/test_logger_singleton.pyindex 29d7d92d..31ff59b0 100644--- a/flowsint-core/tests/test_logger_singleton.py+++ b/flowsint-core/tests/core/test_logger_singleton.py@@ -35,9 +35,11 @@ def mock_db_session():@pytest.fixturedef mock_get_db(mock_db_session):"""Mock the get_db generator."""- with patch('flowsint_core.core.logger.get_db') as mock:- mock.return_value = iter([mock_db_session])- yield mock+ def mock_generator():+ yield mock_db_session++ with patch('flowsint_core.core.logger.get_db', mock_generator):+ yield mock_generator@pytest.fixturediff --git a/flowsint-core/tests/enrichers/test_enricher_base_simplified_api.py b/flowsint-core/tests/enrichers/test_enricher_base_simplified_api.pydeleted file mode 100644index 697d1485..00000000--- a/flowsint-core/tests/enrichers/test_enricher_base_simplified_api.py+++ /dev/null@@ -1,110 +0,0 @@-"""Test simplified API for create_node and create_relationship."""--import pytest-from flowsint_core.core.enricher_base import Enricher-from flowsint_types.domain import Domain-from flowsint_types.email import Email-from flowsint_types.individual import Individual-from typing import List---class MockEnricher(Enricher):- """Simple enricher for testing."""-- InputType = Domain- OutputType = Domain-- @classmethod- def name(cls) -> str:- return "test_enricher"-- @classmethod- def category(cls) -> str:- return "Test"-- @classmethod- def key(cls) -> str:- return "domain"-- async def scan(self, data: List[InputType]) -> List[OutputType]:- return data---def test_create_node_with_pydantic_object():- """Test that create_node works with Pydantic objects."""- enricher = MockEnricher(sketch_id="test", scan_id="test")-- # Create a domain object- domain = Domain(domain="example.com")-- # This should not raise an error- enricher.create_node(domain)-- # Verify the helper method works- primary_field = enricher._get_primary_field(domain)- assert primary_field == "domain"---def test_create_relationship_with_pydantic_objects():- """Test that create_relationship works with Pydantic objects."""- enricher = MockEnricher(sketch_id="test", scan_id="test")-- # Create objects- individual = Individual(first_name="John", last_name="Doe", full_name="John Doe")- domain = Domain(domain="example.com")-- # This should not raise an error- enricher.create_relationship(individual, domain, "HAS_DOMAIN")---def test_create_node_legacy_signature():- """Test that legacy create_node signature still works."""- enricher = MockEnricher(sketch_id="test", scan_id="test")-- domain = Domain(domain="example.com")-- # Legacy signature should still work- enricher.create_node("domain", "domain", "example.com", **domain.__dict__)---def test_create_relationship_legacy_signature():- """Test that legacy create_relationship signature still works."""- enricher = MockEnricher(sketch_id="test", scan_id="test")-- # Legacy signature should still work- enricher.create_relationship(- "individual",- "full_name",- "John Doe",- "domain",- "domain",- "example.com",- "HAS_DOMAIN",- )---def test_get_primary_field():- """Test the _get_primary_field helper method."""- enricher = MockEnricher(sketch_id="test", scan_id="test")-- # Test with Domain (has primary field marked)- domain = Domain(domain="example.com")- assert enricher._get_primary_field(domain) == "domain"-- # Test with Email (has primary field marked)- email = Email(email="[email protected]")- assert enricher._get_primary_field(email) == "email"-- # Test with Individual (no primary field marked, falls back to first required field)- individual = Individual(first_name="John", last_name="Doe", full_name="John Doe")- assert enricher._get_primary_field(individual) == "full_name"---def test_create_node_with_property_override():- """Test that property overrides work with Pydantic objects."""- enricher = MockEnricher(sketch_id="test", scan_id="test")-- domain = Domain(domain="example.com")-- # Should be able to override properties- enricher.create_node(domain, type="subdomain")diff --git a/flowsint-core/tests/test_enricher_base_simplified_api.py b/flowsint-core/tests/test_enricher_base_simplified_api.pydeleted file mode 100644index 9569f64e..00000000--- a/flowsint-core/tests/test_enricher_base_simplified_api.py+++ /dev/null@@ -1,117 +0,0 @@-"""Test simplified API for create_node and create_relationship."""-import pytest-from flowsint_core.core.enricher_base import Enricher-from flowsint_types.domain import Domain-from flowsint_types.email import Email-from flowsint_types.individual import Individual-from typing import List---class MockEnricher(Enricher):- """Simple enricher for testing."""-- InputType = Domain- OutputType = Domain-- @classmethod- def name(cls) -> str:- return "test_enricher"-- @classmethod- def category(cls) -> str:- return "Test"-- @classmethod- def key(cls) -> str:- return "domain"-- async def scan(self, data: List[InputType]) -> List[OutputType]:- return data---def test_create_node_with_pydantic_object():- """Test that create_node works with Pydantic objects."""- enricher = MockEnricher(sketch_id="test", scan_id="test")-- # Create a domain object- domain = Domain(domain="example.com")-- # This should not raise an error- enricher.create_node(domain)-- # Verify the helper method works- primary_field = enricher._get_primary_field(domain)- assert primary_field == "domain"---def test_create_relationship_with_pydantic_objects():- """Test that create_relationship works with Pydantic objects."""- enricher = MockEnricher(sketch_id="test", scan_id="test")-- # Create objects- individual = Individual(- first_name="John",- last_name="Doe",- full_name="John Doe"- )- domain = Domain(domain="example.com")-- # This should not raise an error- enricher.create_relationship(individual, domain, "HAS_DOMAIN")---def test_create_node_legacy_signature():- """Test that legacy create_node signature still works."""- enricher = MockEnricher(sketch_id="test", scan_id="test")-- domain = Domain(domain="example.com")-- # Legacy signature should still work- enricher.create_node("domain", "domain", "example.com", **domain.__dict__)---def test_create_relationship_legacy_signature():- """Test that legacy create_relationship signature still works."""- enricher = MockEnricher(sketch_id="test", scan_id="test")-- # Legacy signature should still work- enricher.create_relationship(- "individual",- "full_name",- "John Doe",- "domain",- "domain",- "example.com",- "HAS_DOMAIN"- )---def test_get_primary_field():- """Test the _get_primary_field helper method."""- enricher = MockEnricher(sketch_id="test", scan_id="test")-- # Test with Domain (has primary field marked)- domain = Domain(domain="example.com")- assert enricher._get_primary_field(domain) == "domain"-- # Test with Email (has primary field marked)- email = Email(email="[email protected]")- assert enricher._get_primary_field(email) == "email"-- # Test with Individual (no primary field marked, falls back to first required field)- individual = Individual(- first_name="John",- last_name="Doe",- full_name="John Doe"- )- assert enricher._get_primary_field(individual) == "full_name"---def test_create_node_with_property_override():- """Test that property overrides work with Pydantic objects."""- enricher = MockEnricher(sketch_id="test", scan_id="test")-- domain = Domain(domain="example.com")-- # Should be able to override properties- enricher.create_node(domain, type="subdomain")