A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat(core): cleaner code separation + usage of nodeLabel
dextmorgn3c39f9aeb797
diff --git a/flowsint-core/src/flowsint_core/__init__.py b/flowsint-core/src/flowsint_core/__init__.pyindex 9f3ddb92..f6d37ca2 100644--- a/flowsint-core/src/flowsint_core/__init__.py+++ b/flowsint-core/src/flowsint_core/__init__.py@@ -7,7 +7,7 @@ from .core.celery import *from .core.config import *from .core.enums import *from .core.events import *-from .core.graph_db import *+from .core.graph import *from .core.logger import *from .core.models import *from .core.postgre_db import *diff --git a/flowsint-core/src/flowsint_core/core/enricher_base.py b/flowsint-core/src/flowsint_core/core/enricher_base.pyindex 43b351c8..5e272007 100644--- a/flowsint-core/src/flowsint_core/core/enricher_base.py+++ b/flowsint-core/src/flowsint_core/core/enricher_base.py@@ -1,12 +1,13 @@from abc import ABC, abstractmethod-from typing import List, Dict, Any, Optional-from pydantic import ValidationError, BaseModel, Field, create_model, TypeAdapter+from typing import Any, Dict, List, Optional++from pydantic import BaseModel, Field, TypeAdapter, ValidationError, create_modelfrom pydantic.config import ConfigDict-from .graph_db import Neo4jConnection++from ..utils import resolve_type+from .graph import GraphService, create_graph_servicefrom .logger import Loggerfrom .vault import VaultProtocol-from .graph_service import GraphService, create_graph_service-from ..utils import resolve_typeclass InvalidEnricherParams(Exception):@@ -111,7 +112,6 @@ class Enricher(ABC):self,sketch_id: Optional[str] = None,scan_id: Optional[str] = None,- neo4j_conn: Optional[Neo4jConnection] = None,params_schema: Optional[List[Dict[str, Any]]] = None,vault: Optional[VaultProtocol] = None,params: Optional[Dict[str, Any]] = None,@@ -119,20 +119,17 @@ class Enricher(ABC):):self.scan_id = scan_id or "default"self.sketch_id = sketch_id or "system"- self.neo4j_conn = neo4j_conn # Kept for backward compatibilityself.vault = vaultself.params_schema = params_schema or []self.ParamsModel = build_params_model(self.params_schema)self.params: Dict[str, Any] = params or {}- # Initialize graph service (new architecture)+ # Initialize graph service (uses singleton connection by default)if graph_service:self._graph_service = graph_serviceelse:- # Create graph service with the provided or singleton connectionself._graph_service = create_graph_service(sketch_id=self.sketch_id,- neo4j_connection=neo4j_conn,enable_batching=True,)@@ -439,7 +436,7 @@ class Enricher(ABC):)return []- def create_node(self, node_obj, **properties) -> None:+ def create_node(self, node_obj) -> None:"""Create a single Neo4j node.@@ -458,7 +455,7 @@ class Enricher(ABC):node_obj: Either a Pydantic object or node label string**properties: Additional node properties or overrides"""- self._graph_service.create_node(node_obj=node_obj, **properties)+ self._graph_service.create_node_from_flowsint_type(node_obj=node_obj)def create_relationship(self,diff --git a/flowsint-core/src/flowsint_core/core/graph/__init__.py b/flowsint-core/src/flowsint_core/core/graph/__init__.pynew file mode 100644index 00000000..75cc9a8e--- /dev/null+++ b/flowsint-core/src/flowsint_core/core/graph/__init__.py@@ -0,0 +1,43 @@+"""+Graph module for Neo4j operations.++This module provides all graph-related functionality including:+- Neo4j connection management+- Repository pattern for graph operations+- Serialization utilities+- High-level graph service+"""++from .connection import Neo4jConnection, neo4j_connection+from .repository import Neo4jGraphRepository+from .repository_protocol import GraphRepositoryProtocol+from .serializer import GraphSerializer+from .service import GraphService, create_graph_service, LoggerProtocol+from .types import (+ GraphData,+ GraphEdge,+ GraphNode,+ Neo4jDict,+ NodeMetadata,+)++__all__ = [+ # Connection+ "Neo4jConnection",+ "neo4j_connection",+ # Repository+ "Neo4jGraphRepository",+ "GraphRepositoryProtocol",+ # Serializer+ "GraphSerializer",+ # Service+ "GraphService",+ "create_graph_service",+ "LoggerProtocol",+ # Types+ "GraphData",+ "GraphEdge",+ "GraphNode",+ "Neo4jDict",+ "NodeMetadata",+]diff --git a/flowsint-core/src/flowsint_core/core/graph_db.py b/flowsint-core/src/flowsint_core/core/graph/connection.pysimilarity index 82%rename from flowsint-core/src/flowsint_core/core/graph_db.pyrename to flowsint-core/src/flowsint_core/core/graph/connection.pyindex bf29fce3..f4986a6d 100644--- a/flowsint-core/src/flowsint_core/core/graph_db.py+++ b/flowsint-core/src/flowsint_core/core/graph/connection.py@@ -1,8 +1,16 @@+"""+Neo4j connection management.++This module provides a singleton connection manager for Neo4j with proper+connection pooling and transaction management.+"""+import osfrom threading import Lock-from typing import Optional, Dict, Any, List-from neo4j import GraphDatabase, Driver, Session+from typing import Any, Dict, List, Optional+from dotenv import load_dotenv+from neo4j import Driver, GraphDatabaseload_dotenv()@@ -16,7 +24,7 @@ class Neo4jConnection:connection pooling and resource management."""- _instance: Optional['Neo4jConnection'] = None+ _instance: Optional["Neo4jConnection"] = None_lock: Lock = Lock()_driver: Optional[Driver] = None@@ -44,22 +52,26 @@ class Neo4jConnection:"""# Only initialize onceif self._driver is None:- self._uri = uri or os.getenv("NEO4J_URI_BOLT")- self._user = user or os.getenv("NEO4J_USERNAME")- self._password = password or os.getenv("NEO4J_PASSWORD")+ resolved_uri = uri or os.getenv("NEO4J_URI_BOLT")+ resolved_user = user or os.getenv("NEO4J_USERNAME")+ resolved_password = password or os.getenv("NEO4J_PASSWORD")- if not all([self._uri, self._user, self._password]):+ if not resolved_uri or not resolved_user or not resolved_password:raise ValueError("Neo4j connection credentials are required")+ self._uri = resolved_uri+ self._user = resolved_user+ self._password = resolved_password+self._driver = GraphDatabase.driver(self._uri,auth=(self._user, self._password),max_connection_pool_size=50,- connection_acquisition_timeout=60.0+ connection_acquisition_timeout=60.0,)@classmethod- def get_instance(cls) -> 'Neo4jConnection':+ def get_instance(cls) -> "Neo4jConnection":"""Get the singleton instance.@@ -79,7 +91,9 @@ class Neo4jConnection:"""return self._driver- def query(self, query: str, parameters: Dict[str, Any] = None) -> List[Dict[str, Any]]:+ def query(+ self, query: str, parameters: Dict[str, Any] = None+ ) -> List[Dict[str, Any]]:"""Execute a single query.@@ -102,7 +116,9 @@ class Neo4jConnection:return {}return {k: v for k, v in parameters.items() if k is not None}- def execute_write(self, query: str, parameters: Dict[str, Any] = None) -> List[Dict[str, Any]]:+ def execute_write(+ self, query: str, parameters: Dict[str, Any] = None+ ) -> List[Dict[str, Any]]:"""Execute a write query within a write transaction.@@ -113,6 +129,7 @@ class Neo4jConnection:Returns:List of result records as dictionaries"""+def _execute(tx):cleaned_params = self._clean_parameters(parameters)result = tx.run(query, cleaned_params)@@ -121,7 +138,9 @@ class Neo4jConnection:with self._driver.session() as session:return session.execute_write(_execute)- def execute_batch(self, queries: List[tuple[str, Dict[str, Any]]]) -> List[List[Dict[str, Any]]]:+ def execute_batch(+ self, queries: List[tuple[str, Dict[str, Any]]]+ ) -> List[List[Dict[str, Any]]]:"""Execute multiple queries in a single transaction.@@ -131,6 +150,7 @@ class Neo4jConnection:Returns:List of results, one for each query"""+def _execute_batch(tx):results = []for query, params in queries:@@ -189,5 +209,6 @@ try:neo4j_connection = Noneexcept Exception as e:import logging+logging.getLogger(__name__).warning(f"Failed to initialize Neo4j connection: {e}")neo4j_connection = Nonediff --git a/flowsint-core/src/flowsint_core/core/graph_repository.py b/flowsint-core/src/flowsint_core/core/graph/repository.pysimilarity index 57%rename from flowsint-core/src/flowsint_core/core/graph_repository.pyrename to flowsint-core/src/flowsint_core/core/graph/repository.pyindex e1d48876..cae99897 100644--- a/flowsint-core/src/flowsint_core/core/graph_repository.py+++ b/flowsint-core/src/flowsint_core/core/graph/repository.py@@ -2,24 +2,24 @@Graph database repository for Neo4j operations.This module provides a repository pattern implementation for Neo4j,-handling node and relationship operations with batching support.+handling raw Neo4jDict object and operations with batching support."""from datetime import datetime, timezone-from typing import Any, Dict, List, Optional, Tuple, Union+from typing import Any, Dict, List, Optional, Tuple-from pydantic import BaseModel+from .connection import Neo4jConnection+from .types import Neo4jDict-from .graph_db import Neo4jConnection-from .graph_serializer import GraphSerializer--class GraphRepository:+class Neo4jGraphRepository:"""- Repository for Neo4j graph database operations.+ Neo4j main implementation of the graph repository.This class follows the Repository pattern, providing a clean abstractionover Neo4j operations and handling batching for improved performance.++ Implements GraphRepositoryProtocol for dependency injection."""def __init__(self, neo4j_connection: Optional[Neo4jConnection] = None):@@ -34,64 +34,20 @@ class GraphRepository:self._batch_operations: List[Tuple[str, Dict[str, Any]]] = []self._batch_size = 100- @staticmethod- def _get_primary_field(obj: BaseModel) -> str:- """Get the primary field of a Pydantic object."""- model_fields = obj.__class__.model_fields-- primary_field = None- for field_name, field_info in model_fields.items():- if field_info.json_schema_extra and field_info.json_schema_extra.get(- "primary"- ):- primary_field = field_name- break-- if primary_field is None:- for field_name, field_info in model_fields.items():- if field_info.is_required():- primary_field = field_name- break- if primary_field is None:- primary_field = next(iter(model_fields.keys()))-- return primary_field-- @staticmethod- def _extract_primary_value(obj: BaseModel) -> Any:- """- Extract the primitive value from a Pydantic object recursively.- Uses model_dump to properly serialize Pydantic types like HttpUrl.- """- primary_field = GraphRepository._get_primary_field(obj)- obj_dict = (- obj.model_dump(mode="json") if hasattr(obj, "model_dump") else obj.dict()- )- value = obj_dict.get(primary_field)-- if isinstance(value, dict):- nested_obj = getattr(obj, primary_field)- if isinstance(nested_obj, BaseModel):- return GraphRepository._extract_primary_value(nested_obj)-- return value-def create_node(self,- node_obj: BaseModel,- sketch_id: Optional[str] = None,- **properties: Any,+ node_obj: Neo4jDict,+ sketch_id: str,) -> Optional[str]:"""Create or update a single node in Neo4j.- Supports 1 signature:- 1. Pydantic object: create_node(obj, sketch_id="...", **overrides)+ Supported signature:+ 1. Neo4jDict object: create_node(obj)Args:- node_obj: a Pydantic object- sketch_id: Investigation sketch ID (required)- **properties: Additional node properties or overrides+ node_obj: a Neo4jDict object+ sketch_id: strReturns:Element ID of created/updated node@@ -99,37 +55,29 @@ class GraphRepository:if not self._connection:return None- query, params = self._build_node_query(node_obj, sketch_id, **properties)+ query, params = self._build_node_query(node_obj, sketch_id)result = self._connection.query(query, params)return result[0]["id"] if result else Nonedef create_relationship(self,- from_obj: BaseModel,- to_obj: BaseModel,- rel_label: Optional[str] = None,- sketch_id: Optional[str] = None,- **properties: Any,+ rel_obj: Neo4jDict,+ sketch_id: str,) -> None:"""Create a relationship between two nodes.Supports one signature:- - Pydantic objects: create_relationship(obj1, obj2, "REL_TYPE", sketch_id="...")+ - Pydantic objects: create_relationship(relation, sketch_id="...")Args:- from_obj: a Pydantic object (source)- to_obj: a Pydantic object (target)- rel_label: Relationship label (ex: "IS_CONNECTED_TO")+ rel_obj: the Neo4jDict objectsketch_id: Investigation sketch ID (required)- **properties: Additional relationship properties"""if not self._connection:return- query, params = self._build_relationship_query(- from_obj, to_obj, rel_label, sketch_id, **properties- )+ query, params = self._build_relationship_query(rel_obj, sketch_id)self._connection.execute_write(query, params)@@ -155,155 +103,53 @@ class GraphRepository:self.flush_batch()def _build_node_query(- self,- node_obj: BaseModel,- sketch_id: str,- **properties: Any,+ self, node_obj: Neo4jDict, sketch_id: str) -> Tuple[str, Dict[str, Any]]:- """Build a node creation query."""-- node_type = node_obj.__class__.__name__.lower()- primary_field = self._get_primary_field(node_obj)- key_prop = primary_field- key_value = getattr(node_obj, primary_field, None)-- # If primary key value is None or empty, fallback to label- if key_value is None or (isinstance(key_value, str) and not key_value):- key_prop = "label"- key_value = properties.get("label") or getattr(node_obj, "label", "Node")+ node_label = node_obj.get("nodeLabel")+ node_type = node_obj.get("nodeType")- if isinstance(key_value, BaseModel):- key_value = self._extract_primary_value(key_value)-- obj_dict = (- node_obj.model_dump(mode="json")- if hasattr(node_obj, "model_dump")- else node_obj.dict()- )- # Extract only non-dict values, skip None keys- obj_properties = {- k: (v if v is not None else "")- for k, v in obj_dict.items()- if k is not None and not isinstance(v, dict)- }- obj_properties.update(properties)- properties = obj_properties-- if not sketch_id:- raise ValueError("sketch_id is required")- if not key_prop or key_prop is None:- raise ValueError(f"key_prop cannot be None for node type {node_type}")-- serialized_props = GraphSerializer.serialize_properties(properties)- serialized_props["type"] = node_type.lower()- serialized_props["sketch_id"] = sketch_id- label = serialized_props.get("label", key_value)- serialized_props["label"] = label-- # Clean None keys from serialized_props- serialized_props = {k: v for k, v in serialized_props.items() if k is not None}-- set_clauses = [- f"n.{prop} = ${prop}"- for prop in serialized_props.keys()- if prop != "sketch_id"- ]-- # Build params, ensuring no None keys+ # paramètres Neo4jparams = {- key_prop: key_value,+ "props": node_obj, # flat with keys containing "."+ "node_label": node_label,+ "sketch_id": sketch_id,"created_at": datetime.now(timezone.utc).isoformat(),- **serialized_props,}- # MERGE on both key_prop AND sketch_id for uniqueness per sketch- # Use ON CREATE SET to only set created_at when creating, not updatingquery = f"""- MERGE (n:{node_type} {{{key_prop}: ${key_prop}, sketch_id: $sketch_id}})+ MERGE (n:{node_type} {{ nodeLabel: $node_label, sketch_id: $sketch_id }})ON CREATE SET n.created_at = $created_at- SET {", ".join(set_clauses)}- RETURN elementId(n) as id+ SET n += $props+ RETURN elementId(n) AS id"""return query, paramsdef _build_relationship_query(self,- from_obj: BaseModel,- to_obj: BaseModel,- rel_label: Optional[str] = None,- sketch_id: Optional[str] = None,- **properties: Any,+ rel_obj: Neo4jDict,+ sketch_id: str,) -> Tuple[str, Dict[str, Any]]:- """Build a relationship creation query."""- # From object- from_node_type = from_obj.__class__.__name__.lower()- from_primary_field = self._get_primary_field(from_obj)- from_obj_dict = (- from_obj.model_dump(mode="json")- if hasattr(from_obj, "model_dump")- else from_obj.dict()- )- from_key_value = from_obj_dict.get(from_primary_field)-- if isinstance(from_key_value, dict):- nested_obj = getattr(from_obj, from_primary_field)- if isinstance(nested_obj, BaseModel):- from_key_value = self._extract_primary_value(nested_obj)- # To object- to_node_type = to_obj.__class__.__name__.lower()- to_primary_field = self._get_primary_field(to_obj)- to_obj_dict = (- to_obj.model_dump(mode="json")- if hasattr(to_obj, "model_dump")- else to_obj.dict()- )- to_key_value = to_obj_dict.get(to_primary_field)-- if isinstance(to_key_value, dict):- nested_obj = getattr(to_obj, to_primary_field)- if isinstance(nested_obj, BaseModel):- to_key_value = self._extract_primary_value(nested_obj)-- from_type = from_node_type- from_key = from_primary_field- from_value = from_key_value- to_type = to_node_type- to_key = to_primary_field- to_value = to_key_value- rel_type = rel_label-- if not sketch_id:- raise ValueError("sketch_id is required")- if not from_key or from_key is None:- raise ValueError(- f"from_key cannot be None for relationship type {rel_type}"- )- if not to_key or to_key is None:- raise ValueError(f"to_key cannot be None for relationship type {rel_type}")-- serialized_props = GraphSerializer.serialize_properties(properties)- serialized_props["sketch_id"] = sketch_id+ from_type = rel_obj["from_type"]+ from_label = rel_obj["from_label"]+ to_type = rel_obj["to_type"]+ to_label = rel_obj["to_label"]+ rel_label = rel_obj["rel_label"]- # Clean None keys- serialized_props = {k: v for k, v in serialized_props.items() if k is not None}-- if serialized_props:- props_str = ", ".join([f"{k}: ${k}" for k in serialized_props.keys()])- rel_props = f"{{{props_str}}}"- else:- rel_props = "{sketch_id: $sketch_id}"+ params = {+ "from_label": from_label,+ "to_label": to_label,+ "sketch_id": sketch_id,+ "props": rel_obj,+ }query = f"""- MATCH (from:{from_type} {{{from_key}: $from_value, sketch_id: $sketch_id}})- MATCH (to:{to_type} {{{to_key}: $to_value, sketch_id: $sketch_id}})- MERGE (from)-[:{rel_type} {rel_props}]->(to)+ MATCH (from:{from_type} {{nodeLabel: $from_label, sketch_id: $sketch_id}})+ MATCH (to:{to_type} {{nodeLabel: $to_label, sketch_id: $sketch_id}})+ MERGE (from)-[r:{rel_label}]->(to)+ SET r += $props"""- params = {"from_value": from_value, "to_value": to_value, **serialized_props}- # Clean None keys from params- params = {k: v for k, v in params.items() if k is not None}-return query, paramsdef flush_batch(self) -> None:@@ -337,15 +183,15 @@ class GraphRepository:def batch_create_nodes(self,- nodes: List[BaseModel],+ nodes: List[Neo4jDict],sketch_id: str,) -> Dict[str, Any]:"""Create multiple nodes in a single batch transaction.Args:- nodes: List of Pydantic model instances to create- sketch_id: Investigation sketch ID+ nodes: List of Neo4jDict model instances to insert+ sketch_id: investigation sketch idReturns:Dictionary with:@@ -370,8 +216,7 @@ class GraphRepository:for idx, node_obj in enumerate(nodes):try:query, params = self._build_node_query(- node_obj=node_obj,- sketch_id=sketch_id,+ node_obj=node_obj, sketch_id=sketch_id)batch_operations.append((query, params))except Exception as e:@@ -411,11 +256,7 @@ class GraphRepository:Args:edges: List of edge dictionaries, each with:- - from_obj: Source node Pydantic model- - to_obj: Target node Pydantic model- - rel_label: Relationship type/label- - properties: Optional dict of relationship properties- sketch_id: Investigation sketch ID+ - rel_obj: Source node Neo4jDict modelReturns:Dictionary with:@@ -434,23 +275,8 @@ class GraphRepository:for idx, edge in enumerate(edges):try:- from_obj = edge.get("from_obj")- to_obj = edge.get("to_obj")- rel_label = edge.get("rel_label")- properties = edge.get("properties", {})-- if not from_obj or not to_obj or not rel_label:- errors.append(- f"Edge {idx}: Missing required fields (from_obj, to_obj, or rel_label)"- )- continue-query, params = self._build_relationship_query(- from_obj=from_obj,- to_obj=to_obj,- rel_label=rel_label,- sketch_id=sketch_id,- **properties,+ rel_obj=edge, sketch_id=sketch_id)batch_operations.append((query, params))except Exception as e:@@ -474,9 +300,9 @@ class GraphRepository:def batch_create_edges_by_element_id(self,- edges: List[Dict[str, Any]],+ edges: List[Neo4jDict],sketch_id: str,- ) -> Dict[str, Any]:+ ) -> Neo4jDict:"""Create multiple edges/relationships using element IDs in a single batch transaction.@@ -487,7 +313,7 @@ class GraphRepository:edges: List of edge dictionaries, each with:- from_element_id: Source node element ID- to_element_id: Target node element ID- - rel_type: Relationship type/label+ - rel_label: Relationship type/label- properties: Optional dict of relationship propertiessketch_id: Investigation sketch ID@@ -510,8 +336,7 @@ class GraphRepository:try:from_element_id = edge.get("from_element_id")to_element_id = edge.get("to_element_id")- rel_type = edge.get("rel_type", "RELATED_TO")- properties = edge.get("properties", {})+ rel_label = edge.get("rel_label", "RELATED_TO")if not from_element_id or not to_element_id:errors.append(@@ -519,14 +344,10 @@ class GraphRepository:)continue- # Serialize properties- serialized_props = GraphSerializer.serialize_properties(properties)- serialized_props["sketch_id"] = sketch_id+ edge["sketch_id"] = sketch_id# Build relationship properties string- props_str = ", ".join(- [f"{k}: ${k}_{idx}" for k in serialized_props.keys()]- )+ props_str = ", ".join([f"{k}: ${k}_{idx}" for k in edge.keys()])rel_props = (f"{{{props_str}}}" if props_str else "{sketch_id: $sketch_id}")@@ -534,7 +355,7 @@ class GraphRepository:query = f"""MATCH (from) WHERE elementId(from) = $from_id_{idx}MATCH (to) WHERE elementId(to) = $to_id_{idx}- MERGE (from)-[r:`{rel_type}` {rel_props}]->(to)+ MERGE (from)-[r:`{rel_label}` {rel_props}]->(to)"""# Build params with unique keys for batch execution@@ -543,7 +364,7 @@ class GraphRepository:f"to_id_{idx}": to_element_id,}# Add serialized properties with index suffix- for k, v in serialized_props.items():+ for k, v in edge.items():params[f"{k}_{idx}"] = vbatch_operations.append((query, params))@@ -567,54 +388,24 @@ class GraphRepository:return {"edges_created": 0, "errors": errors}def update_node(- self,- element_id: str,- node_obj: BaseModel,- sketch_id: str,+ self, element_id: str, updates: Neo4jDict, sketch_id: str) -> Optional[str]:- """- Update an existing node's properties using a Pydantic model.-- Args:- element_id: Neo4j element ID of the node to update- node_obj: Pydantic model instance with updated data- sketch_id: Investigation sketch ID (for safety)-- Returns:- Element ID of the updated node or None if not found- """if not self._connection:return None- # Extract properties from Pydantic object- obj_dict = (- node_obj.model_dump(mode="json")- if hasattr(node_obj, "model_dump")- else node_obj.dict()- )-- # Extract only non-dict values, skip None keys- properties = {- k: (v if v is not None else "")- for k, v in obj_dict.items()- if k is not None and not isinstance(v, dict)- }-- # Serialize properties- serialized_props = GraphSerializer.serialize_properties(properties)- serialized_props["type"] = node_obj.__class__.__name__.lower()-- # Build SET clauses- set_clauses = [f"n.{prop} = ${prop}" for prop in serialized_props.keys()]-- query = f"""+ query = """MATCH (n)WHERE elementId(n) = $element_id AND n.sketch_id = $sketch_id- SET {", ".join(set_clauses)}- RETURN elementId(n) as id+ SET n += $props+ RETURN elementId(n) AS id"""- params = {"element_id": element_id, "sketch_id": sketch_id, **serialized_props}+ params = {+ "element_id": element_id,+ "sketch_id": sketch_id,+ "props": updates,+ }+result = self._connection.query(query, params)return result[0]["id"] if result else None@@ -699,17 +490,17 @@ class GraphRepository:self, sketch_id: str, limit: int = 100000) -> Dict[str, List[Dict[str, Any]]]:"""- Get all nodes and relationships for a sketch.+ Get all nodes and edges for a sketch.Args:sketch_id: Investigation sketch IDlimit: Maximum number of nodes to returnReturns:- Dictionary with 'nodes' and 'relationships' lists+ Dictionary with 'nodes' and 'edges' lists"""if not self._connection:- return {"nodes": [], "relationships": []}+ return {"nodes": [], "edges": []}# Get all nodes for the sketch# Use OPTIONAL MATCH to avoid Neo4j warning when sketch_id property doesn't exist yet@@ -726,11 +517,10 @@ class GraphRepository:)if not nodes_result:- return {"nodes": [], "relationships": []}+ return {"nodes": [], "edges": []}node_ids = [record["id"] for record in nodes_result]-- # Get all relationships between these nodes+ # Get all edges between these nodesrels_query = """UNWIND $node_ids AS nidMATCH (a)-[r]->(b)@@ -739,50 +529,30 @@ class GraphRepository:elementId(b) as target, properties(r) as data"""rels_result = self._connection.query(rels_query, {"node_ids": node_ids})-- return {"nodes": nodes_result, "relationships": rels_result or []}+ return {"nodes": nodes_result, "edges": rels_result or []}def update_relationship(- self,- element_id: str,- properties: Dict[str, Any],- sketch_id: str,+ self, element_id: str, rel_obj: Neo4jDict, sketch_id: str) -> Optional[Dict[str, Any]]:- """- Update an existing relationship's properties.-- Args:- element_id: Neo4j element ID of the edge to update- properties: Dictionary of properties to update- sketch_id: Investigation sketch ID (for safety)-- Returns:- Updated edge data or None if not found- """if not self._connection:return None- # Filter out None values- filtered_props = {- k: (v if v is not None else "")- for k, v in properties.items()- if k is not None- }-- # Serialize properties- serialized_props = GraphSerializer.serialize_properties(filtered_props)-- # Build SET clauses- set_clauses = [f"r.{prop} = ${prop}" for prop in serialized_props.keys()]+ query = """+ MATCH ()-[r]->()+ WHERE elementId(r) = $element_id AND r.sketch_id = $sketch_id+ SET r += $props+ RETURN+ elementId(r) AS id,+ type(r) AS type,+ properties(r) AS data+ """- query = f"""- MATCH ()-[r]->()- WHERE elementId(r) = $element_id AND r.sketch_id = $sketch_id- SET {", ".join(set_clauses)}- RETURN elementId(r) as id, COALESCE(r.label, type(r)) as type, properties(r) as data- """+ params = {+ "element_id": element_id,+ "sketch_id": sketch_id,+ "props": rel_obj,+ }- params = {"element_id": element_id, "sketch_id": sketch_id, **serialized_props}result = self._connection.query(query, params)return result[0] if result else None@@ -790,9 +560,8 @@ class GraphRepository:self,from_element_id: str,to_element_id: str,- rel_type: str,+ rel_label: str,sketch_id: str,- **properties: Any,) -> Optional[Dict[str, Any]]:"""Create a relationship between two nodes using their element IDs.@@ -800,17 +569,15 @@ class GraphRepository:Args:from_element_id: Source node element IDto_element_id: Target node element ID- rel_type: Relationship type+ rel_label: Relationship typesketch_id: Investigation sketch ID- **properties: Additional relationship propertiesReturns:Created relationship properties or None"""if not self._connection:return None-- serialized_props = GraphSerializer.serialize_properties(properties)+ serialized_props = {}serialized_props["sketch_id"] = sketch_idprops_str = ", ".join([f"{k}: ${k}" for k in serialized_props.keys()])@@ -819,14 +586,14 @@ class GraphRepository:query = f"""MATCH (a) WHERE elementId(a) = $from_idMATCH (b) WHERE elementId(b) = $to_id- MERGE (a)-[r:`{rel_type}` {rel_props}]->(b)+ MERGE (a)-[r:`{rel_label}` {rel_props}]->(b)RETURN properties(r) as rel"""params = {"from_id": from_element_id,"to_id": to_element_id,- **serialized_props,+ "sketch_id": sketch_id,}result = self._connection.query(query, params)@@ -906,7 +673,7 @@ class GraphRepository:query, {"node_ids": node_ids, "sketch_id": sketch_id})- return [record["data"] for record in result] if result else []+ return resultdef merge_nodes(self,@@ -931,7 +698,7 @@ class GraphRepository:return Nonenode_type = new_node_data.get("type", "Node")- properties = GraphSerializer.serialize_properties(new_node_data)+ properties = {}properties["sketch_id"] = sketch_idis_reusing_node = new_node_id and new_node_id in old_node_ids@@ -1008,131 +775,93 @@ class GraphRepository:return new_node_element_id- def get_related_nodes(self, node_id: str, sketch_id: str) -> Dict[str, Any]:+ def get_neighbors(self, node_id: str, sketch_id: str) -> Dict[str, Any]:"""- Get a node and all its direct relationships and connected nodes.-- Args:- node_id: Element ID of the center node- sketch_id: Investigation sketch ID+ Get a node and all its direct relationships and connected nodes+ within the same sketch.Returns:- Dictionary with 'nds' (nodes) and 'rls' (relationships) lists+ {+ "nodes": [ { "id", "data" } ],+ "edges": [ { "id", "source", "target", "label" } ]+ }"""+if not self._connection:- return {"nds": [], "rls": []}+ return {"nodes": [], "edges": []}- center_query = """+ query = """MATCH (n)WHERE elementId(n) = $node_id AND n.sketch_id = $sketch_id- RETURN elementId(n) as id, labels(n) as labels, properties(n) as data- """-- center_result = self._connection.query(- center_query, {"sketch_id": sketch_id, "node_id": node_id}- )- if not center_result:- return {"nds": [], "rls": []}+ OPTIONAL MATCH (n)-[r]-(other)+ WHERE other.sketch_id = $sketch_id AND other <> n- relationships_query = """- MATCH (n)- WHERE elementId(n) = $node_id AND n.sketch_id = $sketch_id- OPTIONAL MATCH (n)-[r]->(other)- WHERE other.sketch_id = $sketch_id- OPTIONAL MATCH (other)-[r2]->(n)- WHERE other.sketch_id = $sketch_id- RETURN- elementId(r) as rel_id,- type(r) as rel_type,- properties(r) as rel_data,- elementId(other) as other_node_id,- labels(other) as other_node_labels,- properties(other) as other_node_data,- 'outgoing' as direction- UNION- MATCH (n)- WHERE elementId(n) = $node_id AND n.sketch_id = $sketch_id- OPTIONAL MATCH (other)-[r]->(n)- WHERE other.sketch_id = $sketch_idRETURN- elementId(r) as rel_id,- type(r) as rel_type,- properties(r) as rel_data,- elementId(other) as other_node_id,- labels(other) as other_node_labels,- properties(other) as other_node_data,- 'incoming' as direction+ elementId(n) AS center_id,+ properties(n) AS center_data,++ elementId(r) AS rel_id,+ type(r) AS rel_label,++ elementId(other) AS other_id,+ properties(other) AS other_data,++ CASE+ WHEN r IS NULL THEN NULL+ WHEN startNode(r) = n THEN 'outgoing'+ ELSE 'incoming'+ END AS direction"""result = self._connection.query(- relationships_query, {"sketch_id": sketch_id, "node_id": node_id}+ query,+ {"node_id": node_id, "sketch_id": sketch_id},)- center_record = center_result[0]+ if not result:+ return {"nodes": [], "edges": []}++ first = result[0]center_node = {- "id": center_record["id"],- "labels": center_record["labels"],- "data": center_record["data"],- "label": center_record["data"].get("label", "Node"),- "type": "custom",- "caption": center_record["data"].get("label", "Node"),+ "id": first["center_id"],+ "data": first["center_data"],}- related_nodes = []- relationships = []- seen_nodes = set()- seen_relationships = set()+ nodes = {center_node["id"]: center_node}+ edges = {}for record in result:if not record["rel_id"]:continue- if record["rel_id"] not in seen_relationships:+ other_id = record["other_id"]++ # nodes+ if other_id not in nodes:+ nodes[other_id] = {+ "id": other_id,+ "data": record["other_data"],+ }++ # edges+ if record["rel_id"] not in edges:if record["direction"] == "outgoing":- relationships.append(- {- "id": record["rel_id"],- "type": "straight",- "source": center_node["id"],- "target": record["other_node_id"],- "data": record["rel_data"],- "caption": record["rel_type"],- }- )+ source, target = center_node["id"], other_idelse:- relationships.append(- {- "id": record["rel_id"],- "type": "straight",- "source": record["other_node_id"],- "target": center_node["id"],- "data": record["rel_data"],- "caption": record["rel_type"],- }- )- seen_relationships.add(record["rel_id"])-- if (- record["other_node_id"]- and record["other_node_id"] not in seen_nodes- and record["other_node_id"] != center_node["id"]- ):- related_nodes.append(- {- "id": record["other_node_id"],- "labels": record["other_node_labels"],- "data": record["other_node_data"],- "label": record["other_node_data"].get("label", "Node"),- "type": "custom",- "caption": record["other_node_data"].get("label", "Node"),- }- )- seen_nodes.add(record["other_node_id"])+ source, target = other_id, center_node["id"]- all_nodes = [center_node] + related_nodes+ edges[record["rel_id"]] = {+ "id": record["rel_id"],+ "source": source,+ "target": target,+ "label": record["rel_label"],+ }- return {"nds": all_nodes, "rls": relationships}+ return {+ "nodes": list(nodes.values()),+ "edges": list(edges.values()),+ }def count_nodes_by_sketch(self, sketch_id: str) -> int:"""diff --git a/flowsint-core/src/flowsint_core/core/graph/repository_protocol.py b/flowsint-core/src/flowsint_core/core/graph/repository_protocol.pynew file mode 100644index 00000000..9a12ed53--- /dev/null+++ b/flowsint-core/src/flowsint_core/core/graph/repository_protocol.py@@ -0,0 +1,131 @@+"""+Protocol for graph repository implementations.++This module defines the interface contract that all graph repository+implementations must follow, enabling dependency injection and easier testing.+"""++from typing import Any, Dict, List, Optional, Protocol, Tuple++from .types import Neo4jDict+++class GraphRepositoryProtocol(Protocol):+ """+ Protocol defining the interface for graph repository implementations.++ This protocol enables:+ - Dependency injection in GraphService+ - Easy mocking and testing without patching+ - Alternative implementations (e.g., InMemoryGraphRepository)+ """++ # Core node operations+ def create_node(self, node_obj: Neo4jDict, sketch_id: str) -> Optional[str]:+ """Create or update a single node. Returns element ID."""+ ...++ def update_node(+ self, element_id: str, updates: Neo4jDict, sketch_id: str+ ) -> Optional[str]:+ """Update a node by its element ID. Returns element ID."""+ ...++ def delete_nodes(self, node_ids: List[str], sketch_id: str) -> int:+ """Delete nodes by their element IDs. Returns count deleted."""+ ...++ def delete_all_sketch_nodes(self, sketch_id: str) -> int:+ """Delete all nodes for a sketch. Returns count deleted."""+ ...++ def get_nodes_by_ids(+ self, node_ids: List[str], sketch_id: str+ ) -> List[Dict[str, Any]]:+ """Get nodes by their element IDs."""+ ...++ def update_nodes_positions(+ self, positions: List[Dict[str, Any]], sketch_id: str+ ) -> int:+ """Update positions for multiple nodes. Returns count updated."""+ ...++ # Core relationship operations+ def create_relationship(self, rel_obj: Neo4jDict, sketch_id: str) -> None:+ """Create a relationship between two nodes."""+ ...++ def create_relationship_by_element_id(+ self,+ from_element_id: str,+ to_element_id: str,+ rel_label: str,+ sketch_id: str,+ ) -> Optional[Dict[str, Any]]:+ """Create a relationship using element IDs."""+ ...++ def update_relationship(+ self, element_id: str, rel_obj: Neo4jDict, sketch_id: str+ ) -> Optional[Dict[str, Any]]:+ """Update a relationship by its element ID."""+ ...++ def delete_relationships(self, relationship_ids: List[str], sketch_id: str) -> int:+ """Delete relationships by their element IDs. Returns count deleted."""+ ...++ # Graph queries+ def get_sketch_graph(+ self, sketch_id: str, limit: int = 100000+ ) -> Dict[str, List[Dict[str, Any]]]:+ """Get all nodes and edges for a sketch."""+ ...++ def get_neighbors(self, node_id: str, sketch_id: str) -> Dict[str, Any]:+ """Get a node and all its direct relationships."""+ ...++ # Merge operations+ def merge_nodes(+ self,+ old_node_ids: List[str],+ new_node_data: Dict[str, Any],+ new_node_id: Optional[str],+ sketch_id: str,+ ) -> Optional[str]:+ """Merge multiple nodes into one. Returns new element ID."""+ ...++ # Batch operations+ def batch_create_nodes(+ self, nodes: List[Neo4jDict], sketch_id: str+ ) -> Dict[str, Any]:+ """Create multiple nodes in a single batch."""+ ...++ def batch_create_edges_by_element_id(+ self, edges: List[Neo4jDict], sketch_id: str+ ) -> Dict[str, Any]:+ """Create multiple edges using element IDs in a single batch."""+ ...++ def add_to_batch(self, operation_type: str, **kwargs: Any) -> None:+ """Add an operation to the batch queue."""+ ...++ def flush_batch(self) -> None:+ """Execute all batched operations."""+ ...++ def set_batch_size(self, size: int) -> None:+ """Set the batch size for auto-flushing."""+ ...++ # Custom queries+ def query(+ self, cypher: str, parameters: Dict[str, Any] = {}+ ) -> List[Dict[str, Any]]:+ """Execute a custom Cypher query."""+ ...diff --git a/flowsint-core/src/flowsint_core/core/graph/serializer.py b/flowsint-core/src/flowsint_core/core/graph/serializer.pynew file mode 100644index 00000000..d5829319--- /dev/null+++ b/flowsint-core/src/flowsint_core/core/graph/serializer.py@@ -0,0 +1,199 @@+"""+Graph property serialization utilities.++This module provides utilities for serializing complex Python objects+into Neo4j-compatible primitive types, following the Single Responsibility Principle.+"""++from typing import Any, Dict, List++from flowsint_types import TYPE_REGISTRY, FlowsintType+from pydantic import BaseModel++from flowsint_core.utils import flatten, unflatten++from .types import GraphEdge, GraphNode, NodeMetadata+++class GraphSerializer:+ """+ Handles serialization of complex objects to Neo4j-compatible types.++ This class is responsible for converting Pydantic models, nested objects,+ and other complex types into primitive types that can be stored in Neo4j.+ """++ @staticmethod+ def _clean_empty_values(data: Dict[str, Any]) -> dict:+ """Remove empty string values from dict to avoid Pydantic validation errors."""+ cleaned = {}+ for key, value in data.items():+ if value == "" or value is None:+ continue+ if isinstance(value, dict):+ cleaned_nested = GraphSerializer._clean_empty_values(value)+ if cleaned_nested:+ cleaned[key] = cleaned_nested+ elif isinstance(value, list):+ cleaned_list = [+ GraphSerializer._clean_empty_values(item)+ if isinstance(item, dict)+ else item+ for item in value+ if item != "" and item is not None+ ]+ if cleaned_list:+ cleaned[key] = cleaned_list+ else:+ cleaned[key] = value+ return cleaned++ @staticmethod+ def flatten(dict: Dict[str, Any]):+ return flatten(dict, remove_empty=False)++ @staticmethod+ def parse_flowsint_type(entity: Dict, nodeType: str) -> FlowsintType:+ DetectedType = TYPE_REGISTRY.get_lowercase(nodeType)+ if not DetectedType:+ raise ValueError(f"Unknown type: {nodeType}")+ properties = GraphSerializer._clean_empty_values(entity)+ return DetectedType(**properties)++ @staticmethod+ def graph_node_to_flowsint_type(node: GraphNode) -> FlowsintType:+ return node.nodeProperties++ @staticmethod+ def neo4j_dict_to_graph_node(node_dict: Dict[str, Any]) -> GraphNode:+ """Convert a flattened Neo4j node record to a GraphNode instance.++ Unflattens the data, parses the nodeProperties into the appropriate+ FlowsintType subclass, and constructs a complete GraphNode.+ """+ data = node_dict.get("data")+ node_id = str(node_dict.get("id"))+ if not data:+ raise Exception("Could not find node data to extract.")+ # unflatten object from neo4j+ node_dict = unflatten(data)+ node_type = node_dict.get(+ "nodeType", node_dict.get("type", "")+ ) # legacy support type+ nodeLabel = str(node_dict.get("nodeLabel", node_dict.get("label", "")))+ node_properties = node_dict.get("nodeProperties", {})+ node_metadata = node_dict.get("nodeMetadata", {})++ node_properties.pop(+ "nodeLabel", None+ ) # remove nodeLabel from original pydantic++ entity = GraphSerializer.parse_flowsint_type(node_properties, node_type)+ return GraphNode(+ id=node_id,+ nodeLabel=nodeLabel,+ nodeType=node_type,+ nodeColor=data.get("nodeColor"),+ nodeSize=data.get("nodeSize"),+ nodeImage=data.get("nodeImage"),+ nodeIcon=data.get("nodeIcon"),+ nodeFlag=data.get("nodeFlag"),+ x=data.get("x"),+ y=data.get("y"),+ nodeProperties=entity,+ nodeMetadata=node_metadata,+ )++ @staticmethod+ def flowsint_type_to_neo4j_dict(entity: FlowsintType) -> Dict[str, Any]:+ node_type = entity.__class__.__name__.lower()+ node_label = entity.nodeLabel+ graph_node = GraphNode(+ id="",+ nodeColor=None,+ nodeIcon=None,+ nodeImage=None,+ nodeFlag=None,+ nodeLabel=node_label or "",+ nodeType=node_type,+ nodeProperties=entity,+ nodeMetadata=NodeMetadata(),+ )+ return GraphSerializer.graph_node_to_neo4j_dict(graph_node)++ @staticmethod+ def graph_node_to_neo4j_dict(node: GraphNode) -> Dict[str, Any]:+ """Convert a GraphNode to a flattened Neo4j-compatible dict.++ Serializes the model to JSON-compatible types and flattens nested+ structures into dot-notation keys for Neo4j property storage.+ """+ neo4j_dict = node.model_dump(mode="json", serialize_as_any=True)+ neo4j_dict_flatten = flatten(neo4j_dict, remove_empty=False)+ neo4j_dict_flatten.pop(+ "nodeProperties.nodeLabel", None+ ) # remove nodeLabel from original pydantic+ return neo4j_dict_flatten++ @staticmethod+ def neo4j_dict_to_graph_edge(edge_dict: Dict[str, Any]) -> GraphEdge:+ """Convert a Neo4j relationship record to a GraphEdge instance."""+ return GraphEdge(+ id=str(edge_dict.get("id")),+ source=str(edge_dict.get("source")),+ target=str(edge_dict.get("target")),+ label=str(edge_dict.get("type")),+ )++ @staticmethod+ def graph_edge_to_neo4j_dict(+ from_obj: BaseModel, to_obj: BaseModel, label: str+ ) -> Dict[str, Any]:+ """Build a Neo4j relationship dict for matching nodes by type and label.++ Creates a relationship descriptor that identifies source and target nodes+ by their nodeType and nodeLabel, allowing relationship creation without IDs.+ """+ from_type = (+ from_obj.__class__.__name__.lower()+ if isinstance(from_obj, FlowsintType)+ else from_obj.nodeType+ )+ to_type = (+ to_obj.__class__.__name__.lower()+ if isinstance(to_obj, FlowsintType)+ else to_obj.nodeType+ )+ return {+ "from_type": from_type,+ "from_label": from_obj.nodeLabel,+ "to_type": to_type,+ "to_label": to_obj.nodeLabel,+ "rel_label": label,+ }++ @staticmethod+ def deserialize_nodes(node_dicts: List[Dict[str, Any]]) -> List[GraphNode]:+ """Convert a list of Neo4j node records to GraphNode instances."""+ return [+ GraphSerializer.neo4j_dict_to_graph_node(node_dict)+ for node_dict in node_dicts+ ]++ @staticmethod+ def serialize_nodes(nodes: List[GraphNode]) -> List[Dict[str, Any]]:+ """Convert a list of Neo4j node records to GraphNode instances."""+ return [GraphSerializer.graph_node_to_neo4j_dict(node) for node in nodes]++ @staticmethod+ def serialize_flowsint_types(nodes: List[FlowsintType]) -> List[Dict[str, Any]]:+ """Convert a list of Neo4j node records to GraphNode instances."""+ return [GraphSerializer.flowsint_type_to_neo4j_dict(node) for node in nodes]++ @staticmethod+ def deserialize_edges(edge_dicts: List[Dict[str, Any]]) -> List[GraphEdge]:+ """Convert a list of Neo4j relationship records to GraphEdge instances."""+ return [+ GraphSerializer.neo4j_dict_to_graph_edge(edge_dict)+ for edge_dict in edge_dicts+ ]diff --git a/flowsint-core/src/flowsint_core/core/graph/service.py b/flowsint-core/src/flowsint_core/core/graph/service.pynew file mode 100644index 00000000..209194ba--- /dev/null+++ b/flowsint-core/src/flowsint_core/core/graph/service.py@@ -0,0 +1,361 @@+"""+Graph service for high-level graph operations.++This module provides a service layer for graph operations,+integrating repository and logging functionality.+"""++from typing import Any, Dict, List, Optional, Protocol++from flowsint_types import FlowsintType+from pydantic import BaseModel++from .repository import Neo4jGraphRepository+from .repository_protocol import GraphRepositoryProtocol+from .serializer import GraphSerializer+from .types import GraphData, GraphNode, Neo4jDict+++class LoggerProtocol(Protocol):+ """Protocol for logger implementations."""++ @staticmethod+ def graph_append(sketch_id: str, message: Dict[str, Any]) -> None:+ """Log a graph append message."""+ ...+++class GraphService:+ """+ High-level service for graph operations.++ This service provides a clean interface for enricher operations,+ handling both graph persistence and logging with proper separation of concerns.++ This service can support FlowsintType input or GraphNode input.+ """++ def __init__(+ self,+ sketch_id: str,+ repository: GraphRepositoryProtocol,+ logger: Optional[LoggerProtocol] = None,+ enable_batching: bool = False,+ ):+ """+ Initialize the graph service.++ Args:+ sketch_id: Investigation sketch ID+ repository: Repository instance (required - dependency injection)+ logger: Optional logger instance+ enable_batching: Enable batch operations++ Raises:+ ValueError: If repository is not provided+ """+ if repository is None:+ raise ValueError(+ "repository is required. Use create_graph_service() factory "+ "or provide a GraphRepositoryProtocol implementation."+ )+ self._sketch_id = sketch_id+ self._repository = repository+ self._logger = logger+ self._enable_batching = enable_batching++ @property+ def sketch_id(self) -> str:+ """Get the sketch ID."""+ return self._sketch_id++ @property+ def repository(self) -> GraphRepositoryProtocol:+ """Get the underlying repository."""+ return self._repository++ def create_node(self, node_obj: GraphNode) -> str | None:+ """+ Create or update a node in the graph.++ Supports one signatures:+ - GraphNode object: create_node(obj)++ Args:+ node_obj: a GraphNode object+ """++ if isinstance(node_obj, FlowsintType):+ raise Exception(+ "create_node method takes a GraphNode as input. If you want to insert a node from a FlowsintType, please use create_node_from_flowsint_type method."+ )++ neo4j_node_dict: Neo4jDict = GraphSerializer.graph_node_to_neo4j_dict(node_obj)++ if self._enable_batching:+ self._repository.add_to_batch(+ "node",+ node_obj=neo4j_node_dict,+ sketch_id=self._sketch_id,+ )+ else:+ return self._repository.create_node(+ node_obj=neo4j_node_dict,+ sketch_id=self._sketch_id,+ )++ def create_node_from_flowsint_type(self, node_obj: FlowsintType) -> str | None:+ """+ Create or update a node in the graph.++ Supports one signatures:+ - FlowsintType object: create_node(obj)++ Args:+ node_obj: a FlowsintType object+ """++ if isinstance(node_obj, GraphNode):+ raise Exception(+ "create_node_from_flowsint_type method takes a FlowsintType as input. If you want to insert a node from a GraphNode, please use create_node method."+ )++ neo4j_node_dict: Neo4jDict = GraphSerializer.flowsint_type_to_neo4j_dict(+ node_obj+ )++ if self._enable_batching:+ self._repository.add_to_batch(+ "node",+ node_obj=neo4j_node_dict,+ sketch_id=self._sketch_id,+ )+ else:+ return self._repository.create_node(+ node_obj=neo4j_node_dict,+ sketch_id=self._sketch_id,+ )++ def get_sketch_graph(self) -> GraphData:+ graph_data = self.repository.get_sketch_graph(self.sketch_id)+ nodes = GraphSerializer.deserialize_nodes(graph_data.get("nodes", []))+ edges = GraphSerializer.deserialize_edges(graph_data.get("edges", []))+ return GraphData(nodes=nodes, edges=edges)++ def get_nodes_by_ids(self, node_ids: List[str]) -> List[GraphNode]:+ nodes = self.repository.get_nodes_by_ids(node_ids, self.sketch_id)+ return GraphSerializer.deserialize_nodes(nodes)++ def get_nodes_by_ids_for_task(self, node_ids: List[str]) -> List[BaseModel]:+ nodes = self.get_nodes_by_ids(node_ids)+ return [GraphSerializer.graph_node_to_flowsint_type(node) for node in nodes]++ def create_relationship(+ self,+ from_obj: BaseModel,+ to_obj: BaseModel,+ rel_label: str = "IS_RELATED_TO",+ ) -> None:+ """+ Create a relationship between two nodes.++ Supports 1 signature:+ - Pydantic objects: create_relationship(obj1, obj2, "rel_label")++ Args:+ from_obj: A GraphNode object (source)+ to_obj: A GraphNode object (target)+ rel_label: Relationship label (ex: "IS_CONNECTED_TO")+ **properties: Additional relationship properties+ """++ neo4j_rel_dict: Neo4jDict = GraphSerializer.graph_edge_to_neo4j_dict(+ from_obj, to_obj, rel_label+ )++ if self._enable_batching:+ self._repository.add_to_batch(+ "relationship",+ rel_obj=neo4j_rel_dict,+ sketch_id=self._sketch_id,+ )+ else:+ self._repository.create_relationship(+ rel_obj=neo4j_rel_dict,+ sketch_id=self._sketch_id,+ )++ def create_relationship_by_element_id(+ self,+ from_element_id: str,+ to_element_id: str,+ rel_label: str = "IS_RELATED_TO",+ ):+ return self._repository.create_relationship_by_element_id(+ from_element_id=from_element_id,+ to_element_id=to_element_id,+ rel_label=rel_label,+ sketch_id=self._sketch_id,+ )++ def get_neighbors(self, node_id: str) -> GraphData:+ graph_data = self._repository.get_neighbors(+ node_id=node_id,+ sketch_id=self._sketch_id,+ )+ nodes = GraphSerializer.deserialize_nodes(graph_data.get("nodes", []))+ edges = GraphSerializer.deserialize_edges(graph_data.get("edges", []))+ return GraphData(nodes=nodes, edges=edges)++ def update_node(self, element_id: str, updates: Dict[str, Any]) -> str | None:+ flatten_updates = GraphSerializer.flatten(updates)+ """Update a node by its element ID."""+ return self._repository.update_node(+ element_id=element_id,+ updates=flatten_updates,+ sketch_id=self._sketch_id,+ )++ def update_nodes_positions(self, positions: List[Dict[str, Any]]) -> int:+ """Update positions (x, y) for multiple nodes in batch."""+ return self._repository.update_nodes_positions(+ positions=positions,+ sketch_id=self._sketch_id,+ )++ def delete_nodes(self, node_ids: List[str]) -> int:+ """Delete nodes by their element IDs."""+ return self._repository.delete_nodes(+ node_ids=node_ids,+ sketch_id=self._sketch_id,+ )++ def delete_relationships(self, relationship_ids: List[str]) -> int:+ """Delete relationships by their element IDs."""+ return self._repository.delete_relationships(+ relationship_ids=relationship_ids,+ sketch_id=self._sketch_id,+ )++ def delete_all_sketch_nodes(self) -> int:+ """Delete all nodes and relationships for the sketch."""+ return self._repository.delete_all_sketch_nodes(+ sketch_id=self._sketch_id,+ )++ def update_relationship(+ self, element_id: str, properties: Dict[str, Any]+ ) -> Dict[str, Any] | None:+ """Update a relationship by its element ID."""+ return self._repository.update_relationship(+ element_id=element_id,+ rel_obj=properties,+ sketch_id=self._sketch_id,+ )++ def merge_nodes(+ self,+ old_node_ids: List[str],+ new_node_data: Dict[str, Any],+ new_node_id: str | None = None,+ ) -> str | None:+ """Merge multiple nodes into one, transferring all relationships."""+ return self._repository.merge_nodes(+ old_node_ids=old_node_ids,+ new_node_data=new_node_data,+ new_node_id=new_node_id,+ sketch_id=self._sketch_id,+ )++ def batch_create_nodes(self, nodes: List[Neo4jDict]) -> Dict[str, Any]:+ """Create multiple nodes in a single batch transaction."""+ return self._repository.batch_create_nodes(+ nodes=nodes,+ sketch_id=self._sketch_id,+ )++ def batch_create_edges_by_element_id(+ self, edges: List[Neo4jDict]+ ) -> Dict[str, Any]:+ """Create multiple edges using element IDs in a single batch transaction."""+ return self._repository.batch_create_edges_by_element_id(+ edges=edges,+ sketch_id=self._sketch_id,+ )++ def log_graph_message(self, message: str) -> None:+ """+ Log a graph operation message.++ Args:+ message: Message to log+ """+ if self._logger:+ self._logger.graph_append(self._sketch_id, {"message": message})++ def flush(self) -> None:+ """Flush any pending batch operations."""+ if self._enable_batching:+ self._repository.flush_batch()++ def query(self, cypher: str, parameters: Dict[str, Any] = None) -> list:+ """+ Execute a custom Cypher query.++ Args:+ cypher: Cypher query string+ parameters: Query parameters++ Returns:+ List of result records+ """+ return self._repository.query(cypher, parameters)++ def set_batch_size(self, size: int) -> None:+ """+ Set the batch size for operations.++ Args:+ size: Number of operations to batch+ """+ self._repository.set_batch_size(size)++ def __enter__(self):+ """Context manager entry."""+ return self++ def __exit__(self, exc_type, exc_val, exc_tb):+ """Context manager exit - auto-flush batch."""+ if exc_type is None:+ self.flush()+++def create_graph_service(+ sketch_id: str,+ enable_batching: bool = True,+) -> GraphService:+ """+ Factory function to create a GraphService instance with Neo4j repository.++ This is the recommended way to create a GraphService for production use.+ For testing, inject an InMemoryGraphRepository or mock directly into GraphService.++ Args:+ sketch_id: Investigation sketch ID+ enable_batching: Enable batch operations++ Returns:+ Configured GraphService instance+ """+ # Import Logger here to avoid circular imports+ from flowsint_core.core.logger import Logger++ # Neo4jGraphRepository uses Neo4jConnection.get_instance() singleton+ repository = Neo4jGraphRepository()++ return GraphService(+ sketch_id=sketch_id,+ repository=repository,+ logger=Logger,+ enable_batching=enable_batching,+ )diff --git a/flowsint-core/src/flowsint_core/core/graph/types.py b/flowsint-core/src/flowsint_core/core/graph/types.pynew file mode 100644index 00000000..f6ed4fb6--- /dev/null+++ b/flowsint-core/src/flowsint_core/core/graph/types.py@@ -0,0 +1,59 @@+"""+Graph-related type definitions.++This module contains Pydantic models for graph nodes, edges, and related data structures.+"""++from datetime import datetime+from typing import Any, Dict, List, Optional, Union++from flowsint_types import FlowsintType+from pydantic import BaseModel, Field++Neo4jDict = Dict[str, Any]+++class NodeMetadata(BaseModel):+ created_at: datetime = Field(default_factory=datetime.now)++ class Config:+ extra = "allow"+++class GraphNode(BaseModel):+ """+ Represents object that's being manipulated in the frontend, and throughout the app.+ It represents a complete node object, close to what is stored in the neo4j db.+ """++ id: Optional[str]+ nodeLabel: str+ nodeType: str+ nodeSize: Optional[int] = None+ nodeColor: Optional[str] = None+ nodeIcon: Optional[str] = None+ nodeImage: Optional[str] = None+ nodeFlag: Optional[str] = None++ nodeMetadata: NodeMetadata+ nodeProperties: FlowsintType++ x: Optional[float] = 100.0+ y: Optional[float] = 100.0+++class GraphEdge(BaseModel):+ id: str+ source: str+ target: str+ label: str+ date: Optional[str] = None+ caption: Optional[str] = None+ type: Optional[str] = None+ weight: Optional[float] = None+ confidence_level: Optional[Union[float, str]] = None+++class GraphData(BaseModel):+ nodes: List[GraphNode]+ edges: List[GraphEdge]diff --git a/flowsint-core/src/flowsint_core/core/graph_serializer.py b/flowsint-core/src/flowsint_core/core/graph_serializer.pydeleted file mode 100644index 1546286f..00000000--- a/flowsint-core/src/flowsint_core/core/graph_serializer.py+++ /dev/null@@ -1,177 +0,0 @@-"""-Graph property serialization utilities.--This module provides utilities for serializing complex Python objects-into Neo4j-compatible primitive types, following the Single Responsibility Principle.-"""--from typing import Any, Dict, List---class GraphSerializer:- """- Handles serialization of complex objects to Neo4j-compatible types.-- This class is responsible for converting Pydantic models, nested objects,- and other complex types into primitive types that can be stored in Neo4j.- """-- @staticmethod- def serialize_properties(properties: Dict[str, Any]) -> Dict[str, Any]:- """- Convert properties to Neo4j-compatible values.-- Handles:- - Pydantic models (flattened into individual properties)- - Nested objects (flattened with prefixed keys)- - Lists (converted to primitive types)- - Dictionaries (flattened with prefixed keys)- - None values (skipped)-- Args:- properties: Dictionary of properties to serialize-- Returns:- Dictionary of Neo4j-compatible properties- """- serialized = {}-- for key, value in properties.items():- if key is None:- continue- if value is None:- serialized[key] = ""- elif GraphSerializer._is_pydantic_model(value):- # Flatten Pydantic models- flattened = GraphSerializer._flatten_pydantic(key, value)- serialized.update(flattened)- elif isinstance(value, dict):- # Flatten dictionaries- flattened = GraphSerializer._flatten_dict(key, value)- serialized.update(flattened)- elif isinstance(value, list):- # Handle lists- serialized[key] = GraphSerializer._serialize_list(value)- elif isinstance(value, (str, int, float, bool)):- # Keep primitive types as-is- serialized[key] = "" if value == "None" else value- else:- # Convert other complex types to strings- serialized[key] = str(value) if value != "None" else ""-- return serialized-- @staticmethod- def _is_pydantic_model(obj: Any) -> bool:- """Check if an object is a Pydantic model."""- return (- hasattr(obj, "__dict__")- and not isinstance(obj, (str, int, float, bool))- and (hasattr(obj, "model_dump") or hasattr(obj, "dict"))- )-- @staticmethod- def _flatten_pydantic(prefix: str, model: Any) -> Dict[str, Any]:- """- Flatten a Pydantic model into individual properties.-- Args:- prefix: Key prefix for nested properties- model: Pydantic model instance-- Returns:- Flattened dictionary- """- flattened = {}-- # Try Pydantic v2 first, then v1- if hasattr(model, "model_dump"):- data = model.model_dump(mode="json")- elif hasattr(model, "dict"):- data = model.dict()- else:- # Fallback to __dict__- data = {k: v for k, v in model.__dict__.items() if k is not None}-- for nested_key, nested_value in data.items():- if nested_key is None:- continue- new_key = f"{prefix}_{nested_key}"- if nested_value is None:- flattened[new_key] = ""- elif isinstance(nested_value, (str, int, float, bool)):- flattened[new_key] = nested_value- else:- # Recursively handle nested complex types- flattened[new_key] = str(nested_value)-- return flattened-- @staticmethod- def _flatten_dict(prefix: str, data: Dict[str, Any]) -> Dict[str, Any]:- """- Flatten a dictionary into individual properties.-- Args:- prefix: Key prefix for nested properties- data: Dictionary to flatten-- Returns:- Flattened dictionary- """- flattened = {}-- for dict_key, dict_value in data.items():- if dict_key is None:- continue- new_key = f"{prefix}_{dict_key}"- if dict_value is None:- flattened[new_key] = ""- elif isinstance(dict_value, (str, int, float, bool)):- flattened[new_key] = dict_value- else:- flattened[new_key] = str(dict_value)-- return flattened-- @staticmethod- def _serialize_list(items: List[Any]) -> List[Any]:- """- Serialize a list to Neo4j-compatible types.-- Args:- items: List of items to serialize-- Returns:- List of serialized items- """- serialized = []-- for item in items:- if isinstance(item, (str, int, float, bool)):- serialized.append(item)- elif GraphSerializer._is_pydantic_model(item):- # Convert complex objects to strings- serialized.append(str(item))- else:- serialized.append(str(item))-- return serialized-- @staticmethod- def normalize_property_value(value: Any) -> Any:- """- Normalize a single property value for Neo4j.-- Args:- value: Value to normalize-- Returns:- Normalized value- """- if value is None:- return ""- elif isinstance(value, (str, int, float, bool)):- return "" if value == "None" else value- else:- return str(value)diff --git a/flowsint-core/src/flowsint_core/core/graph_service.py b/flowsint-core/src/flowsint_core/core/graph_service.pydeleted file mode 100644index 7ba2c8be..00000000--- a/flowsint-core/src/flowsint_core/core/graph_service.py+++ /dev/null@@ -1,198 +0,0 @@-"""-Graph service for high-level graph operations.--This module provides a service layer for graph operations,-integrating repository and logging functionality.-"""--from typing import Any, Dict, Optional, Protocol, Union-from uuid import UUID--from pydantic import BaseModel--from .graph_db import Neo4jConnection-from .graph_repository import GraphRepository---class LoggerProtocol(Protocol):- """Protocol for logger implementations."""-- @staticmethod- def graph_append(sketch_id: str, message: Dict[str, Any]) -> None:- """Log a graph append message."""- ...---class GraphService:- """- High-level service for graph operations.-- This service provides a clean interface for enricher operations,- handling both graph persistence and logging with proper separation of concerns.- """-- def __init__(- self,- sketch_id: str,- neo4j_connection: Optional[Neo4jConnection] = None,- logger: Optional[LoggerProtocol] = None,- enable_batching: bool = True,- ):- """- Initialize the graph service.-- Args:- sketch_id: Investigation sketch ID- neo4j_connection: Optional Neo4j connection- logger: Optional logger instance- enable_batching: Enable batch operations- """- self._sketch_id = sketch_id- self._repository = GraphRepository(neo4j_connection)- self._logger = logger- self._enable_batching = enable_batching-- @property- def sketch_id(self) -> str:- """Get the sketch ID."""- return self._sketch_id-- @property- def repository(self) -> GraphRepository:- """Get the underlying repository."""- return self._repository-- def create_node(self, node_obj: BaseModel, **properties: Any) -> None:- """- Create or update a node in the graph.-- Supports one signatures:- - Pydantic object: create_node(obj, **overrides)-- Args:- from_obj: a Pydantic object- **properties: Additional node properties or overrides- """- if self._enable_batching:- self._repository.add_to_batch(- "node",- node_obj=node_obj,- sketch_id=self._sketch_id,- **properties,- )- else:- self._repository.create_node(- node_obj=node_obj,- sketch_id=self._sketch_id,- **properties,- )-- def create_relationship(- self,- from_obj: BaseModel,- to_obj: BaseModel,- rel_label: Optional[str] = None,- **properties: Any,- ) -> None:- """- Create a relationship between two nodes.-- Supports 1 signature:- - Pydantic objects: create_relationship(obj1, obj2, "REL_TYPE")-- Args:- from_obj: Either a Pydantic object (source)- to_obj: Either a Pydantic object (target)- rel_label: Relationship label (ex: "IS_CONNECTED_TO")- **properties: Additional relationship properties- """- if self._enable_batching:- self._repository.add_to_batch(- "relationship",- from_obj=from_obj,- to_obj=to_obj,- rel_label=rel_label,- sketch_id=self._sketch_id,- **properties,- )- else:- self._repository.create_relationship(- from_obj=from_obj,- to_obj=to_obj,- rel_label=rel_label,- sketch_id=self._sketch_id,- **properties,- )-- def log_graph_message(self, message: str) -> None:- """- Log a graph operation message.-- Args:- message: Message to log- """- if self._logger:- self._logger.graph_append(self._sketch_id, {"message": message})-- def flush(self) -> None:- """Flush any pending batch operations."""- if self._enable_batching:- self._repository.flush_batch()-- def query(self, cypher: str, parameters: Dict[str, Any] = None) -> list:- """- Execute a custom Cypher query.-- Args:- cypher: Cypher query string- parameters: Query parameters-- Returns:- List of result records- """- return self._repository.query(cypher, parameters)-- def set_batch_size(self, size: int) -> None:- """- Set the batch size for operations.-- Args:- size: Number of operations to batch- """- self._repository.set_batch_size(size)-- def __enter__(self):- """Context manager entry."""- return self-- def __exit__(self, exc_type, exc_val, exc_tb):- """Context manager exit - auto-flush batch."""- if exc_type is None:- self.flush()---def create_graph_service(- sketch_id: str,- neo4j_connection: Optional[Neo4jConnection] = None,- enable_batching: bool = True,-) -> GraphService:- """- Factory function to create a GraphService instance.-- Args:- sketch_id: Investigation sketch ID- neo4j_connection: Optional Neo4j connection- enable_batching: Enable batch operations-- Returns:- Configured GraphService instance- """- # Import Logger here to avoid circular imports- from .logger import Logger-- return GraphService(- sketch_id=sketch_id,- neo4j_connection=neo4j_connection,- logger=Logger,- enable_batching=enable_batching,- )diff --git a/flowsint-core/src/flowsint_core/core/logger.py b/flowsint-core/src/flowsint_core/core/logger.pyindex 7cb48e79..21a7a286 100644--- a/flowsint-core/src/flowsint_core/core/logger.py+++ b/flowsint-core/src/flowsint_core/core/logger.py@@ -8,17 +8,19 @@ Architecture:- SOLID principles: Dependency injection via protocols- Ordering: Monotonic sequence number + application timestamp"""-from typing import Dict, Union, Optional, Tuple-from uuid import UUID++import atexitimport threadingimport time-import atexit-from queue import Queuefrom datetime import datetime, timezone+from queue import Queue+from typing import Dict, Optional, Union+from uuid import UUID++from ..tasks.event import emit_event_taskfrom .enums import EventLevelfrom .models import Logfrom .postgre_db import get_db-from ..tasks.event import emit_event_taskclass LoggerSingleton:@@ -32,14 +34,11 @@ class LoggerSingleton:- Automatic flush on shutdown"""- _instance: Optional['LoggerSingleton'] = None+ _instance: Optional["LoggerSingleton"] = None_lock = threading.Lock()def __new__(- cls,- batch_size: int = 50,- flush_interval: float = 2.0,- auto_start: bool = True+ cls, batch_size: int = 50, flush_interval: float = 2.0, auto_start: bool = True):"""Thread-safe singleton implementation using double-checked locking."""if cls._instance is None:@@ -51,10 +50,7 @@ class LoggerSingleton:return cls._instancedef __init__(- self,- batch_size: int = 50,- flush_interval: float = 2.0,- auto_start: bool = True+ self, batch_size: int = 50, flush_interval: float = 2.0, auto_start: bool = True):"""Initialize the Logger singleton.@@ -100,9 +96,7 @@ class LoggerSingleton:if self._worker_thread is None or not self._worker_thread.is_alive():self._shutdown_event.clear()self._worker_thread = threading.Thread(- target=self._batch_worker,- daemon=True,- name="LoggerBatchWorker"+ target=self._batch_worker, daemon=True, name="LoggerBatchWorker")self._worker_thread.start()@@ -133,7 +127,9 @@ class LoggerSingleton:logs_to_insert = []# Collect logs from queue- while not self._log_queue.empty() and (force or len(logs_to_insert) < self._batch_size):+ while not self._log_queue.empty() and (+ force or len(logs_to_insert) < self._batch_size+ ):try:logs_to_insert.append(self._log_queue.get_nowait())except:@@ -151,6 +147,7 @@ class LoggerSingleton:except (StopIteration, RuntimeError) as e:# Generator exhausted or error getting DB - log to standard loggingimport logging+logging.error(f"Failed to get database session: {e}")return@@ -161,7 +158,7 @@ class LoggerSingleton:sketch_id=str(sketch_id),type=level.value,content=content,- created_at=timestamp # Use application-side timestamp+ created_at=timestamp, # Use application-side timestamp)log_objects.append(log)@@ -176,16 +173,13 @@ class LoggerSingleton:db.rollback()# Log to standard logging as fallbackimport logging+logging.error(f"Failed to batch insert logs: {e}")finally:db.close()def _emit_event(- self,- log_id: str,- sketch_id: str,- level: EventLevel,- content: Dict+ self, log_id: str, sketch_id: str, level: EventLevel, content: Dict) -> None:"""Emit event immediately for real-time display.@@ -197,19 +191,15 @@ class LoggerSingleton:content: Log content"""try:- emit_event_task.apply(- args=[log_id, str(sketch_id), level, content]- )+ emit_event_task.apply(args=[log_id, str(sketch_id), level, content])except Exception as e:# Don't let event emission errors break loggingimport logging+logging.error(f"Failed to emit event: {e}")def _log(- self,- sketch_id: Union[str, UUID],- level: EventLevel,- content: Dict+ self, sketch_id: Union[str, UUID], level: EventLevel, content: Dict) -> None:"""Internal logging method.@@ -230,6 +220,7 @@ class LoggerSingleton:# Generate a temporary ID for immediate event emissionimport uuid+temp_log_id = str(uuid.uuid4())# 1. IMMEDIATE: Emit event for real-time display@@ -272,13 +263,16 @@ class LoggerSingleton:# Also publish to status channel for graph refreshtry:import uuid+temp_log_id = str(uuid.uuid4())from ..tasks.event import emit_status_event_task+emit_status_event_task.apply(args=[temp_log_id, str(sketch_id), EventLevel.COMPLETED, message])except Exception as e:import logging+logging.error(f"Failed to emit status event: {e}")def pending(self, sketch_id: Union[str, UUID], message: Dict) -> None:diff --git a/flowsint-core/src/flowsint_core/core/orchestrator.py b/flowsint-core/src/flowsint_core/core/orchestrator.pyindex 1b9d825a..8c55fcc9 100644--- a/flowsint-core/src/flowsint_core/core/orchestrator.py+++ b/flowsint-core/src/flowsint_core/core/orchestrator.py@@ -22,10 +22,9 @@ class FlowOrchestrator(Enricher):sketch_id: str,scan_id: str,enricher_branches: List[FlowBranch],- neo4j_conn=None,vault=None,):- super().__init__(sketch_id, scan_id, neo4j_conn=neo4j_conn, vault=vault)+ super().__init__(sketch_id, scan_id, vault=vault)self.enricher_branches = enricher_branchesself.enrichers = {} # Map of nodeId -> enricher instanceself.execution_log_file = None # Path to the execution log file@@ -243,7 +242,6 @@ class FlowOrchestrator(Enricher):enricher_name,self.sketch_id,self.scan_id,- neo4j_conn=self.neo4j_conn,vault=self.vault,params=enricher_params,)diff --git a/flowsint-core/src/flowsint_core/core/types.py b/flowsint-core/src/flowsint_core/core/types.pyindex b9d109e6..2c00ebd1 100644--- a/flowsint-core/src/flowsint_core/core/types.py+++ b/flowsint-core/src/flowsint_core/core/types.py@@ -1,8 +1,16 @@+"""+Core type definitions.++This module contains Pydantic models for core types like events, flows, and roles.+Graph-related types are in flowsint_core.core.graph.types.+"""+import enum-from pydantic import BaseModel, Field-from typing import Dict, Anyfrom datetime import datetime-from typing import Dict, List, Any, Optional, Literal+from typing import Any, Dict, List, Literal, Optional++from pydantic import BaseModel, Field+from .enums import EventLevel@@ -16,7 +24,7 @@ class Event(BaseModel):)-class Node(BaseModel):+class FlowNode(BaseModel):"""Represents a node in a transformation flow with position and data."""id: str = Field(..., description="Unique identifier for the node", title="Node ID")@@ -33,7 +41,7 @@ class Node(BaseModel):)-class Edge(BaseModel):+class FlowEdge(BaseModel):"""Represents an edge connecting two nodes in a transformation flow."""id: str = Field(..., description="Unique identifier for the edge", title="Edge ID")diff --git a/flowsint-core/src/flowsint_core/imports/__init__.py b/flowsint-core/src/flowsint_core/imports/__init__.pyindex 02dfc77e..748f542f 100644--- a/flowsint-core/src/flowsint_core/imports/__init__.py+++ b/flowsint-core/src/flowsint_core/imports/__init__.py@@ -4,6 +4,12 @@ Import utilities for entity parsing and type detection.from .entity_detection import detect_typefrom .file_parser import FileParseResult, parse_import_file+from .import_service import (+ EntityMapping,+ ImportResult,+ ImportService,+ create_import_service,+)from .types import EntityPreview__all__ = [@@ -11,4 +17,9 @@ __all__ = ["parse_import_file","FileParseResult","EntityPreview",+ # Import service+ "EntityMapping",+ "ImportResult",+ "ImportService",+ "create_import_service",]diff --git a/flowsint-core/src/flowsint_core/imports/import_service.py b/flowsint-core/src/flowsint_core/imports/import_service.pynew file mode 100644index 00000000..3229956c--- /dev/null+++ b/flowsint-core/src/flowsint_core/imports/import_service.py@@ -0,0 +1,248 @@+"""+Import service for handling file imports into sketches.++This module provides a service layer for import operations,+handling file parsing, entity conversion, and batch creation.+"""++from dataclasses import dataclass+from typing import Any, Dict, List, Optional++from flowsint_types import FlowsintType++from flowsint_core.core.graph import GraphSerializer, GraphService+++@dataclass+class EntityMapping:+ """Mapping configuration for an entity."""++ id: str+ entity_type: str+ nodeLabel: str+ data: Dict[str, Any]+ include: bool = True+ node_id: Optional[str] = None+++@dataclass+class ImportResult:+ """Result of an import execution."""++ status: str+ nodes_created: int+ nodes_skipped: int+ errors: List[str]+++class ImportService:+ """+ Service for handling file imports into sketches.++ This service handles:+ - File analysis and parsing+ - Entity conversion to FlowsintTypes+ - Batch node and edge creation+ """++ def __init__(self, graph_service: GraphService):+ """+ Initialize the import service.++ Args:+ graph_service: GraphService instance for database operations+ """+ self._graph_service = graph_service++ @staticmethod+ def analyze_file(+ file_content: bytes,+ filename: str,+ max_preview_rows: int = 10000000,+ ):+ """+ Analyze an uploaded file for import.++ Args:+ file_content: Raw file content as bytes+ filename: Name of the file (used for extension detection)+ max_preview_rows: Maximum number of rows to preview++ Returns:+ FileParseResult with detected entities and edges++ Raises:+ ValueError: If file format is unsupported or parsing fails+ """+ from flowsint_core.imports import parse_import_file++ return parse_import_file(+ file_content=file_content,+ filename=filename,+ max_preview_rows=max_preview_rows,+ )++ def execute_import(+ self,+ entity_mappings: List[EntityMapping],+ edges: Optional[List[Dict[str, Any]]] = None,+ ) -> ImportResult:+ """+ Execute the import of entities into the sketch.++ Args:+ entity_mappings: List of EntityMapping objects to import+ edges: Optional list of edge definitions with from_id, to_id, label++ Returns:+ ImportResult with status, counts, and any errors+ """+ # Filter only entities marked for inclusion+ entities_to_import = [m for m in entity_mappings if m.include]++ # Convert entity mappings to FlowsintType objects+ conversion_result = self._convert_entities(entities_to_import)+ pydantic_nodes = conversion_result["nodes"]+ nodes_mapping_indices = conversion_result["mapping_indices"]+ conversion_errors = conversion_result["errors"]++ if not pydantic_nodes:+ return ImportResult(+ status="completed_with_errors" if conversion_errors else "completed",+ nodes_created=0,+ nodes_skipped=len(entities_to_import),+ errors=conversion_errors[:50],+ )++ # Batch create nodes+ try:+ nodes = GraphSerializer.serialize_flowsint_types(pydantic_nodes)+ nodes_result = self._graph_service.batch_create_nodes(nodes=nodes)+ nodes_created = nodes_result["nodes_created"]+ node_element_ids = nodes_result.get("node_ids", [])+ batch_errors = nodes_result.get("errors", [])+ except Exception as e:+ return ImportResult(+ status="failed",+ nodes_created=0,+ nodes_skipped=len(entities_to_import),+ errors=[f"Batch node creation failed: {str(e)}"],+ )++ # Create edges if provided+ edge_errors = []+ if edges and nodes_mapping_indices and node_element_ids:+ edge_errors = self._create_edges(+ edges=edges,+ nodes_mapping_indices=nodes_mapping_indices,+ node_element_ids=node_element_ids,+ )++ all_errors = conversion_errors + batch_errors + edge_errors+ nodes_skipped = len(entities_to_import) - nodes_created++ return ImportResult(+ status="completed" if not all_errors else "completed_with_errors",+ nodes_created=nodes_created,+ nodes_skipped=nodes_skipped,+ errors=all_errors[:50],+ )++ def _convert_entities(self, entities: List[EntityMapping]) -> Dict[str, Any]:+ """+ Convert entity mappings to FlowsintType objects.++ Returns:+ Dict with nodes, mapping_indices, and errors+ """+ pydantic_nodes: List[FlowsintType] = []+ nodes_mapping_indices: Dict[str, int] = {}+ errors: List[str] = []++ for idx, mapping in enumerate(entities):+ entity_data = mapping.data.copy()++ try:+ pydantic_obj = GraphSerializer.parse_flowsint_type(+ entity=entity_data,+ nodeType=mapping.entity_type,+ )+ pydantic_nodes.append(pydantic_obj)++ if mapping.node_id:+ nodes_mapping_indices[mapping.node_id] = len(pydantic_nodes) - 1++ except ValueError as e:+ errors.append(f"Entity {idx + 1} ({mapping.nodeLabel}): {str(e)}")+ except Exception as e:+ errors.append(f"Entity {idx + 1} ({mapping.nodeLabel}): {str(e)}")++ return {+ "nodes": pydantic_nodes,+ "mapping_indices": nodes_mapping_indices,+ "errors": errors,+ }++ def _create_edges(+ self,+ edges: List[Dict[str, Any]],+ nodes_mapping_indices: Dict[str, int],+ node_element_ids: List[str],+ ) -> List[str]:+ """+ Create edges between imported nodes.++ Returns:+ List of error messages+ """+ errors: List[str] = []+ edges_to_insert: List[Dict[str, Any]] = []++ for idx, edge in enumerate(edges):+ from_id = edge.get("from_id")+ to_id = edge.get("to_id")++ from_idx = nodes_mapping_indices.get(from_id)+ to_idx = nodes_mapping_indices.get(to_id)++ if from_idx is None or to_idx is None:+ errors.append(+ f"Edge {idx}: Missing source or target node (from: {from_id}, to: {to_id})"+ )+ continue++ if from_idx >= len(node_element_ids) or to_idx >= len(node_element_ids):+ errors.append(f"Edge {idx}: Node index out of range")+ continue++ edges_to_insert.append(+ {+ "from_element_id": node_element_ids[from_idx],+ "to_element_id": node_element_ids[to_idx],+ "rel_label": edge.get("label", "RELATED_TO"),+ }+ )++ if edges_to_insert:+ try:+ edges_result = self._graph_service.batch_create_edges_by_element_id(+ edges=edges_to_insert+ )+ errors.extend(edges_result.get("errors", []))+ except Exception as e:+ errors.append(f"Batch edge creation failed: {str(e)}")++ return errors+++def create_import_service(graph_service: GraphService) -> ImportService:+ """+ Factory function to create an ImportService instance.++ Args:+ graph_service: GraphService instance for database operations++ Returns:+ Configured ImportService instance+ """+ return ImportService(graph_service=graph_service)diff --git a/flowsint-core/src/flowsint_core/tasks/enricher.py b/flowsint-core/src/flowsint_core/tasks/enricher.pyindex 81f0144f..fa4471e6 100644--- a/flowsint-core/src/flowsint_core/tasks/enricher.py+++ b/flowsint-core/src/flowsint_core/tasks/enricher.py@@ -1,13 +1,10 @@-import osimport uuidimport asyncio-from dotenv import load_dotenvfrom typing import List, Optionalfrom celery import statesfrom flowsint_enrichers import ENRICHER_REGISTRY, load_all_enrichersfrom ..core.celery import celeryfrom ..core.postgre_db import SessionLocal, get_db-from ..core.graph_db import Neo4jConnectionfrom ..core.vault import Vaultfrom ..core.models import Scanfrom sqlalchemy.orm import Session@@ -15,16 +12,9 @@ from ..core.logger import Loggerfrom ..core.enums import EventLevelfrom flowsint_core.utils import to_json_serializable-load_dotenv()-# Auto-discover and register all enrichersload_all_enrichers()-URI = os.getenv("NEO4J_URI_BOLT")-USERNAME = os.getenv("NEO4J_USERNAME")-PASSWORD = os.getenv("NEO4J_PASSWORD")--neo4j_connection = Neo4jConnection(URI, USERNAME, PASSWORD)db: Session = next(get_db())@@ -67,7 +57,6 @@ def run_enricher(name=enricher_name,sketch_id=sketch_id,scan_id=scan_id,- neo4j_conn=neo4j_connection,vault=vault,)diff --git a/flowsint-core/src/flowsint_core/tasks/flow.py b/flowsint-core/src/flowsint_core/tasks/flow.pyindex 4b04b479..4ff2f897 100644--- a/flowsint-core/src/flowsint_core/tasks/flow.py+++ b/flowsint-core/src/flowsint_core/tasks/flow.py@@ -1,12 +1,9 @@-import osimport uuid-from dotenv import load_dotenvfrom typing import List, Optionalfrom celery import statesfrom ..core.celery import celeryfrom ..core.orchestrator import FlowOrchestratorfrom ..core.postgre_db import SessionLocal, get_db-from ..core.graph_db import Neo4jConnectionfrom ..core.vault import Vaultfrom ..core.types import FlowBranchfrom ..core.models import Scan@@ -15,13 +12,6 @@ from ..core.logger import Loggerfrom ..core.enums import EventLevelfrom flowsint_core.utils import to_json_serializable-load_dotenv()--URI = os.getenv("NEO4J_URI_BOLT")-USERNAME = os.getenv("NEO4J_USERNAME")-PASSWORD = os.getenv("NEO4J_PASSWORD")--neo4j_connection = Neo4jConnection(URI, USERNAME, PASSWORD)db: Session = next(get_db())@@ -64,7 +54,6 @@ def run_flow(sketch_id=sketch_id,scan_id=str(scan_id),enricher_branches=enricher_branches,- neo4j_conn=neo4j_connection,vault=vault,)diff --git a/flowsint-core/src/flowsint_core/utils.py b/flowsint-core/src/flowsint_core/utils.pyindex e94d883d..8733213d 100644--- a/flowsint-core/src/flowsint_core/utils.py+++ b/flowsint-core/src/flowsint_core/utils.py@@ -1,18 +1,17 @@-from urllib.parse import urlparse-import phonenumbers+import inspectimport ipaddress-from phonenumbers import NumberParseException-from pydantic import TypeAdapter, BaseModel-from urllib.parse import urlparseimport re-import sslimport socket-from typing import Dict, Any, List, Type-from pydantic import BaseModel-import inspect-from typing import Any, Dict, Type+import ssl+from typing import Any, Dict, List, Type+from urllib.parse import urlparse++import phonenumbers+from phonenumbers import NumberParseExceptionfrom pydantic import BaseModel, TypeAdapter+from .core.graph.types import GraphEdge, GraphNode+def is_valid_ip(address: str) -> bool:try:@@ -35,7 +34,6 @@ def is_valid_email(email: str) -> bool:def is_valid_domain(url_or_domain: str) -> str:-try:parsed = urlparse(url_or_domain if "://" in url_or_domain else "http://" + url_or_domain@@ -248,38 +246,73 @@ def get_label_color(label: str) -> str:return color_map.get(label, color_map["default"])-def flatten(data_dict, prefix=""):- """- Flattens a dictionary to contain only Neo4j-compatible property values.- Neo4j supports primitive types (string, number, boolean) and arrays of those types.- Args:- data_dict (dict): Dictionary to flatten- Returns:- dict: Flattened dictionary with only Neo4j-compatible values- """- flattened = {}- if not isinstance(data_dict, dict):- return flattened- for key, value in data_dict.items():- if value is None:- continue- if isinstance(value, (str, int, float, bool)) or (- isinstance(value, list)- and all(isinstance(item, (str, int, float, bool)) for item in value)- ):- key = f"{prefix}{key}"- flattened[key] = value+Primitive = (str, int, float, bool)+++def flatten(+ data: Any, prefix: str = "", *, remove_empty: bool = False, separator: str = "."+) -> Dict[str, Any]:+ flattened: Dict[str, Any] = {}++ if isinstance(data, dict):+ for key, value in data.items():+ new_key = f"{prefix}{key}" if prefix == "" else f"{prefix}{separator}{key}"++ # Handle None values+ if value is None:+ if not remove_empty:+ flattened[new_key] = None+ continue++ # Ignore empty strings if option enabled+ if remove_empty and value == "":+ continue++ if isinstance(value, Primitive):+ flattened[new_key] = value++ elif isinstance(value, list):+ if all(isinstance(item, Primitive) for item in value):+ if remove_empty:+ value = [item for item in value if item != ""]+ if not value:+ continue+ flattened[new_key] = value+ # else ignore (Neo4j incompatible)++ elif isinstance(value, dict):+ flattened.update(flatten(value, new_key, remove_empty=remove_empty, separator=separator))+return flattened-def get_inline_relationships(nodes: List[Any], edges: List[Any]) -> List[str]:+def unflatten(data: Dict[str, Any], *, separator: str = ".") -> Dict[str, Any]:+ result: Dict[str, Any] = {}++ for flat_key, value in data.items():+ parts = flat_key.split(separator)+ current = result++ for part in parts[:-1]:+ if part not in current or not isinstance(current[part], dict):+ current[part] = {}+ current = current[part]++ current[parts[-1]] = value++ return result+++def get_inline_relationships(+ nodes: List[GraphNode], edges: List[GraphEdge]+) -> List[str]:"""Get the inline relationships for a list of nodes and edges."""relationships = []for edge in edges:- source = next((node for node in nodes if node["id"] == edge["source"]), None)- target = next((node for node in nodes if node["id"] == edge["target"]), None)+ source = next((node for node in nodes if node.id == edge.source), None)+ target = next((node for node in nodes if node.id == edge.target), None)if source and target:relationships.append({"source": source, "edge": edge, "target": target})return relationships@@ -288,6 +321,7 @@ def get_inline_relationships(nodes: List[Any], edges: List[Any]) -> List[str]:def to_json_serializable(obj):"""Convert any object to a JSON-serializable format."""import json+from pydantic import BaseModeltry:diff --git a/flowsint-core/tests/core/__init__.py b/flowsint-core/tests/core/__init__.pynew file mode 100644index 00000000..da8377b0--- /dev/null+++ b/flowsint-core/tests/core/__init__.py@@ -0,0 +1 @@+# Tests for core modulediff --git a/flowsint-core/tests/core/graph/__init__.py b/flowsint-core/tests/core/graph/__init__.pynew file mode 100644index 00000000..6fcc4cbf--- /dev/null+++ b/flowsint-core/tests/core/graph/__init__.py@@ -0,0 +1 @@+# Tests for graph modulediff --git a/flowsint-core/tests/core/graph/in_memory_graph_repository.py b/flowsint-core/tests/core/graph/in_memory_graph_repository.pynew file mode 100644index 00000000..a681262f--- /dev/null+++ b/flowsint-core/tests/core/graph/in_memory_graph_repository.py@@ -0,0 +1,477 @@+"""+In-memory implementation of GraphRepositoryProtocol for testing.++This module provides a lightweight graph repository that stores data in memory,+enabling fast unit tests without requiring a Neo4j database connection.+"""++from datetime import datetime, timezone+from typing import Any, Dict, List, Optional+from uuid import uuid4+++class InMemoryGraphRepository:+ """+ In-memory implementation of GraphRepositoryProtocol for testing.++ Stores nodes and edges in dictionaries, no Neo4j required.+ All operations are synchronous and data is lost when the instance is destroyed.+ """++ def __init__(self):+ self._nodes: Dict[str, Dict[str, Any]] = {} # element_id -> node_data+ self._edges: Dict[str, Dict[str, Any]] = {} # element_id -> edge_data+ self._batch_operations: List[tuple] = []+ self._batch_size = 100++ def _generate_element_id(self, prefix: str = "mem") -> str:+ """Generate a unique element ID."""+ return f"{prefix}:{uuid4()}"++ # -------------------------------------------------------------------------+ # Core node operations+ # -------------------------------------------------------------------------++ def create_node(self, node_obj: Dict[str, Any], sketch_id: str) -> Optional[str]:+ """Create or update a single node. Returns element ID."""+ node_label = node_obj.get("nodeLabel")+ node_type = node_obj.get("nodeType")++ # Check if node already exists (MERGE behavior)+ for element_id, data in self._nodes.items():+ if (+ data.get("nodeLabel") == node_label+ and data.get("sketch_id") == sketch_id+ ):+ # Update existing node+ self._nodes[element_id].update(node_obj)+ return element_id++ # Create new node+ element_id = self._generate_element_id("node")+ self._nodes[element_id] = {+ **node_obj,+ "sketch_id": sketch_id,+ "created_at": datetime.now(timezone.utc).isoformat(),+ "_labels": [node_type] if node_type else ["Node"],+ }+ return element_id++ def update_node(+ self, element_id: str, updates: Dict[str, Any], sketch_id: str+ ) -> Optional[str]:+ """Update a node by its element ID. Returns element ID."""+ if element_id not in self._nodes:+ return None+ if self._nodes[element_id].get("sketch_id") != sketch_id:+ return None+ self._nodes[element_id].update(updates)+ return element_id++ def delete_nodes(self, node_ids: List[str], sketch_id: str) -> int:+ """Delete nodes by their element IDs. Returns count deleted."""+ deleted = 0+ for node_id in node_ids:+ if node_id in self._nodes:+ if self._nodes[node_id].get("sketch_id") == sketch_id:+ # Also delete related edges+ edges_to_delete = [+ eid+ for eid, edge in self._edges.items()+ if edge.get("source") == node_id+ or edge.get("target") == node_id+ ]+ for eid in edges_to_delete:+ del self._edges[eid]+ del self._nodes[node_id]+ deleted += 1+ return deleted++ def delete_all_sketch_nodes(self, sketch_id: str) -> int:+ """Delete all nodes for a sketch. Returns count deleted."""+ to_delete = [+ eid+ for eid, data in self._nodes.items()+ if data.get("sketch_id") == sketch_id+ ]+ for eid in to_delete:+ del self._nodes[eid]++ # Also delete related edges+ edges_to_delete = [+ eid+ for eid, data in self._edges.items()+ if data.get("sketch_id") == sketch_id+ ]+ for eid in edges_to_delete:+ del self._edges[eid]++ return len(to_delete)++ def get_nodes_by_ids(+ self, node_ids: List[str], sketch_id: str+ ) -> List[Dict[str, Any]]:+ """Get nodes by their element IDs."""+ result = []+ for node_id in node_ids:+ if node_id in self._nodes:+ node = self._nodes[node_id]+ if node.get("sketch_id") == sketch_id:+ result.append({"data": node})+ return result++ def update_nodes_positions(+ self, positions: List[Dict[str, Any]], sketch_id: str+ ) -> int:+ """Update positions for multiple nodes. Returns count updated."""+ updated = 0+ for pos in positions:+ node_id = pos.get("nodeId")+ if node_id in self._nodes:+ if self._nodes[node_id].get("sketch_id") == sketch_id:+ self._nodes[node_id]["x"] = pos.get("x")+ self._nodes[node_id]["y"] = pos.get("y")+ updated += 1+ return updated++ # -------------------------------------------------------------------------+ # Core relationship operations+ # -------------------------------------------------------------------------++ def create_relationship(self, rel_obj: Dict[str, Any], sketch_id: str) -> None:+ """Create a relationship between two nodes."""+ from_type = rel_obj.get("from_type")+ from_label = rel_obj.get("from_label")+ to_type = rel_obj.get("to_type")+ to_label = rel_obj.get("to_label")+ rel_label = rel_obj.get("rel_label", "RELATED_TO")++ # Find source and target nodes+ source_id = None+ target_id = None++ for eid, node in self._nodes.items():+ if node.get("sketch_id") != sketch_id:+ continue+ if node.get("nodeLabel") == from_label:+ source_id = eid+ if node.get("nodeLabel") == to_label:+ target_id = eid++ if source_id and target_id:+ element_id = self._generate_element_id("rel")+ self._edges[element_id] = {+ **rel_obj,+ "source": source_id,+ "target": target_id,+ "type": rel_label,+ "sketch_id": sketch_id,+ }++ def create_relationship_by_element_id(+ self,+ from_element_id: str,+ to_element_id: str,+ rel_label: str,+ sketch_id: str,+ ) -> Optional[Dict[str, Any]]:+ """Create a relationship using element IDs."""+ if from_element_id not in self._nodes or to_element_id not in self._nodes:+ return None++ element_id = self._generate_element_id("rel")+ edge_data = {+ "source": from_element_id,+ "target": to_element_id,+ "type": rel_label,+ "sketch_id": sketch_id,+ }+ self._edges[element_id] = edge_data+ return {"sketch_id": sketch_id}++ def update_relationship(+ self, element_id: str, rel_obj: Dict[str, Any], sketch_id: str+ ) -> Optional[Dict[str, Any]]:+ """Update a relationship by its element ID."""+ if element_id not in self._edges:+ return None+ if self._edges[element_id].get("sketch_id") != sketch_id:+ return None+ self._edges[element_id].update(rel_obj)+ return {+ "id": element_id,+ "type": self._edges[element_id].get("type"),+ "data": self._edges[element_id],+ }++ def delete_relationships(self, relationship_ids: List[str], sketch_id: str) -> int:+ """Delete relationships by their element IDs. Returns count deleted."""+ deleted = 0+ for rel_id in relationship_ids:+ if rel_id in self._edges:+ if self._edges[rel_id].get("sketch_id") == sketch_id:+ del self._edges[rel_id]+ deleted += 1+ return deleted++ # -------------------------------------------------------------------------+ # Graph queries+ # -------------------------------------------------------------------------++ def get_sketch_graph(+ self, sketch_id: str, limit: int = 100000+ ) -> Dict[str, List[Dict[str, Any]]]:+ """Get all nodes and edges for a sketch."""+ nodes = []+ node_ids = set()++ for eid, data in self._nodes.items():+ if data.get("sketch_id") == sketch_id:+ nodes.append(+ {+ "id": eid,+ "labels": data.get("_labels", ["Node"]),+ "data": data,+ }+ )+ node_ids.add(eid)+ if len(nodes) >= limit:+ break++ edges = []+ for eid, data in self._edges.items():+ if data.get("sketch_id") == sketch_id:+ if data.get("source") in node_ids and data.get("target") in node_ids:+ edges.append(+ {+ "id": eid,+ "type": data.get("type", "RELATED_TO"),+ "source": data.get("source"),+ "target": data.get("target"),+ "data": data,+ }+ )++ return {"nodes": nodes, "edges": edges}++ def get_neighbors(self, node_id: str, sketch_id: str) -> Dict[str, Any]:+ """Get a node and all its direct relationships."""+ if node_id not in self._nodes:+ return {"nodes": [], "edges": []}++ center = self._nodes[node_id]+ if center.get("sketch_id") != sketch_id:+ return {"nodes": [], "edges": []}++ nodes = {node_id: {"id": node_id, "data": center}}+ edges = {}++ for eid, edge in self._edges.items():+ if edge.get("sketch_id") != sketch_id:+ continue++ source = edge.get("source")+ target = edge.get("target")++ if source == node_id:+ # Outgoing edge+ if target in self._nodes:+ nodes[target] = {"id": target, "data": self._nodes[target]}+ edges[eid] = {+ "id": eid,+ "source": source,+ "target": target,+ "label": edge.get("type"),+ }+ elif target == node_id:+ # Incoming edge+ if source in self._nodes:+ nodes[source] = {"id": source, "data": self._nodes[source]}+ edges[eid] = {+ "id": eid,+ "source": source,+ "target": target,+ "label": edge.get("type"),+ }++ return {"nodes": list(nodes.values()), "edges": list(edges.values())}++ # -------------------------------------------------------------------------+ # Merge operations+ # -------------------------------------------------------------------------++ def merge_nodes(+ self,+ old_node_ids: List[str],+ new_node_data: Dict[str, Any],+ new_node_id: Optional[str],+ sketch_id: str,+ ) -> Optional[str]:+ """Merge multiple nodes into one. Returns new element ID."""+ if not old_node_ids:+ return None++ # Determine target node ID+ if new_node_id and new_node_id in old_node_ids:+ target_id = new_node_id+ self._nodes[target_id].update(new_node_data)+ else:+ target_id = self._generate_element_id("node")+ self._nodes[target_id] = {+ **new_node_data,+ "sketch_id": sketch_id,+ "created_at": datetime.now(timezone.utc).isoformat(),+ }++ # Transfer relationships+ for eid, edge in list(self._edges.items()):+ if edge.get("source") in old_node_ids and edge.get("source") != target_id:+ edge["source"] = target_id+ if edge.get("target") in old_node_ids and edge.get("target") != target_id:+ edge["target"] = target_id++ # Delete old nodes (except target)+ for node_id in old_node_ids:+ if node_id != target_id and node_id in self._nodes:+ del self._nodes[node_id]++ return target_id++ # -------------------------------------------------------------------------+ # Batch operations+ # -------------------------------------------------------------------------++ def batch_create_nodes(+ self, nodes: List[Dict[str, Any]], sketch_id: str+ ) -> Dict[str, Any]:+ """Create multiple nodes in a single batch."""+ node_ids = []+ errors = []++ for idx, node_obj in enumerate(nodes):+ try:+ element_id = self.create_node(node_obj, sketch_id)+ if element_id:+ node_ids.append(element_id)+ except Exception as e:+ errors.append(f"Node {idx}: {str(e)}")++ return {+ "nodes_created": len(node_ids),+ "node_ids": node_ids,+ "errors": errors,+ }++ def batch_create_edges_by_element_id(+ self, edges: List[Dict[str, Any]], sketch_id: str+ ) -> Dict[str, Any]:+ """Create multiple edges using element IDs in a single batch."""+ created = 0+ errors = []++ for idx, edge in enumerate(edges):+ try:+ from_id = edge.get("from_element_id")+ to_id = edge.get("to_element_id")+ rel_label = edge.get("rel_label", "RELATED_TO")++ if not from_id or not to_id:+ errors.append(+ f"Edge {idx}: Missing required fields (from_element_id or to_element_id)"+ )+ continue++ result = self.create_relationship_by_element_id(+ from_id, to_id, rel_label, sketch_id+ )+ if result:+ created += 1+ except Exception as e:+ errors.append(f"Edge {idx}: {str(e)}")++ return {"edges_created": created, "errors": errors}++ def add_to_batch(self, operation_type: str, **kwargs: Any) -> None:+ """Add an operation to the batch queue."""+ if operation_type not in ("node", "relationship"):+ raise ValueError(f"Unknown operation type: {operation_type}")++ self._batch_operations.append((operation_type, kwargs))++ if len(self._batch_operations) >= self._batch_size:+ self.flush_batch()++ def flush_batch(self) -> None:+ """Execute all batched operations."""+ for op_type, kwargs in self._batch_operations:+ if op_type == "node":+ self.create_node(kwargs["node_obj"], kwargs["sketch_id"])+ elif op_type == "relationship":+ self.create_relationship(kwargs["rel_obj"], kwargs["sketch_id"])+ self._batch_operations.clear()++ def clear_batch(self) -> None:+ """Clear the batch without executing."""+ self._batch_operations.clear()++ def set_batch_size(self, size: int) -> None:+ """Set the batch size for auto-flushing."""+ if size < 1:+ raise ValueError("Batch size must be at least 1")+ self._batch_size = size++ # -------------------------------------------------------------------------+ # Custom queries+ # -------------------------------------------------------------------------++ def query(+ self, cypher: str, parameters: Dict[str, Any] = {}+ ) -> List[Dict[str, Any]]:+ """+ Execute a custom Cypher query.++ Note: In-memory implementation doesn't support Cypher.+ Override this method or mock it for specific test cases.+ """+ return []++ # -------------------------------------------------------------------------+ # Context manager+ # -------------------------------------------------------------------------++ def __enter__(self):+ """Context manager entry."""+ return self++ def __exit__(self, exc_type, exc_val, exc_tb):+ """Context manager exit - auto-flush batch on success."""+ if exc_type is None:+ self.flush_batch()+ else:+ self.clear_batch()++ # -------------------------------------------------------------------------+ # Test helpers+ # -------------------------------------------------------------------------++ def get_node_count(self, sketch_id: Optional[str] = None) -> int:+ """Get total node count, optionally filtered by sketch_id."""+ if sketch_id:+ return sum(+ 1 for n in self._nodes.values() if n.get("sketch_id") == sketch_id+ )+ return len(self._nodes)++ def get_edge_count(self, sketch_id: Optional[str] = None) -> int:+ """Get total edge count, optionally filtered by sketch_id."""+ if sketch_id:+ return sum(+ 1 for e in self._edges.values() if e.get("sketch_id") == sketch_id+ )+ return len(self._edges)++ def clear(self) -> None:+ """Clear all data (useful between tests)."""+ self._nodes.clear()+ self._edges.clear()+ self._batch_operations.clear()diff --git a/flowsint-core/tests/core/graph/test_graph_repository.py b/flowsint-core/tests/core/graph/test_graph_repository.pynew file mode 100644index 00000000..003b553b--- /dev/null+++ b/flowsint-core/tests/core/graph/test_graph_repository.py@@ -0,0 +1,874 @@+"""Tests for Neo4jGraphRepository."""++import pytest+from unittest.mock import MagicMock, patch, call+from datetime import datetime, timezone++from flowsint_core.core.graph import Neo4jGraphRepository+++class TestNeo4jGraphRepositoryInit:+ def test_init_with_connection(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)+ assert repo._connection == mock_connection+ assert repo._batch_operations == []+ assert repo._batch_size == 100++ def test_init_without_connection_uses_singleton(self):+ with patch(+ "flowsint_core.core.graph.repository.Neo4jConnection.get_instance"+ ) as mock_get_instance:+ mock_connection = MagicMock()+ mock_get_instance.return_value = mock_connection+ repo = Neo4jGraphRepository()+ assert repo._connection == mock_connection+++class TestCreateNode:+ def test_create_node_success(self):+ mock_connection = MagicMock()+ mock_connection.query.return_value = [{"id": "element-123"}]+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ node_obj = {+ "nodeLabel": "example.com",+ "nodeType": "domain",+ "nodeProperties.domain": "example.com",+ }++ result = repo.create_node(node_obj, sketch_id="sketch-1")++ assert result == "element-123"+ mock_connection.query.assert_called_once()++ def test_create_node_no_connection(self):+ repo = Neo4jGraphRepository(neo4j_connection=None)+ repo._connection = None++ result = repo.create_node({"nodeLabel": "test", "nodeType": "domain"}, "sketch-1")++ assert result is None++ def test_create_node_empty_result(self):+ mock_connection = MagicMock()+ mock_connection.query.return_value = []+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.create_node(+ {"nodeLabel": "test", "nodeType": "domain"}, sketch_id="sketch-1"+ )++ assert result is None+++class TestCreateRelationship:+ def test_create_relationship_success(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ rel_obj = {+ "from_type": "domain",+ "from_label": "source.com",+ "to_type": "ip",+ "to_label": "1.1.1.1",+ "rel_label": "RESOLVES_TO",+ }++ repo.create_relationship(rel_obj, sketch_id="sketch-1")++ mock_connection.execute_write.assert_called_once()++ def test_create_relationship_no_connection(self):+ repo = Neo4jGraphRepository(neo4j_connection=None)+ repo._connection = None++ rel_obj = {+ "from_type": "domain",+ "from_label": "source.com",+ "to_type": "ip",+ "to_label": "1.1.1.1",+ "rel_label": "RESOLVES_TO",+ }++ # Should not raise, just return early+ repo.create_relationship(rel_obj, sketch_id="sketch-1")+++class TestBuildNodeQuery:+ def test_build_node_query_structure(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ node_obj = {+ "nodeLabel": "example.com",+ "nodeType": "domain",+ }++ query, params = repo._build_node_query(node_obj, sketch_id="sketch-1")++ assert "MERGE" in query+ assert "domain" in query # nodeType used as label+ assert params["node_label"] == "example.com"+ assert params["sketch_id"] == "sketch-1"+ assert params["props"] == node_obj+ assert "created_at" in params+++class TestBuildRelationshipQuery:+ def test_build_relationship_query_structure(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ rel_obj = {+ "from_type": "domain",+ "from_label": "source.com",+ "to_type": "ip",+ "to_label": "1.1.1.1",+ "rel_label": "RESOLVES_TO",+ }++ query, params = repo._build_relationship_query(rel_obj, sketch_id="sketch-1")++ assert "MATCH" in query+ assert "MERGE" in query+ assert "domain" in query+ assert "ip" in query+ assert "RESOLVES_TO" in query+ assert params["from_label"] == "source.com"+ assert params["to_label"] == "1.1.1.1"+ assert params["sketch_id"] == "sketch-1"+++class TestBatchOperations:+ def test_add_to_batch_node(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ repo.add_to_batch(+ "node",+ node_obj={"nodeLabel": "test", "nodeType": "domain"},+ sketch_id="sketch-1",+ )++ assert len(repo._batch_operations) == 1++ def test_add_to_batch_relationship(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ repo.add_to_batch(+ "relationship",+ rel_obj={+ "from_type": "domain",+ "from_label": "a.com",+ "to_type": "ip",+ "to_label": "1.1.1.1",+ "rel_label": "REL",+ },+ sketch_id="sketch-1",+ )++ assert len(repo._batch_operations) == 1++ def test_add_to_batch_unknown_type_raises(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ with pytest.raises(ValueError, match="Unknown operation type"):+ repo.add_to_batch("unknown", sketch_id="sketch-1")++ def test_auto_flush_when_batch_full(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)+ repo._batch_size = 2++ repo.add_to_batch(+ "node",+ node_obj={"nodeLabel": "test1", "nodeType": "domain"},+ sketch_id="sketch-1",+ )+ assert len(repo._batch_operations) == 1++ repo.add_to_batch(+ "node",+ node_obj={"nodeLabel": "test2", "nodeType": "domain"},+ sketch_id="sketch-1",+ )++ # Should have auto-flushed+ mock_connection.execute_batch.assert_called_once()+ assert len(repo._batch_operations) == 0++ def test_flush_batch(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ # Track what was passed to execute_batch before it gets cleared+ captured_args = []++ def capture_batch(ops):+ captured_args.extend(list(ops))++ mock_connection.execute_batch.side_effect = capture_batch++ repo._batch_operations = [("query1", {"p": 1}), ("query2", {"p": 2})]++ repo.flush_batch()++ mock_connection.execute_batch.assert_called_once()+ assert len(captured_args) == 2+ assert captured_args[0] == ("query1", {"p": 1})+ assert captured_args[1] == ("query2", {"p": 2})+ assert len(repo._batch_operations) == 0++ def test_flush_batch_empty(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ repo.flush_batch()++ mock_connection.execute_batch.assert_not_called()++ def test_flush_batch_no_connection(self):+ repo = Neo4jGraphRepository(neo4j_connection=None)+ repo._connection = None+ repo._batch_operations = [("query", {})]++ repo.flush_batch()++ assert len(repo._batch_operations) == 0++ def test_clear_batch(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)+ repo._batch_operations = [("query", {})]++ repo.clear_batch()++ assert len(repo._batch_operations) == 0++ def test_set_batch_size(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ repo.set_batch_size(50)++ assert repo._batch_size == 50++ def test_set_batch_size_invalid(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ with pytest.raises(ValueError, match="Batch size must be at least 1"):+ repo.set_batch_size(0)+++class TestBatchCreateNodes:+ def test_batch_create_nodes_success(self):+ mock_connection = MagicMock()+ mock_connection.execute_batch.return_value = [+ [{"id": "id-1"}],+ [{"id": "id-2"}],+ ]+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ nodes = [+ {"nodeLabel": "a.com", "nodeType": "domain"},+ {"nodeLabel": "b.com", "nodeType": "domain"},+ ]++ result = repo.batch_create_nodes(nodes, sketch_id="sketch-1")++ assert result["nodes_created"] == 2+ assert result["node_ids"] == ["id-1", "id-2"]+ assert result["errors"] == []++ def test_batch_create_nodes_no_connection(self):+ repo = Neo4jGraphRepository(neo4j_connection=None)+ repo._connection = None++ result = repo.batch_create_nodes(+ [{"nodeLabel": "test", "nodeType": "domain"}], sketch_id="sketch-1"+ )++ assert result["nodes_created"] == 0+ assert result["node_ids"] == []+ assert "No database connection" in result["errors"]++ def test_batch_create_nodes_empty_list(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.batch_create_nodes([], sketch_id="sketch-1")++ assert result == {"nodes_created": 0, "node_ids": [], "errors": []}++ def test_batch_create_nodes_execution_error(self):+ mock_connection = MagicMock()+ mock_connection.execute_batch.side_effect = Exception("DB error")+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.batch_create_nodes(+ [{"nodeLabel": "test", "nodeType": "domain"}], sketch_id="sketch-1"+ )++ assert result["nodes_created"] == 0+ assert "Batch execution failed" in result["errors"][0]+++class TestBatchCreateEdges:+ def test_batch_create_edges_success(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ edges = [+ {+ "from_type": "domain",+ "from_label": "a.com",+ "to_type": "ip",+ "to_label": "1.1.1.1",+ "rel_label": "RESOLVES",+ }+ ]++ result = repo.batch_create_edges(edges, sketch_id="sketch-1")++ assert result["edges_created"] == 1+ assert result["errors"] == []++ def test_batch_create_edges_no_connection(self):+ repo = Neo4jGraphRepository(neo4j_connection=None)+ repo._connection = None++ result = repo.batch_create_edges([{}], sketch_id="sketch-1")++ assert result["edges_created"] == 0+ assert "No database connection" in result["errors"]++ def test_batch_create_edges_empty_list(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.batch_create_edges([], sketch_id="sketch-1")++ assert result == {"edges_created": 0, "errors": []}+++class TestBatchCreateEdgesByElementId:+ def test_batch_create_edges_by_element_id_success(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ edges = [+ {+ "from_element_id": "elem-1",+ "to_element_id": "elem-2",+ "rel_label": "CONNECTS",+ }+ ]++ result = repo.batch_create_edges_by_element_id(edges, sketch_id="sketch-1")++ assert result["edges_created"] == 1+ assert result["errors"] == []++ def test_batch_create_edges_by_element_id_missing_fields(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ edges = [{"rel_label": "CONNECTS"}] # Missing from/to element IDs++ result = repo.batch_create_edges_by_element_id(edges, sketch_id="sketch-1")++ assert result["edges_created"] == 0+ assert any("Missing required fields" in e for e in result["errors"])++ def test_batch_create_edges_by_element_id_no_connection(self):+ repo = Neo4jGraphRepository(neo4j_connection=None)+ repo._connection = None++ result = repo.batch_create_edges_by_element_id([{}], sketch_id="sketch-1")++ assert result["edges_created"] == 0+ assert "No database connection" in result["errors"]+++class TestUpdateNode:+ def test_update_node_success(self):+ mock_connection = MagicMock()+ mock_connection.query.return_value = [{"id": "elem-1"}]+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.update_node(+ element_id="elem-1",+ updates={"nodeLabel": "updated"},+ sketch_id="sketch-1",+ )++ assert result == "elem-1"++ def test_update_node_no_connection(self):+ repo = Neo4jGraphRepository(neo4j_connection=None)+ repo._connection = None++ result = repo.update_node("elem-1", {"nodeLabel": "x"}, "sketch-1")++ assert result is None+++class TestDeleteNodes:+ def test_delete_nodes_success(self):+ mock_connection = MagicMock()+ mock_connection.query.return_value = [{"deleted_count": 3}]+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.delete_nodes(["id-1", "id-2", "id-3"], sketch_id="sketch-1")++ assert result == 3++ def test_delete_nodes_no_connection(self):+ repo = Neo4jGraphRepository(neo4j_connection=None)+ repo._connection = None++ result = repo.delete_nodes(["id-1"], sketch_id="sketch-1")++ assert result == 0++ def test_delete_nodes_empty_list(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.delete_nodes([], sketch_id="sketch-1")++ assert result == 0+ mock_connection.query.assert_not_called()+++class TestDeleteRelationships:+ def test_delete_relationships_success(self):+ mock_connection = MagicMock()+ mock_connection.query.return_value = [{"deleted_count": 2}]+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.delete_relationships(["rel-1", "rel-2"], sketch_id="sketch-1")++ assert result == 2++ def test_delete_relationships_no_connection(self):+ repo = Neo4jGraphRepository(neo4j_connection=None)+ repo._connection = None++ result = repo.delete_relationships(["rel-1"], sketch_id="sketch-1")++ assert result == 0++ def test_delete_relationships_empty_list(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.delete_relationships([], sketch_id="sketch-1")++ assert result == 0+++class TestDeleteAllSketchNodes:+ def test_delete_all_sketch_nodes_success(self):+ mock_connection = MagicMock()+ mock_connection.query.return_value = [{"deleted_count": 10}]+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.delete_all_sketch_nodes(sketch_id="sketch-1")++ assert result == 10++ def test_delete_all_sketch_nodes_no_connection(self):+ repo = Neo4jGraphRepository(neo4j_connection=None)+ repo._connection = None++ result = repo.delete_all_sketch_nodes(sketch_id="sketch-1")++ assert result == 0+++class TestGetSketchGraph:+ def test_get_sketch_graph_success(self):+ mock_connection = MagicMock()+ mock_connection.query.side_effect = [+ # First call: nodes query+ [+ {"id": "node-1", "labels": ["domain"], "data": {}},+ {"id": "node-2", "labels": ["ip"], "data": {}},+ ],+ # Second call: edges query+ [+ {+ "id": "edge-1",+ "type": "RESOLVES",+ "source": "node-1",+ "target": "node-2",+ "data": {},+ }+ ],+ ]+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.get_sketch_graph(sketch_id="sketch-1")++ assert len(result["nodes"]) == 2+ assert len(result["edges"]) == 1++ def test_get_sketch_graph_no_connection(self):+ repo = Neo4jGraphRepository(neo4j_connection=None)+ repo._connection = None++ result = repo.get_sketch_graph(sketch_id="sketch-1")++ assert result == {"nodes": [], "edges": []}++ def test_get_sketch_graph_no_nodes(self):+ mock_connection = MagicMock()+ mock_connection.query.return_value = []+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.get_sketch_graph(sketch_id="sketch-1")++ assert result == {"nodes": [], "edges": []}+++class TestUpdateRelationship:+ def test_update_relationship_success(self):+ mock_connection = MagicMock()+ mock_connection.query.return_value = [+ {"id": "rel-1", "type": "CONNECTS", "data": {"weight": 5}}+ ]+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.update_relationship(+ element_id="rel-1",+ rel_obj={"weight": 5},+ sketch_id="sketch-1",+ )++ assert result["id"] == "rel-1"++ def test_update_relationship_no_connection(self):+ repo = Neo4jGraphRepository(neo4j_connection=None)+ repo._connection = None++ result = repo.update_relationship("rel-1", {}, "sketch-1")++ assert result is None+++class TestCreateRelationshipByElementId:+ def test_create_relationship_by_element_id_success(self):+ mock_connection = MagicMock()+ mock_connection.query.return_value = [{"rel": {"sketch_id": "sketch-1"}}]+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.create_relationship_by_element_id(+ from_element_id="elem-1",+ to_element_id="elem-2",+ rel_label="CONNECTS",+ sketch_id="sketch-1",+ )++ assert result["sketch_id"] == "sketch-1"++ def test_create_relationship_by_element_id_no_connection(self):+ repo = Neo4jGraphRepository(neo4j_connection=None)+ repo._connection = None++ result = repo.create_relationship_by_element_id(+ "elem-1", "elem-2", "CONNECTS", "sketch-1"+ )++ assert result is None+++class TestQuery:+ def test_query_success(self):+ mock_connection = MagicMock()+ mock_connection.query.return_value = [{"count": 5}]+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.query("MATCH (n) RETURN count(n) as count", {})++ assert result == [{"count": 5}]++ def test_query_no_connection(self):+ repo = Neo4jGraphRepository(neo4j_connection=None)+ repo._connection = None++ result = repo.query("MATCH (n) RETURN n", {})++ assert result == []+++class TestUpdateNodesPositions:+ def test_update_nodes_positions_success(self):+ mock_connection = MagicMock()+ mock_connection.query.return_value = [{"updated_count": 2}]+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ positions = [+ {"nodeId": "node-1", "x": 100, "y": 200},+ {"nodeId": "node-2", "x": 300, "y": 400},+ ]++ result = repo.update_nodes_positions(positions, sketch_id="sketch-1")++ assert result == 2++ def test_update_nodes_positions_no_connection(self):+ repo = Neo4jGraphRepository(neo4j_connection=None)+ repo._connection = None++ result = repo.update_nodes_positions([{"nodeId": "x", "x": 0, "y": 0}], "s")++ assert result == 0++ def test_update_nodes_positions_empty_list(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.update_nodes_positions([], sketch_id="sketch-1")++ assert result == 0+++class TestGetNodesByIds:+ def test_get_nodes_by_ids_success(self):+ mock_connection = MagicMock()+ mock_connection.query.return_value = [+ {"data": {"nodeLabel": "a.com"}},+ {"data": {"nodeLabel": "b.com"}},+ ]+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.get_nodes_by_ids(["id-1", "id-2"], sketch_id="sketch-1")++ assert len(result) == 2++ def test_get_nodes_by_ids_no_connection(self):+ repo = Neo4jGraphRepository(neo4j_connection=None)+ repo._connection = None++ result = repo.get_nodes_by_ids(["id-1"], sketch_id="sketch-1")++ assert result == []++ def test_get_nodes_by_ids_empty_list(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.get_nodes_by_ids([], sketch_id="sketch-1")++ assert result == []+++class TestMergeNodes:+ def test_merge_nodes_create_new_node(self):+ mock_connection = MagicMock()+ mock_connection.query.side_effect = [+ [{"newElementId": "new-elem-1"}], # Create query+ None, # Copy relationships+ None, # Delete old nodes+ ]+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.merge_nodes(+ old_node_ids=["old-1", "old-2"],+ new_node_data={"type": "domain"},+ new_node_id=None,+ sketch_id="sketch-1",+ )++ assert result == "new-elem-1"++ def test_merge_nodes_reuse_existing_node(self):+ mock_connection = MagicMock()+ mock_connection.query.side_effect = [+ [{"newElementId": "old-1"}], # Update existing+ None, # Copy relationships+ None, # Delete old nodes+ ]+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.merge_nodes(+ old_node_ids=["old-1", "old-2"],+ new_node_data={"type": "domain"},+ new_node_id="old-1", # Reusing old-1+ sketch_id="sketch-1",+ )++ assert result == "old-1"++ def test_merge_nodes_no_connection(self):+ repo = Neo4jGraphRepository(neo4j_connection=None)+ repo._connection = None++ result = repo.merge_nodes(["old-1"], {}, None, "sketch-1")++ assert result is None++ def test_merge_nodes_empty_list(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.merge_nodes([], {}, None, "sketch-1")++ assert result is None+++class TestGetNeighbors:+ def test_get_neighbors_success(self):+ mock_connection = MagicMock()+ mock_connection.query.return_value = [+ {+ "center_id": "node-1",+ "center_data": {"nodeLabel": "center"},+ "rel_id": "rel-1",+ "rel_label": "CONNECTS",+ "other_id": "node-2",+ "other_data": {"nodeLabel": "neighbor"},+ "direction": "outgoing",+ }+ ]+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.get_neighbors(node_id="node-1", sketch_id="sketch-1")++ assert len(result["nodes"]) == 2+ assert len(result["edges"]) == 1++ def test_get_neighbors_no_relationships(self):+ mock_connection = MagicMock()+ mock_connection.query.return_value = [+ {+ "center_id": "node-1",+ "center_data": {"nodeLabel": "center"},+ "rel_id": None,+ "rel_label": None,+ "other_id": None,+ "other_data": None,+ "direction": None,+ }+ ]+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.get_neighbors(node_id="node-1", sketch_id="sketch-1")++ assert len(result["nodes"]) == 1+ assert len(result["edges"]) == 0++ def test_get_neighbors_no_connection(self):+ repo = Neo4jGraphRepository(neo4j_connection=None)+ repo._connection = None++ result = repo.get_neighbors("node-1", "sketch-1")++ assert result == {"nodes": [], "edges": []}++ def test_get_neighbors_not_found(self):+ mock_connection = MagicMock()+ mock_connection.query.return_value = []+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.get_neighbors("nonexistent", "sketch-1")++ assert result == {"nodes": [], "edges": []}++ def test_get_neighbors_incoming_direction(self):+ mock_connection = MagicMock()+ mock_connection.query.return_value = [+ {+ "center_id": "node-1",+ "center_data": {"nodeLabel": "center"},+ "rel_id": "rel-1",+ "rel_label": "CONNECTS",+ "other_id": "node-2",+ "other_data": {"nodeLabel": "neighbor"},+ "direction": "incoming",+ }+ ]+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.get_neighbors(node_id="node-1", sketch_id="sketch-1")++ # For incoming, source should be the other node+ edge = result["edges"][0]+ assert edge["source"] == "node-2"+ assert edge["target"] == "node-1"+++class TestCountNodesBySketch:+ def test_count_nodes_by_sketch_success(self):+ mock_connection = MagicMock()+ mock_driver = MagicMock()+ mock_session = MagicMock()+ mock_result = MagicMock()+ mock_record = MagicMock()++ mock_record.__getitem__ = lambda self, key: 5 if key == "total" else None+ mock_result.single.return_value = mock_record+ mock_session.run.return_value = mock_result+ mock_session.__enter__ = MagicMock(return_value=mock_session)+ mock_session.__exit__ = MagicMock(return_value=None)+ mock_driver.session.return_value = mock_session+ mock_connection.get_driver.return_value = mock_driver++ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.count_nodes_by_sketch(sketch_id="sketch-1")++ assert result == 5+++class TestCountEdgesBySketch:+ def test_count_edges_by_sketch_success(self):+ mock_connection = MagicMock()+ mock_driver = MagicMock()+ mock_session = MagicMock()+ mock_result = MagicMock()+ mock_record = MagicMock()++ mock_record.__getitem__ = lambda self, key: 3 if key == "total" else None+ mock_result.single.return_value = mock_record+ mock_session.run.return_value = mock_result+ mock_session.__enter__ = MagicMock(return_value=mock_session)+ mock_session.__exit__ = MagicMock(return_value=None)+ mock_driver.session.return_value = mock_session+ mock_connection.get_driver.return_value = mock_driver++ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)++ result = repo.count_edges_by_sketch(sketch_id="sketch-1")++ assert result == 3+++class TestContextManager:+ def test_context_manager_flushes_on_success(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)+ repo._batch_operations = [("query", {})]++ with repo:+ pass++ mock_connection.execute_batch.assert_called_once()++ def test_context_manager_clears_on_exception(self):+ mock_connection = MagicMock()+ repo = Neo4jGraphRepository(neo4j_connection=mock_connection)+ repo._batch_operations = [("query", {})]++ with pytest.raises(ValueError):+ with repo:+ raise ValueError("Test error")++ # Should have cleared, not flushed+ mock_connection.execute_batch.assert_not_called()+ assert len(repo._batch_operations) == 0diff --git a/flowsint-core/tests/core/graph/test_graph_serializer.py b/flowsint-core/tests/core/graph/test_graph_serializer.pynew file mode 100644index 00000000..dbbd01c0--- /dev/null+++ b/flowsint-core/tests/core/graph/test_graph_serializer.py@@ -0,0 +1,320 @@+from datetime import datetime++import pytest+from flowsint_types import Domain, Ip++from flowsint_core.core.graph import (+ GraphSerializer,+ GraphEdge,+ GraphNode,+ NodeMetadata,+)+++def test_serializer():+ created_at = datetime.now()+ node = GraphNode(+ id="id",+ nodeLabel="nodeLabel",+ nodeFlag="blue",+ nodeType="domain",+ nodeColor="nodeColor",+ nodeSize=4,+ nodeImage="nodeImage",+ nodeIcon="nodeIcon",+ x=100,+ y=100,+ nodeProperties=Domain(domain="domain.com", nodeLabel="domain.com", root=True),+ nodeMetadata=NodeMetadata(created_at=created_at),+ )+ to_neo4j = GraphSerializer.graph_node_to_neo4j_dict(node)++ expected = {+ "id": "id",+ "nodeLabel": "nodeLabel",+ "nodeType": "domain",+ "nodeColor": "nodeColor",+ "nodeFlag": "blue",+ "nodeSize": 4,+ "nodeImage": "nodeImage",+ "nodeIcon": "nodeIcon",+ "x": 100.0,+ "y": 100.0,+ "nodeProperties.domain": "domain.com",+ "nodeProperties.root": True,+ "nodeMetadata.created_at": created_at.isoformat(),+ }++ assert to_neo4j == expected+++def test_deserializer():+ created_at = datetime.now()++ node = {+ "id": "id",+ "data": {+ "nodeLabel": "nodeLabel",+ "nodeType": "domain",+ "nodeColor": "nodeColor",+ "nodeSize": 4,+ "nodeImage": "nodeImage",+ "nodeIcon": "nodeIcon",+ "x": 100.0,+ "y": 100.0,+ "nodeProperties.domain": "domain.com",+ "nodeProperties.root": True,+ "nodeMetadata.created_at": created_at.isoformat(),+ },+ }++ graph_node = GraphNode(+ id="id",+ nodeLabel="nodeLabel",+ nodeType="domain",+ nodeColor="nodeColor",+ nodeSize=4,+ nodeImage="nodeImage",+ nodeIcon="nodeIcon",+ x=100,+ y=100,+ nodeProperties=Domain(domain="domain.com", nodeLabel="domain.com", root=True),+ nodeMetadata=NodeMetadata(created_at=created_at),+ )+ output = GraphSerializer.neo4j_dict_to_graph_node(node)++ assert output == graph_node+++def test_serialize_from_flowsint_type():+ domain = Domain(domain="domain.com", nodeLabel="domain.com", root=True)++ neo4j_dict = GraphSerializer.flowsint_type_to_neo4j_dict(domain)++ # Check static fields+ assert neo4j_dict["id"] == ""+ assert neo4j_dict["nodeLabel"] == "domain.com"+ assert neo4j_dict["nodeType"] == "domain"+ assert neo4j_dict["nodeColor"] is None+ assert neo4j_dict["nodeSize"] is None+ assert neo4j_dict["nodeImage"] is None+ assert neo4j_dict["nodeIcon"] is None+ assert neo4j_dict["nodeFlag"] is None+ assert neo4j_dict["x"] == 100.0+ assert neo4j_dict["y"] == 100.0+ assert neo4j_dict["nodeProperties.domain"] == "domain.com"+ assert neo4j_dict["nodeProperties.root"] is True++ # Timestamp is generated internally, just verify it exists and is ISO format+ assert "nodeMetadata.created_at" in neo4j_dict+ assert "T" in neo4j_dict["nodeMetadata.created_at"]+++class TestCleanEmptyValues:+ def test_removes_empty_strings(self):+ data = {"key1": "value", "key2": "", "key3": "another"}+ result = GraphSerializer._clean_empty_values(data)+ assert result == {"key1": "value", "key3": "another"}++ def test_removes_none_values(self):+ data = {"key1": "value", "key2": None, "key3": "another"}+ result = GraphSerializer._clean_empty_values(data)+ assert result == {"key1": "value", "key3": "another"}++ def test_cleans_nested_dicts(self):+ data = {"outer": {"inner1": "value", "inner2": "", "inner3": None}}+ result = GraphSerializer._clean_empty_values(data)+ assert result == {"outer": {"inner1": "value"}}++ def test_cleans_lists(self):+ data = {"items": ["a", "", "b", None, "c"]}+ result = GraphSerializer._clean_empty_values(data)+ assert result == {"items": ["a", "b", "c"]}++ def test_cleans_list_of_dicts(self):+ data = {"items": [{"a": "1", "b": ""}, {"c": None, "d": "2"}]}+ result = GraphSerializer._clean_empty_values(data)+ assert result == {"items": [{"a": "1"}, {"d": "2"}]}++ def test_removes_empty_nested_dict(self):+ data = {"outer": {"inner": ""}}+ result = GraphSerializer._clean_empty_values(data)+ assert result == {}+++class TestParseFlowsintType:+ def test_parses_domain(self):+ entity = {"domain": "example.com", "root": True}+ result = GraphSerializer.parse_flowsint_type(entity, "domain")+ assert isinstance(result, Domain)+ assert result.domain == "example.com"+ assert result.root is True++ def test_parses_ip(self):+ entity = {"address": "192.168.1.1"}+ result = GraphSerializer.parse_flowsint_type(entity, "ip")+ assert isinstance(result, Ip)+ assert result.address == "192.168.1.1"++ def test_cleans_empty_values_before_parsing(self):+ entity = {"domain": "example.com", "extra": "", "other": None}+ result = GraphSerializer.parse_flowsint_type(entity, "domain")+ assert isinstance(result, Domain)+ assert result.domain == "example.com"++ def test_raises_on_unknown_type(self):+ entity = {"key": "value"}+ with pytest.raises(ValueError, match="Unknown type: unknowntype"):+ GraphSerializer.parse_flowsint_type(entity, "unknowntype")+++class TestGraphNodeToFlowsintType:+ def test_extracts_node_properties(self):+ domain = Domain(domain="example.com")+ node = GraphNode(+ id="123",+ nodeLabel="example.com",+ nodeType="domain",+ nodeProperties=domain,+ nodeMetadata=NodeMetadata(),+ )+ result = GraphSerializer.graph_node_to_flowsint_type(node)+ assert result is domain+ assert isinstance(result, Domain)+ assert result.domain == "example.com"+++class TestNeo4jDictToGraphEdge:+ def test_converts_edge_dict(self):+ edge_dict = {+ "id": "edge-123",+ "source": "node-1",+ "target": "node-2",+ "type": "CONNECTED_TO",+ }+ result = GraphSerializer.neo4j_dict_to_graph_edge(edge_dict)+ assert isinstance(result, GraphEdge)+ assert result.id == "edge-123"+ assert result.source == "node-1"+ assert result.target == "node-2"+ assert result.label == "CONNECTED_TO"++ def test_converts_values_to_string(self):+ edge_dict = {"id": 123, "source": 1, "target": 2, "type": "REL"}+ result = GraphSerializer.neo4j_dict_to_graph_edge(edge_dict)+ assert result.id == "123"+ assert result.source == "1"+ assert result.target == "2"+++class TestGraphEdgeToNeo4jDict:+ def test_with_flowsint_types(self):+ from_obj = Domain(domain="source.com")+ to_obj = Domain(domain="target.com")+ result = GraphSerializer.graph_edge_to_neo4j_dict(from_obj, to_obj, "LINKS_TO")+ assert result == {+ "from_type": "domain",+ "from_label": "source.com",+ "to_type": "domain",+ "to_label": "target.com",+ "rel_label": "LINKS_TO",+ }++ def test_with_graph_nodes(self):+ from_obj = GraphNode(+ id="1",+ nodeLabel="source",+ nodeType="ip",+ nodeProperties=Ip(address="1.1.1.1"),+ nodeMetadata=NodeMetadata(),+ )+ to_obj = GraphNode(+ id="2",+ nodeLabel="target",+ nodeType="domain",+ nodeProperties=Domain(domain="example.com"),+ nodeMetadata=NodeMetadata(),+ )+ result = GraphSerializer.graph_edge_to_neo4j_dict(from_obj, to_obj, "RESOLVES")+ assert result == {+ "from_type": "ip",+ "from_label": "source",+ "to_type": "domain",+ "to_label": "target",+ "rel_label": "RESOLVES",+ }++ def test_with_mixed_types(self):+ from_obj = Domain(domain="source.com")+ to_obj = GraphNode(+ id="2",+ nodeLabel="target",+ nodeType="ip",+ nodeProperties=Ip(address="1.1.1.1"),+ nodeMetadata=NodeMetadata(),+ )+ result = GraphSerializer.graph_edge_to_neo4j_dict(from_obj, to_obj, "HOSTS")+ assert result == {+ "from_type": "domain",+ "from_label": "source.com",+ "to_type": "ip",+ "to_label": "target",+ "rel_label": "HOSTS",+ }+++class TestDeserializeNodes:+ def test_deserializes_multiple_nodes(self):+ node_dicts = [+ {+ "id": "1",+ "data": {+ "nodeLabel": "example.com",+ "nodeType": "domain",+ "nodeProperties.domain": "example.com",+ "nodeMetadata.created_at": "2026-01-01T00:00:00",+ },+ },+ {+ "id": "2",+ "data": {+ "nodeLabel": "1.1.1.1",+ "nodeType": "ip",+ "nodeProperties.address": "1.1.1.1",+ "nodeMetadata.created_at": "2026-01-01T00:00:00",+ },+ },+ ]+ result = GraphSerializer.deserialize_nodes(node_dicts)+ assert len(result) == 2+ assert all(isinstance(node, GraphNode) for node in result)+ assert result[0].nodeLabel == "example.com"+ assert result[1].nodeLabel == "1.1.1.1"++ def test_empty_list(self):+ result = GraphSerializer.deserialize_nodes([])+ assert result == []+++class TestDeserializeEdges:+ def test_deserializes_multiple_edges(self):+ edge_dicts = [+ {"id": "e1", "source": "n1", "target": "n2", "type": "CONNECTS"},+ {"id": "e2", "source": "n2", "target": "n3", "type": "LINKS"},+ ]+ result = GraphSerializer.deserialize_edges(edge_dicts)+ assert len(result) == 2+ assert all(isinstance(edge, GraphEdge) for edge in result)+ assert result[0].label == "CONNECTS"+ assert result[1].label == "LINKS"++ def test_empty_list(self):+ result = GraphSerializer.deserialize_edges([])+ assert result == []+++class TestNeo4jDictToGraphNodeErrors:+ def test_raises_on_missing_data(self):+ node_dict = {"id": "123"}+ with pytest.raises(Exception, match="Could not find node data"):+ GraphSerializer.neo4j_dict_to_graph_node(node_dict)diff --git a/flowsint-core/tests/core/graph/test_graph_service.py b/flowsint-core/tests/core/graph/test_graph_service.pynew file mode 100644index 00000000..c70adf37--- /dev/null+++ b/flowsint-core/tests/core/graph/test_graph_service.py@@ -0,0 +1,682 @@+"""Tests for GraphService using dependency injection."""++import pytest+from unittest.mock import MagicMock, patch++from flowsint_types import Domain, Ip+from flowsint_core.core.graph import (+ GraphService,+ create_graph_service,+ GraphNode,+ GraphData,+ NodeMetadata,+)++from .in_memory_graph_repository import InMemoryGraphRepository+++class TestGraphServiceInit:+ def test_init_with_injected_repository(self):+ """Test that a repository can be injected."""+ mock_repo = MagicMock()+ mock_logger = MagicMock()++ service = GraphService(+ sketch_id="sketch-1",+ repository=mock_repo,+ logger=mock_logger,+ enable_batching=True,+ )++ assert service._sketch_id == "sketch-1"+ assert service._repository == mock_repo+ assert service._logger == mock_logger+ assert service._enable_batching is True++ def test_init_with_in_memory_repository(self):+ """Test initialization with InMemoryGraphRepository."""+ repo = InMemoryGraphRepository()+ service = GraphService(sketch_id="sketch-1", repository=repo)++ assert service._sketch_id == "sketch-1"+ assert service._repository is repo++ def test_init_without_repository_raises(self):+ """Test that without repository injection, ValueError is raised."""+ with pytest.raises(ValueError, match="repository is required"):+ GraphService(sketch_id="sketch-1", repository=None)++ def test_sketch_id_property(self):+ repo = InMemoryGraphRepository()+ service = GraphService(sketch_id="sketch-1", repository=repo)+ assert service.sketch_id == "sketch-1"++ def test_repository_property(self):+ repo = InMemoryGraphRepository()+ service = GraphService(sketch_id="sketch-1", repository=repo)+ assert service.repository is repo+++class TestCreateNode:+ def test_create_node_with_graph_node(self):+ """Test creating a node with injected mock repository."""+ mock_repo = MagicMock()+ mock_repo.create_node.return_value = "elem-123"++ service = GraphService(sketch_id="sketch-1", repository=mock_repo)+ node = GraphNode(+ id="1",+ nodeLabel="example.com",+ nodeType="domain",+ nodeProperties=Domain(domain="example.com"),+ nodeMetadata=NodeMetadata(),+ )++ result = service.create_node(node)++ assert result == "elem-123"+ mock_repo.create_node.assert_called_once()++ def test_create_node_with_in_memory_repository(self):+ """Test creating a node with InMemoryGraphRepository."""+ repo = InMemoryGraphRepository()+ service = GraphService(sketch_id="sketch-1", repository=repo)++ node = GraphNode(+ id="1",+ nodeLabel="example.com",+ nodeType="domain",+ nodeProperties=Domain(domain="example.com"),+ nodeMetadata=NodeMetadata(),+ )++ result = service.create_node(node)++ assert result is not None+ assert repo.get_node_count("sketch-1") == 1++ def test_create_node_with_flowsint_type_raises(self):+ repo = InMemoryGraphRepository()+ service = GraphService(sketch_id="sketch-1", repository=repo)+ domain = Domain(domain="example.com")++ with pytest.raises(Exception, match="create_node method takes a GraphNode"):+ service.create_node(domain)++ def test_create_node_with_batching(self):+ mock_repo = MagicMock()+ service = GraphService(+ sketch_id="sketch-1", repository=mock_repo, enable_batching=True+ )++ node = GraphNode(+ id="1",+ nodeLabel="example.com",+ nodeType="domain",+ nodeProperties=Domain(domain="example.com"),+ nodeMetadata=NodeMetadata(),+ )++ service.create_node(node)++ mock_repo.add_to_batch.assert_called_once()+ mock_repo.create_node.assert_not_called()+++class TestCreateNodeFromFlowsintType:+ def test_create_node_from_flowsint_type(self):+ mock_repo = MagicMock()+ mock_repo.create_node.return_value = "elem-123"++ service = GraphService(sketch_id="sketch-1", repository=mock_repo)+ domain = Domain(domain="example.com")++ result = service.create_node_from_flowsint_type(domain)++ assert result == "elem-123"+ mock_repo.create_node.assert_called_once()++ def test_create_node_from_flowsint_type_with_graph_node_raises(self):+ repo = InMemoryGraphRepository()+ service = GraphService(sketch_id="sketch-1", repository=repo)+ node = GraphNode(+ id="1",+ nodeLabel="example.com",+ nodeType="domain",+ nodeProperties=Domain(domain="example.com"),+ nodeMetadata=NodeMetadata(),+ )++ with pytest.raises(+ Exception, match="create_node_from_flowsint_type method takes a FlowsintType"+ ):+ service.create_node_from_flowsint_type(node)++ def test_create_node_from_flowsint_type_with_batching(self):+ mock_repo = MagicMock()+ service = GraphService(+ sketch_id="sketch-1", repository=mock_repo, enable_batching=True+ )+ domain = Domain(domain="example.com")++ service.create_node_from_flowsint_type(domain)++ mock_repo.add_to_batch.assert_called_once()+++class TestGetSketchGraph:+ def test_get_sketch_graph_with_mock(self):+ mock_repo = MagicMock()+ mock_repo.get_sketch_graph.return_value = {"nodes": [], "edges": []}++ service = GraphService(sketch_id="sketch-1", repository=mock_repo)+ result = service.get_sketch_graph()++ assert isinstance(result, GraphData)+ mock_repo.get_sketch_graph.assert_called_once_with("sketch-1")++ def test_get_sketch_graph_with_in_memory(self):+ """Test getting sketch graph with actual data in InMemoryRepository."""+ repo = InMemoryGraphRepository()+ service = GraphService(sketch_id="sketch-1", repository=repo)++ # Add some nodes+ node1 = GraphNode(+ id="1",+ nodeLabel="example.com",+ nodeType="domain",+ nodeProperties=Domain(domain="example.com"),+ nodeMetadata=NodeMetadata(),+ )+ node2 = GraphNode(+ id="2",+ nodeLabel="1.1.1.1",+ nodeType="ip",+ nodeProperties=Ip(address="1.1.1.1"),+ nodeMetadata=NodeMetadata(),+ )++ service.create_node(node1)+ service.create_node(node2)++ result = service.get_sketch_graph()++ assert isinstance(result, GraphData)+ assert len(result.nodes) == 2+++class TestGetNodesByIds:+ def test_get_nodes_by_ids(self):+ """Test that get_nodes_by_ids calls the repository correctly."""+ mock_repo = MagicMock()+ # Return empty list to avoid serialization issues in this unit test+ mock_repo.get_nodes_by_ids.return_value = []++ service = GraphService(sketch_id="sketch-1", repository=mock_repo)+ result = service.get_nodes_by_ids(["id-1", "id-2"])++ mock_repo.get_nodes_by_ids.assert_called_once_with(["id-1", "id-2"], "sketch-1")+ assert result == []+++class TestGetNodesByIdsForTask:+ def test_get_nodes_by_ids_for_task(self):+ """Test that get_nodes_by_ids_for_task calls the repository correctly."""+ mock_repo = MagicMock()+ # Return empty list to avoid serialization issues in this unit test+ mock_repo.get_nodes_by_ids.return_value = []++ service = GraphService(sketch_id="sketch-1", repository=mock_repo)++ result = service.get_nodes_by_ids_for_task(["id-1"])++ mock_repo.get_nodes_by_ids.assert_called_once_with(["id-1"], "sketch-1")+ assert result == []+++class TestCreateRelationship:+ def test_create_relationship(self):+ mock_repo = MagicMock()+ service = GraphService(sketch_id="sketch-1", repository=mock_repo)++ from_obj = Domain(domain="example.com")+ to_obj = Ip(address="1.1.1.1")++ service.create_relationship(from_obj, to_obj, "RESOLVES")++ mock_repo.create_relationship.assert_called_once()++ def test_create_relationship_with_batching(self):+ mock_repo = MagicMock()+ service = GraphService(+ sketch_id="sketch-1", repository=mock_repo, enable_batching=True+ )++ service.create_relationship(+ Domain(domain="a.com"), Domain(domain="b.com")+ )++ mock_repo.add_to_batch.assert_called_once()+ mock_repo.create_relationship.assert_not_called()+++class TestCreateRelationshipByElementId:+ def test_create_relationship_by_element_id(self):+ mock_repo = MagicMock()+ mock_repo.create_relationship_by_element_id.return_value = {"sketch_id": "s1"}++ service = GraphService(sketch_id="sketch-1", repository=mock_repo)+ result = service.create_relationship_by_element_id(+ from_element_id="elem-1",+ to_element_id="elem-2",+ rel_label="CONNECTS",+ )++ assert result == {"sketch_id": "s1"}+ mock_repo.create_relationship_by_element_id.assert_called_once_with(+ from_element_id="elem-1",+ to_element_id="elem-2",+ rel_label="CONNECTS",+ sketch_id="sketch-1",+ )+++class TestGetNeighbors:+ def test_get_neighbors(self):+ mock_repo = MagicMock()+ mock_repo.get_neighbors.return_value = {"nodes": [], "edges": []}++ service = GraphService(sketch_id="sketch-1", repository=mock_repo)+ result = service.get_neighbors(node_id="node-1")++ assert isinstance(result, GraphData)+ mock_repo.get_neighbors.assert_called_once_with(+ node_id="node-1", sketch_id="sketch-1"+ )+++class TestUpdateNode:+ def test_update_node(self):+ mock_repo = MagicMock()+ mock_repo.update_node.return_value = "elem-1"++ service = GraphService(sketch_id="sketch-1", repository=mock_repo)+ result = service.update_node("elem-1", {"nodeLabel": "updated"})++ assert result == "elem-1"+ mock_repo.update_node.assert_called_once()+++class TestUpdateNodesPositions:+ def test_update_nodes_positions(self):+ mock_repo = MagicMock()+ mock_repo.update_nodes_positions.return_value = 2++ service = GraphService(sketch_id="sketch-1", repository=mock_repo)+ positions = [{"nodeId": "n1", "x": 100, "y": 200}]+ result = service.update_nodes_positions(positions)++ assert result == 2+ mock_repo.update_nodes_positions.assert_called_once_with(+ positions=positions, sketch_id="sketch-1"+ )+++class TestDeleteNodes:+ def test_delete_nodes(self):+ mock_repo = MagicMock()+ mock_repo.delete_nodes.return_value = 3++ service = GraphService(sketch_id="sketch-1", repository=mock_repo)+ result = service.delete_nodes(["id-1", "id-2", "id-3"])++ assert result == 3+ mock_repo.delete_nodes.assert_called_once_with(+ node_ids=["id-1", "id-2", "id-3"], sketch_id="sketch-1"+ )++ def test_delete_nodes_with_in_memory(self):+ """Test delete with InMemoryRepository to verify actual behavior."""+ repo = InMemoryGraphRepository()+ service = GraphService(sketch_id="sketch-1", repository=repo)++ # Create nodes+ node1 = GraphNode(+ id="1",+ nodeLabel="example.com",+ nodeType="domain",+ nodeProperties=Domain(domain="example.com"),+ nodeMetadata=NodeMetadata(),+ )+ elem_id = service.create_node(node1)++ assert repo.get_node_count("sketch-1") == 1++ # Delete+ deleted = service.delete_nodes([elem_id])++ assert deleted == 1+ assert repo.get_node_count("sketch-1") == 0+++class TestDeleteRelationships:+ def test_delete_relationships(self):+ mock_repo = MagicMock()+ mock_repo.delete_relationships.return_value = 2++ service = GraphService(sketch_id="sketch-1", repository=mock_repo)+ result = service.delete_relationships(["rel-1", "rel-2"])++ assert result == 2+ mock_repo.delete_relationships.assert_called_once_with(+ relationship_ids=["rel-1", "rel-2"], sketch_id="sketch-1"+ )+++class TestDeleteAllSketchNodes:+ def test_delete_all_sketch_nodes(self):+ mock_repo = MagicMock()+ mock_repo.delete_all_sketch_nodes.return_value = 10++ service = GraphService(sketch_id="sketch-1", repository=mock_repo)+ result = service.delete_all_sketch_nodes()++ assert result == 10+ mock_repo.delete_all_sketch_nodes.assert_called_once_with(sketch_id="sketch-1")+++class TestUpdateRelationship:+ def test_update_relationship(self):+ mock_repo = MagicMock()+ mock_repo.update_relationship.return_value = {"id": "rel-1", "weight": 5}++ service = GraphService(sketch_id="sketch-1", repository=mock_repo)+ result = service.update_relationship("rel-1", {"weight": 5})++ assert result == {"id": "rel-1", "weight": 5}+ mock_repo.update_relationship.assert_called_once_with(+ element_id="rel-1", rel_obj={"weight": 5}, sketch_id="sketch-1"+ )+++class TestMergeNodes:+ def test_merge_nodes(self):+ mock_repo = MagicMock()+ mock_repo.merge_nodes.return_value = "new-elem"++ service = GraphService(sketch_id="sketch-1", repository=mock_repo)+ result = service.merge_nodes(+ old_node_ids=["old-1", "old-2"],+ new_node_data={"type": "domain"},+ new_node_id="old-1",+ )++ assert result == "new-elem"+ mock_repo.merge_nodes.assert_called_once_with(+ old_node_ids=["old-1", "old-2"],+ new_node_data={"type": "domain"},+ new_node_id="old-1",+ sketch_id="sketch-1",+ )+++class TestBatchCreateNodes:+ def test_batch_create_nodes(self):+ mock_repo = MagicMock()+ mock_repo.batch_create_nodes.return_value = {+ "nodes_created": 2,+ "node_ids": ["id-1", "id-2"],+ "errors": [],+ }++ service = GraphService(sketch_id="sketch-1", repository=mock_repo)+ nodes = [{"nodeLabel": "a"}, {"nodeLabel": "b"}]+ result = service.batch_create_nodes(nodes)++ assert result["nodes_created"] == 2+ mock_repo.batch_create_nodes.assert_called_once_with(+ nodes=nodes, sketch_id="sketch-1"+ )+++class TestBatchCreateEdgesByElementId:+ def test_batch_create_edges_by_element_id(self):+ mock_repo = MagicMock()+ mock_repo.batch_create_edges_by_element_id.return_value = {+ "edges_created": 1,+ "errors": [],+ }++ service = GraphService(sketch_id="sketch-1", repository=mock_repo)+ edges = [{"from_element_id": "e1", "to_element_id": "e2"}]+ result = service.batch_create_edges_by_element_id(edges)++ assert result["edges_created"] == 1+ mock_repo.batch_create_edges_by_element_id.assert_called_once_with(+ edges=edges, sketch_id="sketch-1"+ )+++class TestLogGraphMessage:+ def test_log_graph_message_with_logger(self):+ mock_logger = MagicMock()+ repo = InMemoryGraphRepository()++ service = GraphService(+ sketch_id="sketch-1", repository=repo, logger=mock_logger+ )+ service.log_graph_message("Test message")++ mock_logger.graph_append.assert_called_once_with(+ "sketch-1", {"message": "Test message"}+ )++ def test_log_graph_message_without_logger(self):+ repo = InMemoryGraphRepository()+ service = GraphService(sketch_id="sketch-1", repository=repo, logger=None)+ # Should not raise+ service.log_graph_message("Test message")+++class TestFlush:+ def test_flush_with_batching_enabled(self):+ mock_repo = MagicMock()+ service = GraphService(+ sketch_id="sketch-1", repository=mock_repo, enable_batching=True+ )+ service.flush()++ mock_repo.flush_batch.assert_called_once()++ def test_flush_with_batching_disabled(self):+ mock_repo = MagicMock()+ service = GraphService(+ sketch_id="sketch-1", repository=mock_repo, enable_batching=False+ )+ service.flush()++ mock_repo.flush_batch.assert_not_called()+++class TestQuery:+ def test_query(self):+ mock_repo = MagicMock()+ mock_repo.query.return_value = [{"count": 5}]++ service = GraphService(sketch_id="sketch-1", repository=mock_repo)+ result = service.query("MATCH (n) RETURN count(n)", {"param": "value"})++ assert result == [{"count": 5}]+ mock_repo.query.assert_called_once_with(+ "MATCH (n) RETURN count(n)", {"param": "value"}+ )+++class TestSetBatchSize:+ def test_set_batch_size(self):+ mock_repo = MagicMock()+ service = GraphService(sketch_id="sketch-1", repository=mock_repo)+ service.set_batch_size(50)++ mock_repo.set_batch_size.assert_called_once_with(50)+++class TestContextManager:+ def test_context_manager_flushes_on_success(self):+ mock_repo = MagicMock()+ service = GraphService(+ sketch_id="sketch-1", repository=mock_repo, enable_batching=True+ )++ with service:+ pass++ mock_repo.flush_batch.assert_called_once()++ def test_context_manager_does_not_flush_on_exception(self):+ mock_repo = MagicMock()+ service = GraphService(+ sketch_id="sketch-1", repository=mock_repo, enable_batching=True+ )++ with pytest.raises(ValueError):+ with service:+ raise ValueError("Test error")++ mock_repo.flush_batch.assert_not_called()+++class TestCreateGraphServiceFactory:+ def test_create_graph_service(self):+ mock_repo = MagicMock()++ with patch(+ "flowsint_core.core.graph.service.Neo4jGraphRepository", return_value=mock_repo+ ) as MockRepoClass:+ with patch("flowsint_core.core.logger.Logger") as MockLogger:+ service = create_graph_service(+ sketch_id="sketch-1",+ enable_batching=True,+ )++ # Factory should create the repository+ MockRepoClass.assert_called_once()+ assert service._sketch_id == "sketch-1"+ assert service._repository == mock_repo+ assert service._logger == MockLogger+ assert service._enable_batching is True++ def test_create_graph_service_defaults(self):+ mock_repo = MagicMock()++ with patch(+ "flowsint_core.core.graph.service.Neo4jGraphRepository", return_value=mock_repo+ ):+ with patch("flowsint_core.core.logger.Logger"):+ service = create_graph_service(sketch_id="sketch-1")++ assert service._sketch_id == "sketch-1"+ assert service._repository == mock_repo+ assert service._enable_batching is True # Default is True in factory+++class TestInMemoryRepositoryIntegration:+ """Integration tests using InMemoryGraphRepository to verify full workflows."""++ def test_full_workflow_create_and_query(self):+ """Test a complete workflow: create nodes, create relationships, query."""+ repo = InMemoryGraphRepository()+ service = GraphService(sketch_id="sketch-1", repository=repo)++ # Create two nodes+ node1 = GraphNode(+ id="1",+ nodeLabel="example.com",+ nodeType="domain",+ nodeProperties=Domain(domain="example.com"),+ nodeMetadata=NodeMetadata(),+ )+ node2 = GraphNode(+ id="2",+ nodeLabel="1.1.1.1",+ nodeType="ip",+ nodeProperties=Ip(address="1.1.1.1"),+ nodeMetadata=NodeMetadata(),+ )++ elem_id1 = service.create_node(node1)+ elem_id2 = service.create_node(node2)++ assert elem_id1 is not None+ assert elem_id2 is not None+ assert repo.get_node_count("sketch-1") == 2++ # Create a relationship+ service.create_relationship(+ Domain(domain="example.com"),+ Ip(address="1.1.1.1"),+ "RESOLVES_TO",+ )++ assert repo.get_edge_count("sketch-1") == 1++ # Query the graph+ graph = service.get_sketch_graph()+ assert len(graph.nodes) == 2+ assert len(graph.edges) == 1++ def test_batch_operations_with_in_memory(self):+ """Test batch operations work correctly with InMemoryRepository."""+ repo = InMemoryGraphRepository()+ service = GraphService(+ sketch_id="sketch-1", repository=repo, enable_batching=True+ )++ # Add nodes to batch+ for i in range(3):+ node = GraphNode(+ id=str(i),+ nodeLabel=f"node{i}.com",+ nodeType="domain",+ nodeProperties=Domain(domain=f"node{i}.com"),+ nodeMetadata=NodeMetadata(),+ )+ service.create_node(node)++ # Nodes should not be created yet (batched)+ assert repo.get_node_count("sketch-1") == 0++ # Flush the batch+ service.flush()++ # Now nodes should exist+ assert repo.get_node_count("sketch-1") == 3++ def test_sketch_isolation(self):+ """Test that different sketches are isolated."""+ repo = InMemoryGraphRepository()+ service1 = GraphService(sketch_id="sketch-1", repository=repo)+ service2 = GraphService(sketch_id="sketch-2", repository=repo)++ # Create nodes in different sketches+ node = GraphNode(+ id="1",+ nodeLabel="example.com",+ nodeType="domain",+ nodeProperties=Domain(domain="example.com"),+ nodeMetadata=NodeMetadata(),+ )++ service1.create_node(node)+ service2.create_node(node)++ # Each sketch should have its own node+ assert repo.get_node_count("sketch-1") == 1+ assert repo.get_node_count("sketch-2") == 1+ assert repo.get_node_count() == 2++ # Delete all from sketch-1+ service1.delete_all_sketch_nodes()++ assert repo.get_node_count("sketch-1") == 0+ assert repo.get_node_count("sketch-2") == 1diff --git a/flowsint-core/tests/core/test_enricher_base_simplified_api.py b/flowsint-core/tests/core/test_enricher_base_simplified_api.pyindex 9b741eca..3a2d9cd7 100644--- a/flowsint-core/tests/core/test_enricher_base_simplified_api.py+++ b/flowsint-core/tests/core/test_enricher_base_simplified_api.py@@ -1,11 +1,13 @@"""Test simplified API for create_node and create_relationship."""+from typing import List+import pytest-from flowsint_core.core.enricher_base import Enricherfrom flowsint_types.domain import Domainfrom flowsint_types.email import Emailfrom flowsint_types.individual import Individual-from typing import List++from flowsint_core.core.enricher_base import Enricherclass MockEnricher(Enricher):@@ -49,4 +51,4 @@ def test_create_node_with_property_override():domain = Domain(domain="example.com")# Should be able to override properties- enricher.create_node(domain, type="subdomain")+ enricher.create_node(domain)diff --git a/flowsint-core/tests/core/test_graph_repository.py b/flowsint-core/tests/core/test_graph_repository.pydeleted file mode 100644index a603071b..00000000--- a/flowsint-core/tests/core/test_graph_repository.py+++ /dev/null@@ -1,260 +0,0 @@-"""Test GraphRepository batch operations."""--import pytest-from unittest.mock import Mock, MagicMock-from flowsint_core.core.graph_repository import GraphRepository-from flowsint_types.domain import Domain-from flowsint_types.ip import Ip-from flowsint_types.email import Email---def mock_connection():- """Create a mock Neo4j connection."""- conn = Mock()- conn.execute_batch = Mock(return_value=[])- conn.query = Mock(return_value=[])- return conn---def graph_repo(mock_connection):- """Create a GraphRepository with mocked connection."""- return GraphRepository(neo4j_connection=mock_connection)---def test_batch_create_nodes_empty_list(graph_repo):- """Test batch_create_nodes with empty list."""- result = graph_repo.batch_create_nodes(nodes=[], sketch_id="test-sketch")-- assert result["nodes_created"] == 0- assert result["node_ids"] == []- assert result["errors"] == []---def test_batch_create_nodes_single_node(graph_repo, mock_connection):- """Test batch_create_nodes with a single node."""- domain = Domain(domain="example.com")-- # Mock the batch execution to return a node ID- mock_connection.execute_batch.return_value = [- [{"id": "element-id-123"}]- ]-- result = graph_repo.batch_create_nodes(- nodes=[domain],- sketch_id="test-sketch"- )-- assert result["nodes_created"] == 1- assert len(result["node_ids"]) == 1- assert result["node_ids"][0] == "element-id-123"- assert result["errors"] == []-- # Verify execute_batch was called once- assert mock_connection.execute_batch.call_count == 1---def test_batch_create_nodes_multiple_nodes(graph_repo, mock_connection):- """Test batch_create_nodes with multiple nodes of different types."""- domain = Domain(domain="example.com")- ip = Ip(address="192.168.1.1")- email = Email(email="[email protected]")-- # Mock the batch execution to return multiple node IDs- mock_connection.execute_batch.return_value = [- [{"id": "element-id-1"}],- [{"id": "element-id-2"}],- [{"id": "element-id-3"}]- ]-- result = graph_repo.batch_create_nodes(- nodes=[domain, ip, email],- sketch_id="test-sketch"- )-- assert result["nodes_created"] == 3- assert len(result["node_ids"]) == 3- assert result["errors"] == []-- # Verify execute_batch was called once with 3 operations- assert mock_connection.execute_batch.call_count == 1- batch_operations = mock_connection.execute_batch.call_args[0][0]- assert len(batch_operations) == 3---def test_batch_create_nodes_with_validation_errors(graph_repo, mock_connection):- """Test batch_create_nodes when some nodes have validation errors."""- valid_domain = Domain(domain="example.com")-- # Mock execute_batch to succeed for valid nodes- mock_connection.execute_batch.return_value = [- [{"id": "element-id-1"}]- ]-- result = graph_repo.batch_create_nodes(- nodes=[valid_domain],- sketch_id="test-sketch"- )-- assert result["nodes_created"] == 1- assert len(result["errors"]) == 0---def test_batch_create_nodes_batch_execution_failure(graph_repo, mock_connection):- """Test batch_create_nodes when the batch execution fails."""- domain = Domain(domain="example.com")-- # Mock execute_batch to raise an exception- mock_connection.execute_batch.side_effect = Exception("Database connection error")-- result = graph_repo.batch_create_nodes(- nodes=[domain],- sketch_id="test-sketch"- )-- assert result["nodes_created"] == 0- assert result["node_ids"] == []- assert len(result["errors"]) > 0- assert "Database connection error" in result["errors"][0]---def test_batch_create_nodes_no_connection():- """Test batch_create_nodes when there's no database connection."""- # Create a repository with no connection by manually setting it to None- repo = GraphRepository(neo4j_connection=Mock())- repo._connection = None # Force no connection- domain = Domain(domain="example.com")-- result = repo.batch_create_nodes(- nodes=[domain],- sketch_id="test-sketch"- )-- assert result["nodes_created"] == 0- assert result["node_ids"] == []- assert len(result["errors"]) == 1- assert "No database connection" in result["errors"][0]---def test_batch_create_nodes_with_label_fallback(graph_repo, mock_connection):- """Test batch_create_nodes with nodes that need label fallback."""- from pydantic import BaseModel-- class CustomNode(BaseModel):- """Node type without a clear primary field."""- name: str = ""- label: str = "Custom Node"-- node = CustomNode(name="", label="Fallback Label")-- # Mock the batch execution- mock_connection.execute_batch.return_value = [- [{"id": "element-id-fallback"}]- ]-- result = graph_repo.batch_create_nodes(- nodes=[node],- sketch_id="test-sketch"- )-- # Should succeed with fallback to label- assert result["nodes_created"] == 1- assert len(result["node_ids"]) == 1---def test_batch_create_nodes_large_batch(graph_repo, mock_connection):- """Test batch_create_nodes with a large number of nodes."""- # Create 1000 domains- domains = [Domain(domain=f"example{i}.com") for i in range(1000)]-- # Mock the batch execution to return 1000 node IDs- mock_connection.execute_batch.return_value = [- [{"id": f"element-id-{i}"}] for i in range(1000)- ]-- result = graph_repo.batch_create_nodes(- nodes=domains,- sketch_id="test-sketch"- )-- assert result["nodes_created"] == 1000- assert len(result["node_ids"]) == 1000- assert result["errors"] == []-- # Verify execute_batch was called once (single transaction)- assert mock_connection.execute_batch.call_count == 1- batch_operations = mock_connection.execute_batch.call_args[0][0]- assert len(batch_operations) == 1000---# Tests for update_node---def test_update_node_success(graph_repo, mock_connection):- """Test update_node with a valid Pydantic object."""- domain = Domain(domain="updated-example.com", label="Updated Domain")-- # Mock the query to return the element ID- mock_connection.query.return_value = [{"id": "element-id-123"}]-- result = graph_repo.update_node(- element_id="element-id-123",- node_obj=domain,- sketch_id="test-sketch"- )-- assert result == "element-id-123"- assert mock_connection.query.call_count == 1---def test_update_node_not_found(graph_repo, mock_connection):- """Test update_node when node doesn't exist."""- domain = Domain(domain="example.com")-- # Mock the query to return empty result- mock_connection.query.return_value = []-- result = graph_repo.update_node(- element_id="non-existent-id",- node_obj=domain,- sketch_id="test-sketch"- )-- assert result is None---def test_update_node_no_connection():- """Test update_node when there's no database connection."""- repo = GraphRepository(neo4j_connection=Mock())- repo._connection = None- domain = Domain(domain="example.com")-- result = repo.update_node(- element_id="element-id-123",- node_obj=domain,- sketch_id="test-sketch"- )-- assert result is None---def test_update_node_different_types(graph_repo, mock_connection):- """Test update_node with different Pydantic types."""- ip = Ip(address="10.0.0.1", label="Updated IP")-- # Mock the query- mock_connection.query.return_value = [{"id": "element-id-456"}]-- result = graph_repo.update_node(- element_id="element-id-456",- node_obj=ip,- sketch_id="test-sketch"- )-- assert result == "element-id-456"-- # Verify the query contains the correct type- query_call = mock_connection.query.call_args- params = query_call[0][1]- assert params["type"] == "ip"diff --git a/flowsint-core/tests/utils/__init__.py b/flowsint-core/tests/utils/__init__.pynew file mode 100644index 00000000..e69de29bdiff --git a/flowsint-core/tests/utils/flatten.py b/flowsint-core/tests/utils/flatten.pynew file mode 100644index 00000000..e194cfd4--- /dev/null+++ b/flowsint-core/tests/utils/flatten.py@@ -0,0 +1,38 @@+from flowsint_core.utils import flatten, unflatten++my_dict = {+ "root_key_1": "value 1",+ "root_key_2": 2,+ "root_key_3": "value 3",+ "root_key_4": {+ "child_key_1": "child 1",+ "child_key_2": "child 2",+ "child_key_3": {"grand_child_1": 0},+ },+}++my_flat_dict = {+ "root_key_1": "value 1",+ "root_key_2": 2,+ "root_key_3": "value 3",+ "root_key_4.child_key_1": "child 1",+ "root_key_4.child_key_2": "child 2",+ "root_key_4.child_key_3.grand_child_1": 0,+}++my_flat_dict_other_separator = {+ "root_key_1": "value 1",+ "root_key_2": 2,+ "root_key_3": "value 3",+ "root_key_4_child_key_1": "child 1",+ "root_key_4_child_key_2": "child 2",+ "root_key_4_child_key_3_grand_child_1": 0,+}+++def test_flatten():+ assert flatten(my_dict) == my_flat_dict+++def test_unflatten():+ assert unflatten(my_flat_dict) == my_dict