A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat(api): use of graph_repository to import nodes + add created_at on nodes
dextmorgn8e587a7340e0
diff --git a/flowsint-api/app/api/routes/sketches.py b/flowsint-api/app/api/routes/sketches.pyindex c8ea36ad..395829af 100644--- a/flowsint-api/app/api/routes/sketches.py+++ b/flowsint-api/app/api/routes/sketches.py@@ -11,6 +11,7 @@ from fastapi import ()from pydantic import BaseModel, Fieldfrom typing import Literal, List, Optional, Dict, Any+from datetime import datetime, timezonefrom flowsint_core.utils import flattenfrom sqlalchemy.orm import Sessionfrom app.api.schemas.sketch import SketchCreate, SketchRead, SketchUpdate@@ -258,13 +259,20 @@ def add_node(cypher_props = dict_to_cypher_props(properties)+ # Add created_at to parameters+ properties_with_timestamp = {+ **properties,+ "created_at": datetime.now(timezone.utc).isoformat()+ }+create_query = f"""MERGE (d:`{node_type}` {{ {cypher_props} }})+ ON CREATE SET d.created_at = $created_atRETURN d as node, elementId(d) as id"""try:- create_result = neo4j_connection.query(create_query, properties)+ create_result = neo4j_connection.query(create_query, properties_with_timestamp)except Exception as e:print(f"Query execution error: {e}")raise HTTPException(status_code=500, detail=f"Database error: {str(e)}")@@ -474,7 +482,9 @@ def merge_nodes("""params = {"nodeId": newNode.id, "sketch_id": sketch_id, **properties}else:- # Create a completely new node+ # Create a completely new node with created_at timestamp+ properties["created_at"] = datetime.now(timezone.utc).isoformat()+create_query = f"""CREATE (n:`{node_type}`)SET n = $properties@@ -861,7 +871,8 @@ async def execute_import(# Create mapping lookup by row indexmappings_by_row = {m.row_index: m for m in entity_mappings if m.include}- # Import entities+ # Import entities using GraphRepository+ graph_repo = GraphRepository(neo4j_connection)nodes_created = 0nodes_skipped = 0errors = []@@ -878,31 +889,22 @@ async def execute_import(label = entity.primary_valueentity_data = entity.data- # Create node properties with all data- properties = {- "type": entity_type.lower(),- "sketch_id": sketch_id,- "label": label,- "caption": label,- **entity_data, # Include all row data (edited or original) as properties- }-- # Flatten nested dicts if any- flattened_props = flatten(properties)-- # Create node in Neo4j- create_query = f"""- MERGE (n:`{entity_type}` {{sketch_id: $sketch_id, label: $label}})- ON CREATE SET n += $props- RETURN elementId(n) as id- """+ # Flatten entity data for storage+ flattened_data = flatten(entity_data)+ # Create node using GraphRepositorytry:- neo4j_connection.query(- create_query,- {"sketch_id": sketch_id, "label": label, "props": flattened_props},+ node_id = graph_repo.create_node_from_import(+ node_type=entity_type,+ label=label,+ sketch_id=sketch_id,+ **flattened_data)- nodes_created += 1+ if node_id:+ nodes_created += 1+ else:+ nodes_skipped += 1+ errors.append(f"Row {entity.row_index + 1}: Failed to create node")except Exception as e:error_msg = f"Row {entity.row_index + 1}: {str(e)}"errors.append(error_msg)diff --git a/flowsint-core/src/flowsint_core/core/graph_repository.py b/flowsint-core/src/flowsint_core/core/graph_repository.pyindex b09e794f..b864fcdc 100644--- a/flowsint-core/src/flowsint_core/core/graph_repository.py+++ b/flowsint-core/src/flowsint_core/core/graph_repository.py@@ -6,6 +6,7 @@ handling node and relationship operations with batching support."""from typing import Dict, Any, List, Optional, Tuple+from datetime import datetime, timezonefrom .graph_db import Neo4jConnectionfrom .graph_serializer import GraphSerializer@@ -61,11 +62,17 @@ class GraphRepository:# Build SET clauses (exclude sketch_id as it's in MERGE)set_clauses = [f"n.{prop} = ${prop}" for prop in serialized_props.keys() if prop != "sketch_id"]- params = {key_prop: key_value, **serialized_props}+ params = {+ key_prop: key_value,+ "created_at": datetime.now(timezone.utc).isoformat(),+ **serialized_props+ }# Build and execute query - MERGE on both key_prop AND sketch_id for uniqueness+ # Use ON CREATE SET to only set created_at when creating, not updatingquery = f"""MERGE (n:{node_type} {{{key_prop}: ${key_prop}, sketch_id: $sketch_id}})+ ON CREATE SET n.created_at = $created_atSET {', '.join(set_clauses)}"""@@ -167,11 +174,17 @@ class GraphRepository:# Build SET clauses (exclude sketch_id as it's in MERGE)set_clauses = [f"n.{prop} = ${prop}" for prop in serialized_props.keys() if prop != "sketch_id"]- params = {key_prop: key_value, **serialized_props}+ params = {+ key_prop: key_value,+ "created_at": datetime.now(timezone.utc).isoformat(),+ **serialized_props+ }# MERGE on both key_prop AND sketch_id for uniqueness per sketch+ # Use ON CREATE SET to only set created_at when creating, not updatingquery = f"""MERGE (n:{node_type} {{{key_prop}: ${key_prop}, sketch_id: $sketch_id}})+ ON CREATE SET n.created_at = $created_atSET {', '.join(set_clauses)}"""@@ -243,6 +256,54 @@ class GraphRepository:raise ValueError("Batch size must be at least 1")self._batch_size = size+ def create_node_from_import(+ self,+ node_type: str,+ label: str,+ sketch_id: str,+ **properties: Any+ ) -> Optional[str]:+ """+ Create a node from import operation (MERGE on sketch_id + label).++ This method is specifically designed for bulk imports where nodes+ are identified by their label within a sketch, rather than by+ a type-specific key property.++ Args:+ node_type: Node label (e.g., "Domain", "Ip")+ label: Human-readable label for the node+ sketch_id: Investigation sketch ID+ **properties: All additional node properties++ Returns:+ Element ID of the created/found node or None+ """+ if not self._connection:+ return None++ # Serialize and prepare all properties+ serialized_props = GraphSerializer.serialize_properties(properties)+ serialized_props["type"] = node_type.lower()+ serialized_props["label"] = label+ serialized_props["caption"] = label+ serialized_props["created_at"] = datetime.now(timezone.utc).isoformat()++ query = f"""+ MERGE (n:`{node_type}` {{sketch_id: $sketch_id, label: $label}})+ ON CREATE SET n += $props+ RETURN elementId(n) as id+ """++ params = {+ "sketch_id": sketch_id,+ "label": label,+ "props": serialized_props+ }++ result = self._connection.query(query, params)+ return result[0]["id"] if result else None+def update_node(self,node_type: str,diff --git a/flowsint-core/src/flowsint_core/core/graph_service.py b/flowsint-core/src/flowsint_core/core/graph_service.pyindex 31152a3e..94dada58 100644--- a/flowsint-core/src/flowsint_core/core/graph_service.py+++ b/flowsint-core/src/flowsint_core/core/graph_service.py@@ -69,6 +69,12 @@ class GraphService:"""Create or update a node in the graph.+ Automatically adds the following properties:+ - type: Lowercase version of node_type+ - sketch_id: Current sketch ID+ - label: Defaults to key_value if not provided+ - created_at: ISO 8601 UTC timestamp (only on creation via ON CREATE SET)+Args:node_type: Node label (e.g., "domain", "ip")key_prop: Property name used as unique identifierdiff --git a/flowsint-core/src/flowsint_core/core/transform_base.py b/flowsint-core/src/flowsint_core/core/transform_base.pyindex 130cf7c8..69431b45 100644--- a/flowsint-core/src/flowsint_core/core/transform_base.py+++ b/flowsint-core/src/flowsint_core/core/transform_base.py@@ -398,11 +398,21 @@ class Transform(ABC):This method now uses the GraphService for improved performance andbetter separation of concerns.+ The following properties are automatically added to every node:+ - type: Lowercase version of node_type+ - sketch_id: Current sketch ID from transform context+ - label: Defaults to key_value if not provided+ - created_at: ISO 8601 UTC timestamp (only on creation, not updates)+Args:node_type: Node label (e.g., "domain", "ip")key_prop: Property name used as unique identifierkey_value: Value of the key property**properties: Additional node properties++ Note:+ Uses MERGE semantics - if a node with the same (key_prop, sketch_id)+ exists, it will be updated. The created_at field is only set on creation."""self._graph_service.create_node(node_type=node_type,