Skip to content

A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.

feat(api): use of graph_repository to import nodes + add created_at on nodes

dextmorgn8e587a7340e0
diff --git a/flowsint-api/app/api/routes/sketches.py b/flowsint-api/app/api/routes/sketches.py
index c8ea36ad..395829af 100644
--- a/flowsint-api/app/api/routes/sketches.py
+++ b/flowsint-api/app/api/routes/sketches.py
@@ -11,6 +11,7 @@ from fastapi import (
)
from pydantic import BaseModel, Field
from typing import Literal, List, Optional, Dict, Any
+from datetime import datetime, timezone
from flowsint_core.utils import flatten
from sqlalchemy.orm import Session
from app.api.schemas.sketch import SketchCreate, SketchRead, SketchUpdate
@@ -258,13 +259,20 @@ def add_node(
cypher_props = dict_to_cypher_props(properties)
+ # Add created_at to parameters
+ properties_with_timestamp = {
+ **properties,
+ "created_at": datetime.now(timezone.utc).isoformat()
+ }
+
create_query = f"""
MERGE (d:`{node_type}` {{ {cypher_props} }})
+ ON CREATE SET d.created_at = $created_at
RETURN d as node, elementId(d) as id
"""
try:
- create_result = neo4j_connection.query(create_query, properties)
+ create_result = neo4j_connection.query(create_query, properties_with_timestamp)
except Exception as e:
print(f"Query execution error: {e}")
raise HTTPException(status_code=500, detail=f"Database error: {str(e)}")
@@ -474,7 +482,9 @@ def merge_nodes(
"""
params = {"nodeId": newNode.id, "sketch_id": sketch_id, **properties}
else:
- # Create a completely new node
+ # Create a completely new node with created_at timestamp
+ properties["created_at"] = datetime.now(timezone.utc).isoformat()
+
create_query = f"""
CREATE (n:`{node_type}`)
SET n = $properties
@@ -861,7 +871,8 @@ async def execute_import(
# Create mapping lookup by row index
mappings_by_row = {m.row_index: m for m in entity_mappings if m.include}
- # Import entities
+ # Import entities using GraphRepository
+ graph_repo = GraphRepository(neo4j_connection)
nodes_created = 0
nodes_skipped = 0
errors = []
@@ -878,31 +889,22 @@ async def execute_import(
label = entity.primary_value
entity_data = entity.data
- # Create node properties with all data
- properties = {
- "type": entity_type.lower(),
- "sketch_id": sketch_id,
- "label": label,
- "caption": label,
- **entity_data, # Include all row data (edited or original) as properties
- }
-
- # Flatten nested dicts if any
- flattened_props = flatten(properties)
-
- # Create node in Neo4j
- create_query = f"""
- MERGE (n:`{entity_type}` {{sketch_id: $sketch_id, label: $label}})
- ON CREATE SET n += $props
- RETURN elementId(n) as id
- """
+ # Flatten entity data for storage
+ flattened_data = flatten(entity_data)
+ # Create node using GraphRepository
try:
- neo4j_connection.query(
- create_query,
- {"sketch_id": sketch_id, "label": label, "props": flattened_props},
+ node_id = graph_repo.create_node_from_import(
+ node_type=entity_type,
+ label=label,
+ sketch_id=sketch_id,
+ **flattened_data
)
- nodes_created += 1
+ if node_id:
+ nodes_created += 1
+ else:
+ nodes_skipped += 1
+ errors.append(f"Row {entity.row_index + 1}: Failed to create node")
except Exception as e:
error_msg = f"Row {entity.row_index + 1}: {str(e)}"
errors.append(error_msg)
diff --git a/flowsint-core/src/flowsint_core/core/graph_repository.py b/flowsint-core/src/flowsint_core/core/graph_repository.py
index b09e794f..b864fcdc 100644
--- a/flowsint-core/src/flowsint_core/core/graph_repository.py
+++ b/flowsint-core/src/flowsint_core/core/graph_repository.py
@@ -6,6 +6,7 @@ handling node and relationship operations with batching support.
"""
from typing import Dict, Any, List, Optional, Tuple
+from datetime import datetime, timezone
from .graph_db import Neo4jConnection
from .graph_serializer import GraphSerializer
@@ -61,11 +62,17 @@ class GraphRepository:
# Build SET clauses (exclude sketch_id as it's in MERGE)
set_clauses = [f"n.{prop} = ${prop}" for prop in serialized_props.keys() if prop != "sketch_id"]
- params = {key_prop: key_value, **serialized_props}
+ params = {
+ key_prop: key_value,
+ "created_at": datetime.now(timezone.utc).isoformat(),
+ **serialized_props
+ }
# Build and execute query - MERGE on both key_prop AND sketch_id for uniqueness
+ # Use ON CREATE SET to only set created_at when creating, not updating
query = f"""
MERGE (n:{node_type} {{{key_prop}: ${key_prop}, sketch_id: $sketch_id}})
+ ON CREATE SET n.created_at = $created_at
SET {', '.join(set_clauses)}
"""
@@ -167,11 +174,17 @@ class GraphRepository:
# Build SET clauses (exclude sketch_id as it's in MERGE)
set_clauses = [f"n.{prop} = ${prop}" for prop in serialized_props.keys() if prop != "sketch_id"]
- params = {key_prop: key_value, **serialized_props}
+ params = {
+ key_prop: key_value,
+ "created_at": datetime.now(timezone.utc).isoformat(),
+ **serialized_props
+ }
# MERGE on both key_prop AND sketch_id for uniqueness per sketch
+ # Use ON CREATE SET to only set created_at when creating, not updating
query = f"""
MERGE (n:{node_type} {{{key_prop}: ${key_prop}, sketch_id: $sketch_id}})
+ ON CREATE SET n.created_at = $created_at
SET {', '.join(set_clauses)}
"""
@@ -243,6 +256,54 @@ class GraphRepository:
raise ValueError("Batch size must be at least 1")
self._batch_size = size
+ def create_node_from_import(
+ self,
+ node_type: str,
+ label: str,
+ sketch_id: str,
+ **properties: Any
+ ) -> Optional[str]:
+ """
+ Create a node from import operation (MERGE on sketch_id + label).
+
+ This method is specifically designed for bulk imports where nodes
+ are identified by their label within a sketch, rather than by
+ a type-specific key property.
+
+ Args:
+ node_type: Node label (e.g., "Domain", "Ip")
+ label: Human-readable label for the node
+ sketch_id: Investigation sketch ID
+ **properties: All additional node properties
+
+ Returns:
+ Element ID of the created/found node or None
+ """
+ if not self._connection:
+ return None
+
+ # Serialize and prepare all properties
+ serialized_props = GraphSerializer.serialize_properties(properties)
+ serialized_props["type"] = node_type.lower()
+ serialized_props["label"] = label
+ serialized_props["caption"] = label
+ serialized_props["created_at"] = datetime.now(timezone.utc).isoformat()
+
+ query = f"""
+ MERGE (n:`{node_type}` {{sketch_id: $sketch_id, label: $label}})
+ ON CREATE SET n += $props
+ RETURN elementId(n) as id
+ """
+
+ params = {
+ "sketch_id": sketch_id,
+ "label": label,
+ "props": serialized_props
+ }
+
+ result = self._connection.query(query, params)
+ return result[0]["id"] if result else None
+
def update_node(
self,
node_type: str,
diff --git a/flowsint-core/src/flowsint_core/core/graph_service.py b/flowsint-core/src/flowsint_core/core/graph_service.py
index 31152a3e..94dada58 100644
--- a/flowsint-core/src/flowsint_core/core/graph_service.py
+++ b/flowsint-core/src/flowsint_core/core/graph_service.py
@@ -69,6 +69,12 @@ class GraphService:
"""
Create or update a node in the graph.
+ Automatically adds the following properties:
+ - type: Lowercase version of node_type
+ - sketch_id: Current sketch ID
+ - label: Defaults to key_value if not provided
+ - created_at: ISO 8601 UTC timestamp (only on creation via ON CREATE SET)
+
Args:
node_type: Node label (e.g., "domain", "ip")
key_prop: Property name used as unique identifier
diff --git a/flowsint-core/src/flowsint_core/core/transform_base.py b/flowsint-core/src/flowsint_core/core/transform_base.py
index 130cf7c8..69431b45 100644
--- a/flowsint-core/src/flowsint_core/core/transform_base.py
+++ b/flowsint-core/src/flowsint_core/core/transform_base.py
@@ -398,11 +398,21 @@ class Transform(ABC):
This method now uses the GraphService for improved performance and
better separation of concerns.
+ The following properties are automatically added to every node:
+ - type: Lowercase version of node_type
+ - sketch_id: Current sketch ID from transform context
+ - label: Defaults to key_value if not provided
+ - created_at: ISO 8601 UTC timestamp (only on creation, not updates)
+
Args:
node_type: Node label (e.g., "domain", "ip")
key_prop: Property name used as unique identifier
key_value: Value of the key property
**properties: Additional node properties
+
+ Note:
+ Uses MERGE semantics - if a node with the same (key_prop, sketch_id)
+ exists, it will be updated. The created_at field is only set on creation.
"""
self._graph_service.create_node(
node_type=node_type,