Skip to content

A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.

feat(api): type registry + better import

dextmorgn7692fc76f61d
diff --git a/flowsint-api/app/api/routes/enrichers.py b/flowsint-api/app/api/routes/enrichers.py
index da28d3cc..85e11ada 100644
--- a/flowsint-api/app/api/routes/enrichers.py
+++ b/flowsint-api/app/api/routes/enrichers.py
@@ -95,5 +95,4 @@ async def launch_enricher(
except HTTPException:
raise
except Exception as e:
- print(e)
raise HTTPException(status_code=500, detail=f"Error launching enricher: {str(e)}")
diff --git a/flowsint-api/app/api/routes/flows.py b/flowsint-api/app/api/routes/flows.py
index 8d999e84..0ca52b24 100644
--- a/flowsint-api/app/api/routes/flows.py
+++ b/flowsint-api/app/api/routes/flows.py
@@ -280,7 +280,6 @@ async def launch_flow(
except HTTPException:
raise
except Exception as e:
- print(e)
raise HTTPException(status_code=500, detail=f"Error launching flow: {str(e)}")
diff --git a/flowsint-api/app/api/routes/keys.py b/flowsint-api/app/api/routes/keys.py
index 297b7850..861dea08 100644
--- a/flowsint-api/app/api/routes/keys.py
+++ b/flowsint-api/app/api/routes/keys.py
@@ -66,7 +66,6 @@ def create_key(
)
return key
except Exception as e:
- print(e)
raise HTTPException(
status_code=500, detail="An error occured creating the key."
)
diff --git a/flowsint-api/app/api/routes/sketches.py b/flowsint-api/app/api/routes/sketches.py
index b1428eda..63ef53a9 100644
--- a/flowsint-api/app/api/routes/sketches.py
+++ b/flowsint-api/app/api/routes/sketches.py
@@ -21,7 +21,7 @@ from flowsint_core.core.graph_db import neo4j_connection
from flowsint_core.core.graph_repository import GraphRepository
from flowsint_core.core.postgre_db import get_db
from app.api.deps import get_current_user
-from flowsint_core.imports import parse_file
+from flowsint_core.imports import FileParseResult, parse_import_file
from app.api.sketch_utils import update_sketch_timestamp
router = APIRouter()
@@ -799,26 +799,7 @@ def get_related_nodes(
return {"nds": all_nodes, "rls": relationships}
-class EntityPreviewModel(BaseModel):
- """Preview model for a single entity."""
-
- row_index: int
- data: Dict[str, Any]
- detected_type: str
- primary_value: str
- confidence: str
-
-
-class AnalyzeFileResponse(BaseModel):
- """Response model for file analysis."""
-
- entities: List[EntityPreviewModel]
- total_entities: int
- type_distribution: Dict[str, int]
- columns: List[str]
-
-
[email protected]("/{sketch_id}/import/analyze", response_model=AnalyzeFileResponse)
[email protected]("/{sketch_id}/import/analyze", response_model=FileParseResult)
async def analyze_import_file(
sketch_id: str,
file: UploadFile = File(...),
@@ -826,8 +807,8 @@ async def analyze_import_file(
current_user: Profile = Depends(get_current_user),
):
"""
- Analyze an uploaded file for import.
- Each row represents one entity. Detects entity types and provides preview.
+ Analyze an uploaded TXT file for import.
+ Each line represents one entity. Detects entity types and provides preview.
"""
# Verify sketch exists and user has access
sketch = db.query(Sketch).filter(Sketch.id == sketch_id).first()
@@ -837,6 +818,13 @@ async def analyze_import_file(
current_user.id, sketch.investigation_id, actions=["read"], db=db
)
+ # Validate file extension
+ if not file.filename or not file.filename.lower().endswith(".txt"):
+ raise HTTPException(
+ status_code=400,
+ detail="Only .txt files are supported. Please upload a text file with one value per line.",
+ )
+
# Read file content
try:
content = await file.read()
@@ -845,7 +833,7 @@ async def analyze_import_file(
# Parse and analyze the file
try:
- result = parse_file(
+ result = parse_import_file(
file_content=content,
filename=file.filename or "unknown.csv",
max_preview_rows=10000000,
@@ -855,34 +843,17 @@ async def analyze_import_file(
except Exception as e:
raise HTTPException(status_code=500, detail=f"Failed to parse file: {str(e)}")
- # Convert entities to response models (no slicing)
- entity_previews = [
- EntityPreviewModel(
- row_index=e.row_index,
- data=e.data,
- detected_type=e.detected_type,
- primary_value=e.primary_value,
- confidence=e.confidence,
- )
- for e in result.entities
- ]
-
- return AnalyzeFileResponse(
- entities=entity_previews,
- total_entities=result.total_entities,
- type_distribution=result.type_distribution,
- columns=result.columns,
- )
+ return result
class EntityMapping(BaseModel):
- """Mapping configuration for an entity (row)."""
+ """Mapping configuration for an entity."""
- row_index: int
+ id: str # Unique identifier for this entity (generated by frontend)
entity_type: str
include: bool = True
- label: Optional[str] = None
- data: Optional[Dict[str, Any]] = None # Edited data from frontend
+ label: str
+ data: Dict[str, Any] # Entity data from frontend
class ImportExecuteResponse(BaseModel):
@@ -898,15 +869,14 @@ class ImportExecuteResponse(BaseModel):
@update_sketch_timestamp
async def execute_import(
sketch_id: str,
- file: UploadFile = File(...),
entity_mappings_json: str = Form(...),
background_tasks: BackgroundTasks = BackgroundTasks(),
db: Session = Depends(get_db),
current_user: Profile = Depends(get_current_user),
):
"""
- Execute the import of entities from a file into the sketch.
- Each row is one entity with all columns stored in data property.
+ Execute the import of entities into the sketch.
+ Uses the entity mappings provided by the frontend (no file re-parsing needed).
"""
import json
@@ -929,21 +899,8 @@ async def execute_import(
status_code=400, detail=f"Failed to parse entity_mappings: {str(e)}"
)
- # Read and parse file
- try:
- content = await file.read()
- result = parse_file(
- file_content=content,
- filename=file.filename or "unknown.csv",
- max_preview_rows=10000000,
- )
- except ValueError as e:
- raise HTTPException(status_code=400, detail=str(e))
- except Exception as e:
- raise HTTPException(status_code=500, detail=f"Failed to parse file: {str(e)}")
-
- # Create mapping lookup by row index
- mappings_by_row = {m.row_index: m for m in entity_mappings if m.include}
+ # Filter only entities marked for inclusion
+ entities_to_import = [m for m in entity_mappings if m.include]
# Import entities using GraphRepository
graph_repo = GraphRepository(neo4j_connection)
@@ -951,21 +908,20 @@ async def execute_import(
nodes_skipped = 0
errors = []
- for entity in result.entities:
- # Prefer explicit mapping when provided; otherwise import using detected defaults
- mapping = mappings_by_row.get(entity.row_index)
- if mapping:
- entity_type = mapping.entity_type
- label = mapping.label if mapping.label else entity.primary_value
- entity_data = mapping.data if mapping.data is not None else entity.data
- else:
- entity_type = entity.detected_type
- label = entity.primary_value
- entity_data = entity.data
+ for idx, mapping in enumerate(entities_to_import):
+ # Use data from mapping directly
+ entity_type = mapping.entity_type
+ label = mapping.label
+ entity_data = mapping.data
# Flatten entity data for storage
flattened_data = flatten(entity_data)
+ # Remove fields that are passed as explicit parameters to avoid conflicts
+ flattened_data.pop('label', None)
+ flattened_data.pop('type', None)
+ flattened_data.pop('sketch_id', None)
+
# Create node using GraphRepository
try:
node_id = graph_repo.create_node_from_import(
@@ -978,9 +934,9 @@ async def execute_import(
nodes_created += 1
else:
nodes_skipped += 1
- errors.append(f"Row {entity.row_index + 1}: Failed to create node")
+ errors.append(f"Entity {idx + 1} ({label}): Failed to create node")
except Exception as e:
- error_msg = f"Row {entity.row_index + 1}: {str(e)}"
+ error_msg = f"Entity {idx + 1} ({label}): {str(e)}"
errors.append(error_msg)
nodes_skipped += 1