A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat(api): type registry + better import
dextmorgn7692fc76f61d
diff --git a/flowsint-api/app/api/routes/enrichers.py b/flowsint-api/app/api/routes/enrichers.pyindex da28d3cc..85e11ada 100644--- a/flowsint-api/app/api/routes/enrichers.py+++ b/flowsint-api/app/api/routes/enrichers.py@@ -95,5 +95,4 @@ async def launch_enricher(except HTTPException:raiseexcept Exception as e:- print(e)raise HTTPException(status_code=500, detail=f"Error launching enricher: {str(e)}")diff --git a/flowsint-api/app/api/routes/flows.py b/flowsint-api/app/api/routes/flows.pyindex 8d999e84..0ca52b24 100644--- a/flowsint-api/app/api/routes/flows.py+++ b/flowsint-api/app/api/routes/flows.py@@ -280,7 +280,6 @@ async def launch_flow(except HTTPException:raiseexcept Exception as e:- print(e)raise HTTPException(status_code=500, detail=f"Error launching flow: {str(e)}")diff --git a/flowsint-api/app/api/routes/keys.py b/flowsint-api/app/api/routes/keys.pyindex 297b7850..861dea08 100644--- a/flowsint-api/app/api/routes/keys.py+++ b/flowsint-api/app/api/routes/keys.py@@ -66,7 +66,6 @@ def create_key()return keyexcept Exception as e:- print(e)raise HTTPException(status_code=500, detail="An error occured creating the key.")diff --git a/flowsint-api/app/api/routes/sketches.py b/flowsint-api/app/api/routes/sketches.pyindex b1428eda..63ef53a9 100644--- a/flowsint-api/app/api/routes/sketches.py+++ b/flowsint-api/app/api/routes/sketches.py@@ -21,7 +21,7 @@ from flowsint_core.core.graph_db import neo4j_connectionfrom flowsint_core.core.graph_repository import GraphRepositoryfrom flowsint_core.core.postgre_db import get_dbfrom app.api.deps import get_current_user-from flowsint_core.imports import parse_file+from flowsint_core.imports import FileParseResult, parse_import_filefrom app.api.sketch_utils import update_sketch_timestamprouter = APIRouter()@@ -799,26 +799,7 @@ def get_related_nodes(return {"nds": all_nodes, "rls": relationships}-class EntityPreviewModel(BaseModel):- """Preview model for a single entity."""-- row_index: int- data: Dict[str, Any]- detected_type: str- primary_value: str- confidence: str---class AnalyzeFileResponse(BaseModel):- """Response model for file analysis."""-- entities: List[EntityPreviewModel]- total_entities: int- type_distribution: Dict[str, int]- columns: List[str]--[email protected]("/{sketch_id}/import/analyze", response_model=AnalyzeFileResponse)[email protected]("/{sketch_id}/import/analyze", response_model=FileParseResult)async def analyze_import_file(sketch_id: str,file: UploadFile = File(...),@@ -826,8 +807,8 @@ async def analyze_import_file(current_user: Profile = Depends(get_current_user),):"""- Analyze an uploaded file for import.- Each row represents one entity. Detects entity types and provides preview.+ Analyze an uploaded TXT file for import.+ Each line represents one entity. Detects entity types and provides preview."""# Verify sketch exists and user has accesssketch = db.query(Sketch).filter(Sketch.id == sketch_id).first()@@ -837,6 +818,13 @@ async def analyze_import_file(current_user.id, sketch.investigation_id, actions=["read"], db=db)+ # Validate file extension+ if not file.filename or not file.filename.lower().endswith(".txt"):+ raise HTTPException(+ status_code=400,+ detail="Only .txt files are supported. Please upload a text file with one value per line.",+ )+# Read file contenttry:content = await file.read()@@ -845,7 +833,7 @@ async def analyze_import_file(# Parse and analyze the filetry:- result = parse_file(+ result = parse_import_file(file_content=content,filename=file.filename or "unknown.csv",max_preview_rows=10000000,@@ -855,34 +843,17 @@ async def analyze_import_file(except Exception as e:raise HTTPException(status_code=500, detail=f"Failed to parse file: {str(e)}")- # Convert entities to response models (no slicing)- entity_previews = [- EntityPreviewModel(- row_index=e.row_index,- data=e.data,- detected_type=e.detected_type,- primary_value=e.primary_value,- confidence=e.confidence,- )- for e in result.entities- ]-- return AnalyzeFileResponse(- entities=entity_previews,- total_entities=result.total_entities,- type_distribution=result.type_distribution,- columns=result.columns,- )+ return resultclass EntityMapping(BaseModel):- """Mapping configuration for an entity (row)."""+ """Mapping configuration for an entity."""- row_index: int+ id: str # Unique identifier for this entity (generated by frontend)entity_type: strinclude: bool = True- label: Optional[str] = None- data: Optional[Dict[str, Any]] = None # Edited data from frontend+ label: str+ data: Dict[str, Any] # Entity data from frontendclass ImportExecuteResponse(BaseModel):@@ -898,15 +869,14 @@ class ImportExecuteResponse(BaseModel):@update_sketch_timestampasync def execute_import(sketch_id: str,- file: UploadFile = File(...),entity_mappings_json: str = Form(...),background_tasks: BackgroundTasks = BackgroundTasks(),db: Session = Depends(get_db),current_user: Profile = Depends(get_current_user),):"""- Execute the import of entities from a file into the sketch.- Each row is one entity with all columns stored in data property.+ Execute the import of entities into the sketch.+ Uses the entity mappings provided by the frontend (no file re-parsing needed)."""import json@@ -929,21 +899,8 @@ async def execute_import(status_code=400, detail=f"Failed to parse entity_mappings: {str(e)}")- # Read and parse file- try:- content = await file.read()- result = parse_file(- file_content=content,- filename=file.filename or "unknown.csv",- max_preview_rows=10000000,- )- except ValueError as e:- raise HTTPException(status_code=400, detail=str(e))- except Exception as e:- raise HTTPException(status_code=500, detail=f"Failed to parse file: {str(e)}")-- # Create mapping lookup by row index- mappings_by_row = {m.row_index: m for m in entity_mappings if m.include}+ # Filter only entities marked for inclusion+ entities_to_import = [m for m in entity_mappings if m.include]# Import entities using GraphRepositorygraph_repo = GraphRepository(neo4j_connection)@@ -951,21 +908,20 @@ async def execute_import(nodes_skipped = 0errors = []- for entity in result.entities:- # Prefer explicit mapping when provided; otherwise import using detected defaults- mapping = mappings_by_row.get(entity.row_index)- if mapping:- entity_type = mapping.entity_type- label = mapping.label if mapping.label else entity.primary_value- entity_data = mapping.data if mapping.data is not None else entity.data- else:- entity_type = entity.detected_type- label = entity.primary_value- entity_data = entity.data+ for idx, mapping in enumerate(entities_to_import):+ # Use data from mapping directly+ entity_type = mapping.entity_type+ label = mapping.label+ entity_data = mapping.data# Flatten entity data for storageflattened_data = flatten(entity_data)+ # Remove fields that are passed as explicit parameters to avoid conflicts+ flattened_data.pop('label', None)+ flattened_data.pop('type', None)+ flattened_data.pop('sketch_id', None)+# Create node using GraphRepositorytry:node_id = graph_repo.create_node_from_import(@@ -978,9 +934,9 @@ async def execute_import(nodes_created += 1else:nodes_skipped += 1- errors.append(f"Row {entity.row_index + 1}: Failed to create node")+ errors.append(f"Entity {idx + 1} ({label}): Failed to create node")except Exception as e:- error_msg = f"Row {entity.row_index + 1}: {str(e)}"+ error_msg = f"Entity {idx + 1} ({label}): {str(e)}"errors.append(error_msg)nodes_skipped += 1