A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat(core): add json import support
dextmorgnbaa344f364df
diff --git a/flowsint-core/src/flowsint_core/core/graph_db.py b/flowsint-core/src/flowsint_core/core/graph_db.pyindex 8e557877..bf29fce3 100644--- a/flowsint-core/src/flowsint_core/core/graph_db.py+++ b/flowsint-core/src/flowsint_core/core/graph_db.py@@ -121,20 +121,26 @@ class Neo4jConnection:with self._driver.session() as session:return session.execute_write(_execute)- def execute_batch(self, queries: List[tuple[str, Dict[str, Any]]]) -> None:+ def execute_batch(self, queries: List[tuple[str, Dict[str, Any]]]) -> List[List[Dict[str, Any]]]:"""Execute multiple queries in a single transaction.Args:queries: List of (query, parameters) tuples++ Returns:+ List of results, one for each query"""def _execute_batch(tx):+ results = []for query, params in queries:cleaned_params = self._clean_parameters(params)- tx.run(query, cleaned_params)+ result = tx.run(query, cleaned_params)+ results.append(result.data())+ return resultswith self._driver.session() as session:- session.execute_write(_execute_batch)+ return session.execute_write(_execute_batch)def close(self) -> None:"""Close the driver connection."""diff --git a/flowsint-core/src/flowsint_core/core/graph_repository.py b/flowsint-core/src/flowsint_core/core/graph_repository.pyindex 2f88bf10..e1d48876 100644--- a/flowsint-core/src/flowsint_core/core/graph_repository.py+++ b/flowsint-core/src/flowsint_core/core/graph_repository.py@@ -401,6 +401,171 @@ class GraphRepository:errors.append(f"Batch execution failed: {str(e)}")return {"nodes_created": 0, "node_ids": [], "errors": errors}+ def batch_create_edges(+ self,+ edges: List[Dict[str, Any]],+ sketch_id: str,+ ) -> Dict[str, Any]:+ """+ Create multiple edges/relationships in a single batch transaction.++ Args:+ edges: List of edge dictionaries, each with:+ - from_obj: Source node Pydantic model+ - to_obj: Target node Pydantic model+ - rel_label: Relationship type/label+ - properties: Optional dict of relationship properties+ sketch_id: Investigation sketch ID++ Returns:+ Dictionary with:+ - edges_created: Number of successfully created edges+ - errors: List of error messages for failed edges+ """+ if not self._connection:+ return {"edges_created": 0, "errors": ["No database connection"]}++ if not edges:+ return {"edges_created": 0, "errors": []}++ # Build all queries+ batch_operations = []+ errors = []++ for idx, edge in enumerate(edges):+ try:+ from_obj = edge.get("from_obj")+ to_obj = edge.get("to_obj")+ rel_label = edge.get("rel_label")+ properties = edge.get("properties", {})++ if not from_obj or not to_obj or not rel_label:+ errors.append(+ f"Edge {idx}: Missing required fields (from_obj, to_obj, or rel_label)"+ )+ continue++ query, params = self._build_relationship_query(+ from_obj=from_obj,+ to_obj=to_obj,+ rel_label=rel_label,+ sketch_id=sketch_id,+ **properties,+ )+ batch_operations.append((query, params))+ except Exception as e:+ errors.append(f"Edge {idx}: {str(e)}")++ # Execute batch+ if not batch_operations:+ return {"edges_created": 0, "errors": errors}++ try:+ # Execute all operations in a single transaction+ self._connection.execute_batch(batch_operations)++ return {+ "edges_created": len(batch_operations),+ "errors": errors,+ }+ except Exception as e:+ errors.append(f"Batch execution failed: {str(e)}")+ return {"edges_created": 0, "errors": errors}++ def batch_create_edges_by_element_id(+ self,+ edges: List[Dict[str, Any]],+ sketch_id: str,+ ) -> Dict[str, Any]:+ """+ Create multiple edges/relationships using element IDs in a single batch transaction.++ This method is more reliable than batch_create_edges when you have the element IDs,+ as it doesn't need to match nodes by their properties.++ Args:+ edges: List of edge dictionaries, each with:+ - from_element_id: Source node element ID+ - to_element_id: Target node element ID+ - rel_type: Relationship type/label+ - properties: Optional dict of relationship properties+ sketch_id: Investigation sketch ID++ Returns:+ Dictionary with:+ - edges_created: Number of successfully created edges+ - errors: List of error messages for failed edges+ """+ if not self._connection:+ return {"edges_created": 0, "errors": ["No database connection"]}++ if not edges:+ return {"edges_created": 0, "errors": []}++ # Build all queries+ batch_operations = []+ errors = []++ for idx, edge in enumerate(edges):+ try:+ from_element_id = edge.get("from_element_id")+ to_element_id = edge.get("to_element_id")+ rel_type = edge.get("rel_type", "RELATED_TO")+ properties = edge.get("properties", {})++ if not from_element_id or not to_element_id:+ errors.append(+ f"Edge {idx}: Missing required fields (from_element_id or to_element_id)"+ )+ continue++ # Serialize properties+ serialized_props = GraphSerializer.serialize_properties(properties)+ serialized_props["sketch_id"] = sketch_id++ # Build relationship properties string+ props_str = ", ".join(+ [f"{k}: ${k}_{idx}" for k in serialized_props.keys()]+ )+ rel_props = (+ f"{{{props_str}}}" if props_str else "{sketch_id: $sketch_id}"+ )++ query = f"""+ MATCH (from) WHERE elementId(from) = $from_id_{idx}+ MATCH (to) WHERE elementId(to) = $to_id_{idx}+ MERGE (from)-[r:`{rel_type}` {rel_props}]->(to)+ """++ # Build params with unique keys for batch execution+ params = {+ f"from_id_{idx}": from_element_id,+ f"to_id_{idx}": to_element_id,+ }+ # Add serialized properties with index suffix+ for k, v in serialized_props.items():+ params[f"{k}_{idx}"] = v++ batch_operations.append((query, params))+ except Exception as e:+ errors.append(f"Edge {idx}: {str(e)}")++ # Execute batch+ if not batch_operations:+ return {"edges_created": 0, "errors": errors}++ try:+ # Execute all operations in a single transaction+ self._connection.execute_batch(batch_operations)++ return {+ "edges_created": len(batch_operations),+ "errors": errors,+ }+ except Exception as e:+ errors.append(f"Batch execution failed: {str(e)}")+ return {"edges_created": 0, "errors": errors}+def update_node(self,element_id: str,@@ -507,50 +672,6 @@ class GraphRepository:)return result[0]["deleted_count"] if result else 0- def update_relationship(- self,- element_id: str,- properties: Dict[str, Any],- sketch_id: str,- ) -> Optional[Dict[str, Any]]:- """- Update an existing relationship's properties.-- Args:- element_id: Neo4j element ID of the edge to update- properties: Dictionary of properties to update- sketch_id: Investigation sketch ID (for safety)-- Returns:- Updated edge data or None if not found- """- if not self._connection:- return None-- # Filter out None values- filtered_props = {- k: (v if v is not None else "")- for k, v in properties.items()- if k is not None- }-- # Serialize properties- serialized_props = GraphSerializer.serialize_properties(filtered_props)-- # Build SET clauses- set_clauses = [f"r.{prop} = ${prop}" for prop in serialized_props.keys()]-- query = f"""- MATCH ()-[r]->()- WHERE elementId(r) = $element_id AND r.sketch_id = $sketch_id- SET {", ".join(set_clauses)}- RETURN elementId(r) as id, COALESCE(r.label, type(r)) as type, properties(r) as data- """-- params = {"element_id": element_id, "sketch_id": sketch_id, **serialized_props}- result = self._connection.query(query, params)- return result[0] if result else None-def delete_all_sketch_nodes(self, sketch_id: str) -> int:"""Delete all nodes and relationships for a sketch.@@ -565,7 +686,8 @@ class GraphRepository:return 0query = """- MATCH (n {sketch_id: $sketch_id})+ OPTIONAL MATCH (n {sketch_id: $sketch_id})+ WHERE n IS NOT NULLDETACH DELETE nRETURN count(n) as deleted_count"""@@ -590,9 +712,12 @@ class GraphRepository:return {"nodes": [], "relationships": []}# Get all nodes for the sketch+ # Use OPTIONAL MATCH to avoid Neo4j warning when sketch_id property doesn't exist yetnodes_query = """- MATCH (n)+ OPTIONAL MATCH (n)WHERE n.sketch_id = $sketch_id+ WITH n+ WHERE n IS NOT NULLRETURN elementId(n) as id, labels(n) as labels, properties(n) as dataLIMIT $limit"""@@ -610,16 +735,57 @@ class GraphRepository:UNWIND $node_ids AS nidMATCH (a)-[r]->(b)WHERE elementId(a) = nid AND elementId(b) IN $node_ids- RETURN elementId(r) as id,- COALESCE(r.label, type(r)) as type,- elementId(a) as source,- elementId(b) as target,- properties(r) as data+ RETURN elementId(r) as id, type(r) as type, elementId(a) as source,+ elementId(b) as target, properties(r) as data"""rels_result = self._connection.query(rels_query, {"node_ids": node_ids})return {"nodes": nodes_result, "relationships": rels_result or []}+ def update_relationship(+ self,+ element_id: str,+ properties: Dict[str, Any],+ sketch_id: str,+ ) -> Optional[Dict[str, Any]]:+ """+ Update an existing relationship's properties.++ Args:+ element_id: Neo4j element ID of the edge to update+ properties: Dictionary of properties to update+ sketch_id: Investigation sketch ID (for safety)++ Returns:+ Updated edge data or None if not found+ """+ if not self._connection:+ return None++ # Filter out None values+ filtered_props = {+ k: (v if v is not None else "")+ for k, v in properties.items()+ if k is not None+ }++ # Serialize properties+ serialized_props = GraphSerializer.serialize_properties(filtered_props)++ # Build SET clauses+ set_clauses = [f"r.{prop} = ${prop}" for prop in serialized_props.keys()]++ query = f"""+ MATCH ()-[r]->()+ WHERE elementId(r) = $element_id AND r.sketch_id = $sketch_id+ SET {", ".join(set_clauses)}+ RETURN elementId(r) as id, COALESCE(r.label, type(r)) as type, properties(r) as data+ """++ params = {"element_id": element_id, "sketch_id": sketch_id, **serialized_props}+ result = self._connection.query(query, params)+ return result[0] if result else None+def create_relationship_by_element_id(self,from_element_id: str,@@ -646,9 +812,6 @@ class GraphRepository:serialized_props = GraphSerializer.serialize_properties(properties)serialized_props["sketch_id"] = sketch_id- # Store label as a property so it can be updated later- if "label" not in serialized_props:- serialized_props["label"] = rel_typeprops_str = ", ".join([f"{k}: ${k}" for k in serialized_props.keys()])rel_props = f"{{{props_str}}}"@@ -670,7 +833,7 @@ class GraphRepository:return result[0]["rel"] if result else Nonedef query(- self, cypher: str, parameters: Dict[str, Any] = None+ self, cypher: str, parameters: Dict[str, Any] = {}) -> List[Dict[str, Any]]:"""Execute a custom Cypher query.@@ -881,7 +1044,7 @@ class GraphRepository:WHERE other.sketch_id = $sketch_idRETURNelementId(r) as rel_id,- COALESCE(r.label, type(r)) as rel_type,+ type(r) as rel_type,properties(r) as rel_data,elementId(other) as other_node_id,labels(other) as other_node_labels,@@ -894,7 +1057,7 @@ class GraphRepository:WHERE other.sketch_id = $sketch_idRETURNelementId(r) as rel_id,- COALESCE(r.label, type(r)) as rel_type,+ type(r) as rel_type,properties(r) as rel_data,elementId(other) as other_node_id,labels(other) as other_node_labels,@@ -971,6 +1134,48 @@ class GraphRepository:return {"nds": all_nodes, "rls": relationships}+ def count_nodes_by_sketch(self, sketch_id: str) -> int:+ """+ Count total number of nodes for a given sketch.++ Args:+ sketch_id: The sketch ID to count nodes for++ Returns:+ int: Total number of nodes+ """+ query = """+ OPTIONAL MATCH (n)+ WHERE n.sketch_id = $sketch_id AND n IS NOT NULL+ RETURN count(n) as total+ """++ with self._connection.get_driver().session() as session:+ result = session.run(query, sketch_id=sketch_id)+ record = result.single()+ return record["total"] if record else 0++ def count_edges_by_sketch(self, sketch_id: str) -> int:+ """+ Count total number of relationships/edges for a given sketch.++ Args:+ sketch_id: The sketch ID to count edges for++ Returns:+ int: Total number of relationships+ """+ query = """+ OPTIONAL MATCH (n)-[r]->(m)+ WHERE n.sketch_id = $sketch_id AND m.sketch_id = $sketch_id+ RETURN count(r) as total+ """++ with self._connection.get_driver().session() as session:+ result = session.run(query, sketch_id=sketch_id)+ record = result.single()+ return record["total"] if record else 0+def __enter__(self):"""Context manager entry."""return selfdiff --git a/flowsint-core/src/flowsint_core/core/graph_service.py b/flowsint-core/src/flowsint_core/core/graph_service.pyindex d622fa7c..7ba2c8be 100644--- a/flowsint-core/src/flowsint_core/core/graph_service.py+++ b/flowsint-core/src/flowsint_core/core/graph_service.py@@ -5,11 +5,13 @@ This module provides a service layer for graph operations,integrating repository and logging functionality."""-from typing import Dict, Any, Optional, Protocol, Union+from typing import Any, Dict, Optional, Protocol, Unionfrom uuid import UUID+from pydantic import BaseModel-from .graph_repository import GraphRepository+from .graph_db import Neo4jConnection+from .graph_repository import GraphRepositoryclass LoggerProtocol(Protocol):diff --git a/flowsint-core/src/flowsint_core/imports/__init__.py b/flowsint-core/src/flowsint_core/imports/__init__.pyindex 5638a9ab..02dfc77e 100644--- a/flowsint-core/src/flowsint_core/imports/__init__.py+++ b/flowsint-core/src/flowsint_core/imports/__init__.py@@ -3,12 +3,12 @@ Import utilities for entity parsing and type detection."""from .entity_detection import detect_type-from .file_parser import parse_import_file, FileParseResult, EntityPreview, _parse_txt+from .file_parser import FileParseResult, parse_import_file+from .types import EntityPreview__all__ = ["detect_type","parse_import_file","FileParseResult","EntityPreview",- "_parse_txt",]diff --git a/flowsint-core/src/flowsint_core/imports/entity_detection.py b/flowsint-core/src/flowsint_core/imports/entity_detection.pyindex b5fc90cf..e643fa89 100644--- a/flowsint-core/src/flowsint_core/imports/entity_detection.py+++ b/flowsint-core/src/flowsint_core/imports/entity_detection.py@@ -2,7 +2,9 @@Entity type detection utilities for import feature.Provides basic pattern matching for common entity types."""+from typing import Optional, Type+from flowsint_types import TYPE_REGISTRY, FlowsintTypediff --git a/flowsint-core/src/flowsint_core/imports/file_parser.py b/flowsint-core/src/flowsint_core/imports/file_parser.pyindex e443c77b..64abbb15 100644--- a/flowsint-core/src/flowsint_core/imports/file_parser.py+++ b/flowsint-core/src/flowsint_core/imports/file_parser.py@@ -4,44 +4,25 @@ Handles TXT file format only.Each line represents ONE entity with a single value."""-from dataclasses import dataclass-from typing import Any, Dict, List, BinaryIO, Union, Optionalfrom pathlib import Path-from .entity_detection import detect_type+from typing import BinaryIO, Optional, Union+from .json import parse_json+from .txt import parse_txt+from .types import FileParseResult-@dataclass-class EntityPreview:- """Preview of a single entity to be imported."""-- obj: Dict[str, Any]- detected_type: str---@dataclass-class Entity:- """Entity class"""-- type: str- results: List[EntityPreview]---@dataclass-class FileParseResult:- """Result of parsing an import file."""-- entities: Dict[str, Entity]- total_entities: int+ALLOWED_EXTENSIONS = [".txt", ".json"]def parse_import_file(- file_content: Union[bytes, BinaryIO],- filename: str,+ file_content: Optional[Union[bytes, BinaryIO]],+ filename: Optional[str],max_preview_rows: int = 100,-) -> FileParseResult:+) -> FileParseResult | None:"""- Parse an uploaded TXT file and analyze its contents.+ Parse an uploaded file and analyze its contents."""+file_ext = Path(filename).suffix.lower()# Convert file content to bytes if it's a file-like object@@ -50,92 +31,13 @@ def parse_import_file(else:file_bytes = file_content- # Only support TXT files- if file_ext != ".txt":+ # Only supported files+ if file_ext not in ALLOWED_EXTENSIONS:raise ValueError(- f"Unsupported file format: {file_ext}. Only .txt files are supported."- )-- return _parse_txt(file_bytes, max_preview_rows)---def _parse_txt(- file_bytes: bytes,- max_preview_rows: int,-) -> FileParseResult:- """Parse a TXT file where each line is an entity with a single string value."""- try:- # Try to decode as UTF-8, fall back to latin-1- try:- text_content = file_bytes.decode("utf-8")- except UnicodeDecodeError:- text_content = file_bytes.decode("latin-1")-- # Split by lines and filter empty lines- lines = [line.strip() for line in text_content.split("\n")]- lines = [line for line in lines if line]-- if not lines:- raise ValueError("File is empty")-- total_rows = len(lines)-- entities: Dict[str, Entity] = {}-- for _, line in enumerate(lines[:max_preview_rows]):- entity = _create_entity_preview(line)- if entity:- if entity.detected_type in entities:- entities[entity.detected_type].results.append(entity)- else:- entities[entity.detected_type] = Entity(- type=entity.detected_type, results=[entity]- )- return FileParseResult(- entities=entities,- total_entities=total_rows,+ f"Unsupported file format: {file_ext}. Only those files extensions are supported: {', '.join(ALLOWED_EXTENSIONS)} ")- except Exception as e:- # Normalize exceptions to ValueError for callers/tests- raise ValueError(f"Failed to parse TXT file: {str(e)}")---def _create_entity_preview(row_value: str) -> Optional[EntityPreview]:- """- Create an EntityPreview from a row of data.-- Args:- row_value: string value of the entry- Returns:- EntityPreview or None if row is invalid- """- # Detect entity type (detect_type may return a class with from_string or None)- DetectedType = detect_type(row_value)-- # If detection failed (None), provide a minimal Unknown type with from_string- if not DetectedType:-- class UnknownType:- @classmethod- def from_string(cls, value: str):- # fallback: return raw string or a minimal wrapper- return value-- DetectedType = UnknownType- detected_name = "Unknown"- else:- # If detect_type returned a class/type, try to get a readable name- detected_name = getattr(DetectedType, "__name__", str(DetectedType))-- # Build object using the type's from_string (expect classmethod)- try:- obj = DetectedType.from_string(row_value)- except Exception:- # On failure converting to object, return None to skip the row- return None-- return EntityPreview(- obj=obj,- detected_type=detected_name,- )+ if file_ext == ".txt":+ return parse_txt(file_bytes, max_preview_rows)+ elif file_ext in [".json"]:+ return parse_json(file_bytes, max_preview_rows)diff --git a/flowsint-core/src/flowsint_core/imports/json/__init__.py b/flowsint-core/src/flowsint_core/imports/json/__init__.pynew file mode 100644index 00000000..e222ec3f--- /dev/null+++ b/flowsint-core/src/flowsint_core/imports/json/__init__.py@@ -0,0 +1,5 @@+from .parse_json import parse_json++__all__ = [+ "parse_json",+]diff --git a/flowsint-core/src/flowsint_core/imports/json/parse_json.py b/flowsint-core/src/flowsint_core/imports/json/parse_json.pynew file mode 100644index 00000000..1869fc01--- /dev/null+++ b/flowsint-core/src/flowsint_core/imports/json/parse_json.py@@ -0,0 +1,137 @@+import json+from typing import Dict, List++from flowsint_types import TYPE_REGISTRY++from ..types import Edge, Entity, EntityPreview, FileParseResult+from ..utils import create_entity_preview++VALID_NODES_KEYS = ["nodes", "entities"]+VALID_EDGES_KEYS = ["edges", "links", "relations", "rels"]+VALID_EDGE_FROM_KEYS = ["from", "source"]+VALID_EDGE_TO_KEYS = ["to", "target"]+++def parse_json(+ file_bytes: bytes,+ max_preview_rows: int,+) -> FileParseResult:+ """Parse JSON files."""+ try:+ file_bytes = file_bytes.lstrip()+ entities: Dict[str, Entity] = {}+ my_bytes_value = file_bytes.decode().replace("'", '"')+ graph = json.loads(my_bytes_value)+ node_key = next((k for k in VALID_NODES_KEYS if k in graph), None)+ edge_key = next((k for k in VALID_EDGES_KEYS if k in graph), None)+ if node_key is None:+ raise Exception(+ f"No valid nodes keys found in JSON. Please provide one of : {', '.join(VALID_NODES_KEYS)}"+ )++ if edge_key is None:+ raise Exception(+ f"No valid edges keys found in JSON. Please provide one of : {', '.join(VALID_EDGES_KEYS)}"+ )+ nodes = _get_nodes(graph.get(node_key))+ edges = _get_edges(graph.get(edge_key))+ for node in nodes:+ if node:+ if node.detected_type in entities:+ entities[node.detected_type].results.append(node)+ else:+ entities[node.detected_type] = Entity(+ type=node.detected_type, results=[node]+ )+ edges_to_insert = []+ for edge in edges:+ label = edge.get("label", "IS_RELATED_TO")+ from_id = edge.get("source")+ to_id = edge.get("target")+ from_obj = next((node for node in nodes if node.node_id == from_id), None)+ to_obj = next((node for node in nodes if node.node_id == to_id), None)+ if from_obj and to_obj:+ edges_to_insert.append(+ Edge(+ from_obj=from_obj.obj,+ from_id=from_obj.node_id,+ to_obj=to_obj.obj,+ to_id=to_obj.node_id,+ label=label,+ )+ )+ return FileParseResult(+ entities=entities, edges=edges_to_insert, total_entities=0+ )+ except Exception as e:+ raise Exception(f"Invalid JSON: {str(e)}")+++def _get_nodes(nodes: List[Dict]) -> List[EntityPreview]:+ results = []+ for node in nodes:+ node_id = node.get("id")+ node_type = node.get("type", None)+ label = node.get("label", None)+ node_obj = {"type": node_type, "label": label, **node}+ preview = _parse_node(node_obj)+ if preview:+ results.append(+ EntityPreview(+ node_id=node_id,+ obj=preview.obj,+ detected_type=preview.detected_type,+ )+ )++ return results+++def _parse_node(nodeDict: dict) -> EntityPreview | None:+ type = nodeDict.get("type")+ first_key = next(iter(nodeDict), None)+ label = (+ nodeDict.get("label")+ or (nodeDict.get(first_key) if first_key else None)+ or nodeDict.get("id")+ )+ if label is None:+ return None+ if not type:+ return create_entity_preview(label)+ # Try to find a matching type in TYPE_REGISTRY+ DetectedType = TYPE_REGISTRY.get_lowercase(type.lower())+ if not DetectedType:+ # Type not recognized, fall back to generic+ return create_entity_preview(label)++ node = DetectedType.from_string(label)++ detected_type = DetectedType.__name__+ return EntityPreview(obj=node, detected_type=detected_type)+++def _get_edges(edges: List[Dict]) -> List[Dict]:+ results = []+ for edge in edges:+ source_key = next((k for k in VALID_EDGE_FROM_KEYS if k in edge), None)+ target_key = next((k for k in VALID_EDGE_TO_KEYS if k in edge), None)+ if source_key is None:+ print(+ f"No valid source key found in edge object. Please provide one of : {', '.join(VALID_EDGE_FROM_KEYS)}"+ )+ if target_key is None:+ print(+ f"No valid target key found in edge object. Please provide one of : {', '.join(VALID_EDGE_TO_KEYS)}"+ )+ source = edge.get(source_key, None)+ target = edge.get(target_key, None)+ label = edge.get("label", None)+ edge_obj = {+ "source": source,+ "target": target,+ "label": label if label else "IS_RELATED_TO",+ }+ results.append(edge_obj)++ return resultsdiff --git a/flowsint-core/src/flowsint_core/imports/txt/__init__.py b/flowsint-core/src/flowsint_core/imports/txt/__init__.pynew file mode 100644index 00000000..2fcfe669--- /dev/null+++ b/flowsint-core/src/flowsint_core/imports/txt/__init__.py@@ -0,0 +1,5 @@+from .parse_txt import parse_txt++__all__ = [+ "parse_txt",+]diff --git a/flowsint-core/src/flowsint_core/imports/txt/parse_txt.py b/flowsint-core/src/flowsint_core/imports/txt/parse_txt.pynew file mode 100644index 00000000..de6ab69b--- /dev/null+++ b/flowsint-core/src/flowsint_core/imports/txt/parse_txt.py@@ -0,0 +1,44 @@+from typing import Dict++from ..types import Entity, FileParseResult+from ..utils import create_entity_preview+++def parse_txt(+ file_bytes: bytes,+ max_preview_rows: int,+) -> FileParseResult:+ """Parse a TXT file where each line is an entity with a single string value."""+ try:+ # Try to decode as UTF-8, fall back to latin-1+ try:+ text_content = file_bytes.decode("utf-8")+ except UnicodeDecodeError:+ text_content = file_bytes.decode("latin-1")++ # Split by lines and filter empty lines+ lines = [line.strip() for line in text_content.split("\n")]+ lines = [line for line in lines if line]++ if not lines:+ raise ValueError("File is empty")++ entities: Dict[str, Entity] = {}++ for _, line in enumerate(lines[:max_preview_rows]):+ entity = create_entity_preview(line)+ if entity:+ if entity.detected_type in entities:+ entities[entity.detected_type].results.append(entity)+ else:+ entities[entity.detected_type] = Entity(+ type=entity.detected_type, results=[entity]+ )+ return FileParseResult(+ entities=entities,+ total_entities=len(lines),+ )++ except Exception as e:+ # Normalize exceptions to ValueError for callers/tests+ raise ValueError(f"Failed to parse TXT file: {str(e)}")diff --git a/flowsint-core/src/flowsint_core/imports/types.py b/flowsint-core/src/flowsint_core/imports/types.pynew file mode 100644index 00000000..d2768f6c--- /dev/null+++ b/flowsint-core/src/flowsint_core/imports/types.py@@ -0,0 +1,39 @@+from dataclasses import dataclass, field+from typing import Any, Dict, List, Optional+++@dataclass+class EntityPreview:+ """Preview of a single entity to be imported."""++ obj: Dict[str, Any]+ detected_type: str+ node_id: Optional[str] = None+++@dataclass+class Entity:+ """Entity class"""++ type: str+ results: List[EntityPreview]+++@dataclass+class Edge:+ """Edge class"""++ from_obj: Dict[str, Any]+ to_obj: Dict[str, Any]+ from_id: str | None+ to_id: str | None+ label: str+++@dataclass+class FileParseResult:+ """Result of parsing an import file."""++ entities: Dict[str, Entity]+ total_entities: int+ edges: Optional[List[Edge]] = field(default_factory=list)diff --git a/flowsint-core/src/flowsint_core/imports/utils.py b/flowsint-core/src/flowsint_core/imports/utils.pynew file mode 100644index 00000000..8ac20b50--- /dev/null+++ b/flowsint-core/src/flowsint_core/imports/utils.py@@ -0,0 +1,47 @@+import re+from typing import Optional++from .entity_detection import detect_type+from .types import EntityPreview+++def camel_to_screaming_snake(name):+ s1 = re.sub(r"(?<!^)(?=[A-Z])", "_", name)+ return s1.upper()+++def create_entity_preview(row_value: str) -> Optional[EntityPreview]:+ """+ Create an EntityPreview from a row of data.++ Args:+ row_value: string value of the entry+ Returns:+ EntityPreview or None if row is invalid+ """+ # Detect entity type (detect_type may return a class with from_string or None)+ DetectedType = detect_type(row_value)+ # If detection failed (None), provide a minimal Unknown type with from_string+ if not DetectedType:++ class UnknownType:+ @classmethod+ def from_string(cls, value: str):+ # fallback: return raw string or a minimal wrapper+ return value++ DetectedType = UnknownType+ detected_name = "Unknown"+ else:+ # If detect_type returned a class/type, try to get a readable name+ detected_name = getattr(DetectedType, "__name__", str(DetectedType))+ # Build object using the type's from_string (expect classmethod)+ try:+ obj = DetectedType.from_string(row_value)+ except Exception:+ # On failure converting to object, return None to skip the row+ return None+ return EntityPreview(+ obj=obj,+ detected_type=detected_name,+ )diff --git a/flowsint-core/tests/import/json/test_json_import.py b/flowsint-core/tests/import/json/test_json_import.pynew file mode 100644index 00000000..a1930e07--- /dev/null+++ b/flowsint-core/tests/import/json/test_json_import.py@@ -0,0 +1,67 @@+"""Comprehensive tests for JSON import functionality."""++from flowsint_core.imports.json.parse_json import parse_json++# Standard JSON graph test data (node-link format)+standard_json = b"""{+ "nodes": [+ {"id": "1", "label": "Alice Dupont", "type": "individual"},+ {"id": "2", "label": "Bob", "type": "individual"},+ {"id": "3", "label": "Charlie", "type": "individual"}+ ],+ "edges": [+ {"source": "1", "target": "2", "label": "KNOWS"},+ {"source": "2", "target": "3", "label": "WORKS_WITH"}+ ]+}"""++# Standard JSON with just ids+standard_json_without_label = b"""{+ "nodes": [+ {"id": "Myriel"},+ {"id": "Napoleon"},+ {"id": "Baptistine"}+ ],+ "edges": [+ {"source": "Napoleon", "target": "Myriel"},+ {"source": "Mlle.Baptistine", "target": "Myriel"},+ {"source": "Napoleon", "target": "Baptistine"}+ ]+}"""+# JSON with properties+json_with_properties = b"""{+ "nodes": [+ {+ "id": "1",+ "label": "example.com",+ "type": "domain",+ "data": {+ "username": "john.doe",+ "domain": "example.com"+ }+ },+ {+ "id": "2",+ "label": "example.com",+ "type": "Domain",+ "data": {+ "tld": "com"+ }+ }+ ],+ "edges": [+ {"source": "1", "target": "2", "label": "BELONGS_TO"}+ ]+}"""+++def test_standard_json_import():+ """Test basic standard JSON import."""+ results = parse_json(standard_json, max_preview_rows=100)+ assert "Individual" in results.entities+++def test_standard_json_import_without_label():+ """Test basic standard JSON import."""+ results = parse_json(standard_json_without_label, max_preview_rows=100)+ assert "Username" in results.entitiesdiff --git a/flowsint-core/tests/import/test_entity_detection.py b/flowsint-core/tests/import/test_entity_detection.pyindex 533e93e4..48f9b462 100644--- a/flowsint-core/tests/import/test_entity_detection.py+++ b/flowsint-core/tests/import/test_entity_detection.py@@ -1,4 +1,5 @@from flowsint_types import Domain, Ip, Phone+from flowsint_core.imports import detect_typediff --git a/flowsint-core/tests/import/test_file_parser.py b/flowsint-core/tests/import/test_file_parser.pyindex 6fe3559c..41f91b5b 100644--- a/flowsint-core/tests/import/test_file_parser.py+++ b/flowsint-core/tests/import/test_file_parser.py@@ -1,15 +1,15 @@-from flowsint_types import Domain, Ip, Usernamefrom unittest.mock import patch+from flowsint_types import Domain, Ip, Username+from flowsint_core.imports import (- parse_import_file,- FileParseResult,EntityPreview,+ FileParseResult,+ parse_import_file,)def test_parse_import_file_txt_basic_with_mocked_detect_type():-txt_bytes = b"domain.com\nblog.domain.com\n12.34.56.78\nmy_super_username"result = parse_import_file(txt_bytes, "values.txt", max_preview_rows=100)diff --git a/flowsint-core/tests/import/txt/test_txt_import.py b/flowsint-core/tests/import/txt/test_txt_import.pynew file mode 100644index 00000000..833830c2--- /dev/null+++ b/flowsint-core/tests/import/txt/test_txt_import.py@@ -0,0 +1,479 @@+"""Comprehensive tests for TXT import functionality.++Tests automatic entity detection and parsing of plain text files.+"""++import pytest+from flowsint_core.imports.txt.parse_txt import parse_txt+from flowsint_types import CIDR, Domain, Email, Ip, Phone, Username, Website++from flowsint_core.imports import parse_import_file+++class TestBasicTxtParsing:+ """Tests for basic TXT file parsing functionality."""++ def test_simple_txt_file(self):+ """Test parsing a simple TXT file with basic entities."""+ txt_content = b"example.com\n192.168.1.1\[email protected]"++ result = parse_txt(txt_content, max_preview_rows=100)++ assert result.total_entities == 3+ assert len(result.entities) > 0++ def test_single_line(self):+ """Test parsing a file with a single line."""+ txt_content = b"example.com"++ result = parse_txt(txt_content, max_preview_rows=100)++ assert result.total_entities == 1++ def test_multiple_same_type(self):+ """Test parsing multiple entities of the same type."""+ txt_content = b"example.com\ntest.com\ndomain.org"++ result = parse_txt(txt_content, max_preview_rows=100)++ assert result.total_entities == 3+ if "Domain" in result.entities:+ assert len(result.entities["Domain"].results) == 3++ def test_mixed_entity_types(self):+ """Test parsing file with mixed entity types."""+ txt_content = b"""example.com+192.168.1.1++1-555-0100+https://example.com"""++ result = parse_txt(txt_content, max_preview_rows=100)++ assert result.total_entities == 5+ # Should have multiple entity types+ assert len(result.entities) >= 3+++class TestWhitespaceHandling:+ """Tests for handling whitespace and line formatting."""++ def test_lines_with_trailing_whitespace(self):+ """Test that trailing whitespace is stripped."""+ txt_content = b"example.com \ntest.com\t\n"++ result = parse_txt(txt_content, max_preview_rows=100)++ assert result.total_entities == 2++ def test_lines_with_leading_whitespace(self):+ """Test that leading whitespace is stripped."""+ txt_content = b" example.com\n\ttest.com"++ result = parse_txt(txt_content, max_preview_rows=100)++ assert result.total_entities == 2++ def test_empty_lines_ignored(self):+ """Test that empty lines are filtered out."""+ txt_content = b"""example.com++test.com+++domain.org+"""++ result = parse_txt(txt_content, max_preview_rows=100)++ # Only 3 non-empty lines+ assert result.total_entities == 3++ def test_whitespace_only_lines(self):+ """Test that whitespace-only lines are ignored."""+ txt_content = b"""example.com++\t+test.com+"""++ result = parse_txt(txt_content, max_preview_rows=100)++ assert result.total_entities == 2++ def test_different_line_endings(self):+ """Test handling of different line ending styles."""+ # Unix-style (LF)+ unix_content = b"example.com\ntest.com\ndomain.org"+ result_unix = parse_txt(unix_content, max_preview_rows=100)+ assert result_unix.total_entities == 3++ # Windows-style (CRLF)+ windows_content = b"example.com\r\ntest.com\r\ndomain.org"+ result_windows = parse_txt(windows_content, max_preview_rows=100)+ assert result_windows.total_entities == 3++ # Mixed+ mixed_content = b"example.com\ntest.com\r\ndomain.org"+ result_mixed = parse_txt(mixed_content, max_preview_rows=100)+ assert result_mixed.total_entities == 3+++class TestEncodingHandling:+ """Tests for handling different text encodings."""++ def test_utf8_encoding(self):+ """Test parsing UTF-8 encoded file."""+ txt_content = "example.com\ntest.com".encode("utf-8")++ result = parse_txt(txt_content, max_preview_rows=100)++ assert result.total_entities == 2++ def test_latin1_encoding(self):+ """Test parsing Latin-1 encoded file."""+ txt_content = "example.com\ntest.com".encode("latin-1")++ result = parse_txt(txt_content, max_preview_rows=100)++ assert result.total_entities == 2++ def test_utf8_with_special_chars(self):+ """Test UTF-8 file with special characters (should still parse entities)."""+ # Include some special characters but also valid entities+ txt_content = "example.com\ntest.com\nété.com".encode("utf-8")++ result = parse_txt(txt_content, max_preview_rows=100)++ # At minimum, the valid domains should parse+ assert result.total_entities >= 2+++class TestEntityTypeDetection:+ """Tests for automatic entity type detection."""++ def test_domain_detection(self):+ """Test detection of domain entities."""+ txt_content = b"example.com\ntest.org\nsubdomain.example.net"++ result = parse_txt(txt_content, max_preview_rows=100)++ assert "Domain" in result.entities+ assert len(result.entities["Domain"].results) == 3++ def test_ip_detection(self):+ """Test detection of IP address entities."""+ txt_content = b"192.168.1.1\n10.0.0.1\n8.8.8.8"++ result = parse_txt(txt_content, max_preview_rows=100)++ assert "Ip" in result.entities+ assert len(result.entities["Ip"].results) == 3++ def test_email_detection(self):+ """Test detection of email entities."""++ result = parse_txt(txt_content, max_preview_rows=100)++ assert "Email" in result.entities+ assert len(result.entities["Email"].results) == 3++ def test_phone_detection(self):+ """Test detection of phone number entities."""+ txt_content = b"+1-555-0100\n555-0200\n+44-20-1234-5678"++ result = parse_txt(txt_content, max_preview_rows=100)++ # Phone detection might be present+ assert result.total_entities >= 1++ def test_url_detection(self):+ """Test detection of URL/Website entities."""+ txt_content = (+ b"https://example.com\nhttp://test.org\nhttps://subdomain.example.net/path"+ )++ result = parse_txt(txt_content, max_preview_rows=100)++ # URLs should be detected as Website or similar+ assert result.total_entities == 3++ def test_cidr_detection(self):+ """Test detection of CIDR notation."""+ txt_content = b"192.168.1.0/24\n10.0.0.0/8"++ result = parse_txt(txt_content, max_preview_rows=100)++ # CIDR should be detected+ assert result.total_entities >= 1++ def test_username_detection(self):+ """Test detection of username entities."""+ txt_content = b"my_username\nuser123\ntest_user_name"++ result = parse_txt(txt_content, max_preview_rows=100)++ # Usernames should be detected+ assert result.total_entities >= 1+++class TestMaxPreviewRows:+ """Tests for max_preview_rows parameter."""++ def test_respects_max_preview_rows(self):+ """Test that max_preview_rows limits entities in preview."""+ # Create 100 lines+ lines = [f"domain{i}.com" for i in range(100)]+ txt_content = "\n".join(lines).encode("utf-8")++ result = parse_txt(txt_content, max_preview_rows=10)++ # Total should be 100, but entities dict should only have 10+ assert result.total_entities == 100+ total_in_entities = sum(len(e.results) for e in result.entities.values())+ assert total_in_entities == 10++ def test_max_preview_rows_larger_than_file(self):+ """Test when max_preview_rows is larger than file content."""+ txt_content = b"example.com\ntest.com\ndomain.org"++ result = parse_txt(txt_content, max_preview_rows=1000)++ assert result.total_entities == 3+ total_in_entities = sum(len(e.results) for e in result.entities.values())+ assert total_in_entities == 3++ def test_max_preview_rows_zero(self):+ """Test with max_preview_rows=0."""+ txt_content = b"example.com\ntest.com\ndomain.org"++ result = parse_txt(txt_content, max_preview_rows=0)++ # Should still count total but not include in entities+ assert result.total_entities == 3+ # Entities dict should be empty or very small+ total_in_entities = sum(len(e.results) for e in result.entities.values())+ assert total_in_entities == 0+++class TestEdgeCases:+ """Tests for edge cases and error handling."""++ def test_empty_file(self):+ """Test handling of empty file."""+ txt_content = b""++ with pytest.raises(ValueError) as exc_info:+ parse_txt(txt_content, max_preview_rows=100)++ assert "empty" in str(exc_info.value).lower()++ def test_file_with_only_whitespace(self):+ """Test handling of file with only whitespace."""+ txt_content = b" \n\t\n \n"++ with pytest.raises(ValueError) as exc_info:+ parse_txt(txt_content, max_preview_rows=100)++ assert "empty" in str(exc_info.value).lower()++ def test_file_with_only_newlines(self):+ """Test handling of file with only newlines."""+ txt_content = b"\n\n\n\n"++ with pytest.raises(ValueError) as exc_info:+ parse_txt(txt_content, max_preview_rows=100)++ assert "empty" in str(exc_info.value).lower()++ def test_unknown_entity_types(self):+ """Test handling of unrecognized entity types."""+ txt_content = b"example.com\nrandom_text_12345\ntest.com"++ result = parse_txt(txt_content, max_preview_rows=100)++ # Should still parse recognized entities+ # Unknown types might be included or skipped+ assert result.total_entities == 3++ def test_malformed_entities(self):+ """Test handling of malformed entity strings."""+ txt_content = b"example.com\n@@@invalid@@@\ntest.com"++ result = parse_txt(txt_content, max_preview_rows=100)++ # Should parse valid entities, skip invalid ones+ assert result.total_entities == 3++ def test_very_long_lines(self):+ """Test handling of very long lines."""+ long_domain = "subdomain." * 100 + "example.com"+ txt_content = f"{long_domain}\ntest.com".encode("utf-8")++ result = parse_txt(txt_content, max_preview_rows=100)++ # Should handle long lines without crashing+ assert result.total_entities == 2++ def test_binary_content_as_text(self):+ """Test handling of binary content."""+ # Binary data that's not valid text+ binary_content = bytes([0x00, 0x01, 0x02, 0xFF, 0xFE])++ # Should handle gracefully (might decode as latin-1 or raise error)+ try:+ result = parse_txt(binary_content, max_preview_rows=100)+ # If it doesn't raise, check it handles it+ assert isinstance(result.total_entities, int)+ except ValueError:+ # Also acceptable to raise ValueError+ pass+++class TestIntegration:+ """Integration tests with parse_import_file."""++ def test_parse_import_file_txt(self):+ """Test full import pipeline with TXT format."""+ txt_content = b"example.com\n192.168.1.1\[email protected]"+ file_name = "test.txt"++ result = parse_import_file(txt_content, file_name, max_preview_rows=100)++ assert result.total_entities == 3+ assert len(result.entities) >= 2++ def test_parse_import_file_comprehensive(self):+ """Test comprehensive TXT import with various entity types."""+ txt_content = b"""example.com+test.org+192.168.1.1+10.0.0.1+https://example.com+http://test.org+my_username+test_user"""++ file_name = "comprehensive.txt"+ result = parse_import_file(txt_content, file_name, max_preview_rows=100)++ assert result.total_entities == 10+ # Should have detected multiple types+ assert len(result.entities) >= 3+++class TestEntityPreviewObjects:+ """Tests for EntityPreview objects created from TXT parsing."""++ def test_domain_entity_preview(self):+ """Test that Domain entities are properly created."""+ txt_content = b"example.com"++ result = parse_txt(txt_content, max_preview_rows=100)++ if "Domain" in result.entities:+ entity_preview = result.entities["Domain"].results[0]+ assert entity_preview.detected_type == "Domain"+ assert isinstance(entity_preview.obj, Domain)+ assert entity_preview.obj.domain == "example.com"++ def test_ip_entity_preview(self):+ """Test that IP entities are properly created."""+ txt_content = b"192.168.1.1"++ result = parse_txt(txt_content, max_preview_rows=100)++ if "Ip" in result.entities:+ entity_preview = result.entities["Ip"].results[0]+ assert entity_preview.detected_type == "Ip"+ assert isinstance(entity_preview.obj, Ip)+ assert entity_preview.obj.address == "192.168.1.1"++ def test_email_entity_preview(self):+ """Test that Email entities are properly created."""+ txt_content = b"[email protected]"++ result = parse_txt(txt_content, max_preview_rows=100)++ if "Email" in result.entities:+ entity_preview = result.entities["Email"].results[0]+ assert entity_preview.detected_type == "Email"+ assert isinstance(entity_preview.obj, Email)+ assert entity_preview.obj.email == "[email protected]"+++class TestRealWorldScenarios:+ """Tests simulating real-world usage scenarios."""++ def test_ioc_list(self):+ """Test parsing an IOC (Indicators of Compromise) list."""+ ioc_content = b"""# Malicious domains+malware.com+badactor.org+phishing-site.net++# Malicious IPs+198.51.100.1+203.0.113.5++# Command and control URLs+https://c2.malware.com/callback+http://evil.org/payload+"""++ result = parse_txt(ioc_content, max_preview_rows=100)++ # Comments and empty lines should be filtered+ # Should detect domains, IPs, and URLs+ assert result.total_entities >= 7++ def test_asset_inventory(self):+ """Test parsing an asset inventory list."""+ asset_content = b"""web-server-1.company.com+web-server-2.company.com+db-server.company.com+10.0.1.10+10.0.1.11+10.0.2.5+"""++ result = parse_txt(asset_content, max_preview_rows=100)++ assert result.total_entities == 6+ # Should have both domains and IPs+ assert len(result.entities) >= 2++ def test_email_list(self):+ """Test parsing an email distribution list."""+ email_content = b"""[email protected]+"""++ result = parse_txt(email_content, max_preview_rows=100)++ assert result.total_entities == 5+ if "Email" in result.entities:+ assert len(result.entities["Email"].results) == 5++ def test_mixed_investigation_data(self):+ """Test parsing mixed data from an investigation."""+ investigation_content = b"""[email protected]+192.168.1.100+suspicious-domain.com+https://malicious-site.org/payload++1-555-0123+backup-server.company.internal+10.0.0.50+"""++ result = parse_txt(investigation_content, max_preview_rows=100)++ # Should detect multiple entity types+ assert result.total_entities == 7+ assert len(result.entities) >= 3