A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat(core): type registry + better import
dextmorgn95add9c4fd79
diff --git a/flowsint-core/src/flowsint_core/core/graph_repository.py b/flowsint-core/src/flowsint_core/core/graph_repository.pyindex 8377e223..e12ae02b 100644--- a/flowsint-core/src/flowsint_core/core/graph_repository.py+++ b/flowsint-core/src/flowsint_core/core/graph_repository.py@@ -9,6 +9,7 @@ from typing import Dict, Any, List, Optional, Tuplefrom datetime import datetime, timezonefrom .graph_db import Neo4jConnectionfrom .graph_serializer import GraphSerializer+from flowsint_types.registry import get_typeclass GraphRepository:@@ -31,6 +32,36 @@ class GraphRepository:self._batch_operations: List[Tuple[str, Dict[str, Any]]] = []self._batch_size = 100+ @staticmethod+ def _get_primary_field_for_type(type_name: str) -> Optional[str]:+ """+ Get the primary field name for a given type.++ Args:+ type_name: The type name (e.g., "Ip", "Domain", or lowercase variants)++ Returns:+ The primary field name, or None if type not found+ """+ # Get the type class from registry+ type_class = get_type(type_name, case_sensitive=False)+ if not type_class:+ return None++ # Find the primary field (marked with json_schema_extra={"primary": True})+ model_fields = type_class.model_fields+ for field_name, field_info in model_fields.items():+ if field_info.json_schema_extra and field_info.json_schema_extra.get("primary"):+ return field_name++ # Fallback: use first required field or first field+ for field_name, field_info in model_fields.items():+ if field_info.is_required():+ return field_name++ # Last resort: first field+ return next(iter(model_fields.keys())) if model_fields else None+def create_node(self,node_type: str,@@ -264,14 +295,14 @@ class GraphRepository:**properties: Any) -> Optional[str]:"""- Create a node from import operation (MERGE on sketch_id + label).+ Create a node from import operation using the type's primary key.- This method is specifically designed for bulk imports where nodes- are identified by their label within a sketch, rather than by- a type-specific key property.+ This method uses the same MERGE strategy as enrichers: it identifies nodes+ by their primary key field (e.g., "address" for IP, "domain" for Domain)+ combined with sketch_id, preventing duplicates between imports and enrichers.Args:- node_type: Node label (e.g., "Domain", "Ip")+ node_type: Node label (e.g., "Domain", "Ip") - will be normalized to lowercaselabel: Human-readable label for the nodesketch_id: Investigation sketch ID**properties: All additional node properties@@ -282,25 +313,47 @@ class GraphRepository:if not self._connection:return None+ # Normalize node_type to lowercase (same as enrichers)+ node_type = node_type.lower()++ # Get the primary key field for this type+ key_prop = self._get_primary_field_for_type(node_type)+ if not key_prop:+ # Fallback: if type not found in registry, use "label" as key+ key_prop = "label"+ key_value = label+ else:+ # Extract the primary key value from properties+ key_value = properties.get(key_prop)+ if not key_value:+ # Fallback: if primary key not in properties, use label+ key_value = label+# Serialize and prepare all propertiesserialized_props = GraphSerializer.serialize_properties(properties)serialized_props["type"] = node_type.lower()+ serialized_props["sketch_id"] = sketch_idserialized_props["label"] = labelserialized_props["caption"] = label- serialized_props["created_at"] = datetime.now(timezone.utc).isoformat()- query = f"""- MERGE (n:`{node_type}` {{sketch_id: $sketch_id, label: $label}})- ON CREATE SET n += $props- RETURN elementId(n) as id- """+ # Build SET clauses (exclude sketch_id as it's in MERGE, but include key_prop)+ set_clauses = [f"n.{prop} = ${prop}" for prop in serialized_props.keys()+ if prop != "sketch_id"]params = {- "sketch_id": sketch_id,- "label": label,- "props": serialized_props+ key_prop: key_value,+ "created_at": datetime.now(timezone.utc).isoformat(),+ **serialized_props}+ # MERGE on primary key + sketch_id (same as enrichers)+ query = f"""+ MERGE (n:{node_type} {{{key_prop}: ${key_prop}, sketch_id: $sketch_id}})+ ON CREATE SET n.created_at = $created_at+ SET {', '.join(set_clauses)}+ RETURN elementId(n) as id+ """+result = self._connection.query(query, params)return result[0]["id"] if result else Nonediff --git a/flowsint-core/src/flowsint_core/core/registry.py b/flowsint-core/src/flowsint_core/core/registry.pyindex 59972bfe..cff123e0 100644--- a/flowsint-core/src/flowsint_core/core/registry.py+++ b/flowsint-core/src/flowsint_core/core/registry.py@@ -58,6 +58,10 @@ from flowsint_enrichers.individual.to_org import IndividualToOrgEnricher# Integration enrichersfrom flowsint_enrichers.n8n.connector import N8nConnector+# Dummy+from flowsint_enrichers.ip.to_dummy_domains import IpToDummyDomainsEnricher+from flowsint_enrichers.domain.to_dummy_ip import DomainToDummyIpEnricher+class EnricherRegistry:@@ -197,3 +201,7 @@ EnricherRegistry.register(IndividualToDomainsEnricher)# Integration enrichersEnricherRegistry.register(N8nConnector)++# Dummy+EnricherRegistry.register(IpToDummyDomainsEnricher)+EnricherRegistry.register(DomainToDummyIpEnricher)diff --git a/flowsint-core/src/flowsint_core/imports/__init__.py b/flowsint-core/src/flowsint_core/imports/__init__.pyindex 0c77f5cf..5638a9ab 100644--- a/flowsint-core/src/flowsint_core/imports/__init__.py+++ b/flowsint-core/src/flowsint_core/imports/__init__.py@@ -2,16 +2,13 @@Import utilities for entity parsing and type detection."""-from .entity_detection import detect_entity_type, get_default_label-from .file_parser import parse_file, FileParseResult, EntityPreview-from .type_matcher import detect_entity_type_from_row, get_primary_identifier+from .entity_detection import detect_type+from .file_parser import parse_import_file, FileParseResult, EntityPreview, _parse_txt__all__ = [- "detect_entity_type",- "get_default_label",- "parse_file",+ "detect_type",+ "parse_import_file","FileParseResult","EntityPreview",- "detect_entity_type_from_row",- "get_primary_identifier",+ "_parse_txt",]diff --git a/flowsint-core/src/flowsint_core/imports/entity_detection.py b/flowsint-core/src/flowsint_core/imports/entity_detection.pyindex 0cbedfdc..b5fc90cf 100644--- a/flowsint-core/src/flowsint_core/imports/entity_detection.py+++ b/flowsint-core/src/flowsint_core/imports/entity_detection.py@@ -2,130 +2,12 @@Entity type detection utilities for import feature.Provides basic pattern matching for common entity types."""+from typing import Optional, Type+from flowsint_types import TYPE_REGISTRY, FlowsintType-import re-from typing import Optional-import ipaddress---# Ordered registry of detectors. Order matters to avoid false positives-# (e.g., URL should be checked before Domain).-# Each entry is a tuple of (entity_type, predicate_function)-DETECTORS = [- ("Email", lambda v: is_email(v)),- ("IP", lambda v: is_ip_address(v)),- ("Website", lambda v: is_website(v)),- ("Domain", lambda v: is_domain(v)),- ("Phone", lambda v: is_phone(v)),- ("ASN", lambda v: is_asn(v)),- ("Username", lambda v: is_username(v)),-]---def detect_entity_type(value: str) -> Optional[str]:- """- Detect entity type based on value pattern.- Returns the detected entity type or None if no match.-- Args:- value: The string value to analyze-- Returns:- Entity type string (e.g., "Email", "Domain", "IP") or None- """- if not value or not isinstance(value, str):- return None-- value = value.strip()-- # Iterate through ordered detectors to find the first match- for entity_type, predicate in DETECTORS:- try:- if predicate(value):- return entity_type- except Exception:- # If a predicate raises, skip it to avoid breaking detection- # (predicates should generally be safe and return bool)- continue+def detect_type(value: str) -> Optional[Type[FlowsintType]]:+ for model in TYPE_REGISTRY.all_types().values():+ if hasattr(model, "detect") and model.detect(value):+ return modelreturn None---def is_email(value: str) -> bool:- """Check if value matches email pattern."""- email_pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'- return bool(re.match(email_pattern, value))---def is_domain(value: str) -> bool:- """Check if value matches domain pattern."""- # Basic domain pattern: contains dots and valid characters- domain_pattern = r'^(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}$'- return bool(re.match(domain_pattern, value))---def is_ip_address(value: str) -> bool:- """Check if value is a valid IPv4 or IPv6 address."""- try:- ipaddress.ip_address(value)- return True- except ValueError:- return False---def is_website(value: str) -> bool:- """Check if value matches URL/website pattern."""- url_pattern = r'^https?://'- return bool(re.match(url_pattern, value, re.IGNORECASE))---def is_phone(value: str) -> bool:- """Check if value matches phone number pattern."""- # Remove common separators for checking- cleaned = re.sub(r'[\s\-\(\)\.]', '', value)- # Check if it's mostly digits and has reasonable length- phone_pattern = r'^\+?[0-9]{7,15}$'- return bool(re.match(phone_pattern, cleaned))---def is_asn(value: str) -> bool:- """Check if value matches ASN pattern."""- asn_pattern = r'^AS\d+$'- return bool(re.match(asn_pattern, value, re.IGNORECASE))---def is_username(value: str) -> bool:- """Check if value matches username pattern (social media style)."""- # Matches @username format or simple alphanumeric with underscores- username_pattern = r'^@?[a-zA-Z0-9_]{3,30}$'- if re.match(username_pattern, value):- # Additional check: starts with @ or is not purely numeric- return value.startswith('@') or not value.lstrip('@').isdigit()- return False---def get_default_label(entity_type: str, value: str) -> str:- """- Get default label for an entity based on its type and value.-- Args:- entity_type: The detected or selected entity type- value: The entity value-- Returns:- Default label string- """- # For most types, the value itself is a good label- type_defaults = {- "Email": value,- "Domain": value,- "IP": value,- "Website": value,- "Phone": value,- "ASN": value,- "Username": value.lstrip('@'), # Remove @ prefix for label- "Organization": value,- "Individual": value,- }-- return type_defaults.get(entity_type, value)diff --git a/flowsint-core/src/flowsint_core/imports/file_parser.py b/flowsint-core/src/flowsint_core/imports/file_parser.pyindex 84f85596..e443c77b 100644--- a/flowsint-core/src/flowsint_core/imports/file_parser.py+++ b/flowsint-core/src/flowsint_core/imports/file_parser.py@@ -1,81 +1,46 @@"""File parsing utilities for entity imports.-Handles CSV, TXT, and XLSX file formats.-Each row represents ONE entity with all columns as data properties.+Handles TXT file format only.+Each line represents ONE entity with a single value."""-import csv-import iofrom dataclasses import dataclass-from typing import List, Dict, Any, BinaryIO, Union, Optional+from typing import Any, Dict, List, BinaryIO, Union, Optionalfrom pathlib import Path--try:- from openpyxl import load_workbook-- OPENPYXL_AVAILABLE = True-except ImportError:- OPENPYXL_AVAILABLE = False--from .type_matcher import detect_entity_type_from_row, get_primary_identifier+from .entity_detection import detect_type@dataclassclass EntityPreview:"""Preview of a single entity to be imported."""- row_index: int- """Index of the row (0-based)."""-- data: Dict[str, Any]- """All column data as key-value pairs."""-+ obj: Dict[str, Any]detected_type: str- """Detected entity type (e.g., "Email", "Domain")."""- primary_value: str- """Primary identifier value for the entity."""- confidence: str- """Confidence level: high, medium, low."""+@dataclass+class Entity:+ """Entity class"""++ type: str+ results: List[EntityPreview]@dataclassclass FileParseResult:"""Result of parsing an import file."""- entities: List[EntityPreview]- """List of detected entities from the file."""-+ entities: Dict[str, Entity]total_entities: int- """Total number of entities (rows)."""-- type_distribution: Dict[str, int]- """Distribution of detected types: {type_name: count}."""-- columns: List[str]- """Column names found in the file (for reference)."""-def parse_file(+def parse_import_file(file_content: Union[bytes, BinaryIO],filename: str,max_preview_rows: int = 100,) -> FileParseResult:"""- Parse an uploaded file and analyze its contents.- Each row is treated as a single entity.-- Args:- file_content: File content as bytes or file-like object- filename: Original filename (used to determine file type)- max_preview_rows: Maximum number of rows to parse for preview-- Returns:- FileParseResult containing parsed entities and analysis-- Raises:- ValueError: If file format is unsupported or parsing fails+ Parse an uploaded TXT file and analyze its contents."""file_ext = Path(filename).suffix.lower()@@ -85,67 +50,20 @@ def parse_file(else:file_bytes = file_content- # Parse based on file extension- if file_ext == ".csv":- return _parse_csv(file_bytes, max_preview_rows)- elif file_ext == ".txt":- return _parse_txt(file_bytes, max_preview_rows)- elif file_ext in [".xlsx", ".xls"]:- if not OPENPYXL_AVAILABLE:- raise ValueError(- "openpyxl library is required for Excel file support. "- "Install with: pip install openpyxl"- )- return _parse_excel(file_bytes, max_preview_rows)- else:- raise ValueError(f"Unsupported file format: {file_ext}")---def _parse_csv(- file_bytes: bytes,- max_preview_rows: int,-) -> FileParseResult:- """Parse a CSV file where each row is an entity."""- try:- # Try to decode as UTF-8, fall back to latin-1- try:- text_content = file_bytes.decode("utf-8")- except UnicodeDecodeError:- text_content = file_bytes.decode("latin-1")-- # Parse CSV- csv_reader = csv.DictReader(io.StringIO(text_content))-- # Extract columns- columns = csv_reader.fieldnames or []- if not columns:- raise ValueError("No columns found in CSV file")-- # Parse rows as entities- entities: List[EntityPreview] = []- for row_idx, row in enumerate(csv_reader):- if row_idx >= max_preview_rows:- break+ # Only support TXT files+ if file_ext != ".txt":+ raise ValueError(+ f"Unsupported file format: {file_ext}. Only .txt files are supported."+ )- # Skip empty rows- if not any(v and str(v).strip() for v in row.values()):- continue-- entity = _create_entity_preview(row_idx, row, columns)- if entity:- entities.append(entity)-- return _create_result(entities, columns)-- except Exception as e:- raise ValueError(f"Failed to parse CSV file: {str(e)}")+ return _parse_txt(file_bytes, max_preview_rows)def _parse_txt(file_bytes: bytes,max_preview_rows: int,) -> FileParseResult:- """Parse a TXT file where each line is an entity with a single 'value' column."""+ """Parse a TXT file where each line is an entity with a single string value."""try:# Try to decode as UTF-8, fall back to latin-1try:@@ -160,174 +78,64 @@ def _parse_txt(if not lines:raise ValueError("File is empty")- # For TXT files, single column named "value"- columns = ["value"]+ total_rows = len(lines)- # Convert lines to entities- entities: List[EntityPreview] = []- for row_idx, line in enumerate(lines[:max_preview_rows]):- row_data = {"value": line}- entity = _create_entity_preview(row_idx, row_data, columns)- if entity:- entities.append(entity)-- return _create_result(entities, columns)-- except Exception as e:- raise ValueError(f"Failed to parse TXT file: {str(e)}")+ entities: Dict[str, Entity] = {}--def _parse_excel(- file_bytes: bytes,- max_preview_rows: int,-) -> FileParseResult:- """Parse an Excel file where each row is an entity."""- try:- # Load workbook from bytes- workbook = load_workbook(filename=io.BytesIO(file_bytes), read_only=True)-- # Use the first sheet- sheet = workbook.active- if sheet is None:- raise ValueError("No active sheet found in Excel file")-- # Read all rows- all_rows = list(sheet.iter_rows(values_only=True))-- if not all_rows:- raise ValueError("Excel file is empty")-- # First row is header- header_row = all_rows[0]- data_rows = all_rows[1:]-- # Convert header to strings and handle None values- columns = [- str(cell) if cell is not None else f"Column_{i+1}"- for i, cell in enumerate(header_row)- ]-- # Convert data rows to entities- entities: List[EntityPreview] = []- for row_idx, row_values in enumerate(data_rows[:max_preview_rows]):- row_dict = {}- for i, value in enumerate(row_values):- if i < len(columns):- row_dict[columns[i]] = str(value) if value is not None else ""-- # Skip empty rows- if not any(v and str(v).strip() for v in row_dict.values()):- continue-- entity = _create_entity_preview(row_idx, row_dict, columns)+ for _, line in enumerate(lines[:max_preview_rows]):+ entity = _create_entity_preview(line)if entity:- entities.append(entity)-- workbook.close()-- return _create_result(entities, columns)+ if entity.detected_type in entities:+ entities[entity.detected_type].results.append(entity)+ else:+ entities[entity.detected_type] = Entity(+ type=entity.detected_type, results=[entity]+ )+ return FileParseResult(+ entities=entities,+ total_entities=total_rows,+ )except Exception as e:- raise ValueError(f"Failed to parse Excel file: {str(e)}")+ # Normalize exceptions to ValueError for callers/tests+ raise ValueError(f"Failed to parse TXT file: {str(e)}")-def _create_entity_preview(- row_idx: int, row_data: Dict[str, Any], columns: List[str]-) -> Optional[EntityPreview]:+def _create_entity_preview(row_value: str) -> Optional[EntityPreview]:"""Create an EntityPreview from a row of data.Args:- row_idx: Row index- row_data: Dictionary of column: value pairs- columns: List of column names-+ row_value: string value of the entryReturns:EntityPreview or None if row is invalid"""- # Filter out empty values- data = {k: v for k, v in row_data.items() if v and str(v).strip()}+ # Detect entity type (detect_type may return a class with from_string or None)+ DetectedType = detect_type(row_value)- if not data:- return None+ # If detection failed (None), provide a minimal Unknown type with from_string+ if not DetectedType:- # Detect entity type- detected_type = detect_entity_type_from_row(data)- if not detected_type:- detected_type = "Unknown"+ class UnknownType:+ @classmethod+ def from_string(cls, value: str):+ # fallback: return raw string or a minimal wrapper+ return value- # Get primary identifier- primary_value = get_primary_identifier(data, detected_type)- if not primary_value:- # Fallback to first value- primary_value = str(next(iter(data.values())))+ DetectedType = UnknownType+ detected_name = "Unknown"+ else:+ # If detect_type returned a class/type, try to get a readable name+ detected_name = getattr(DetectedType, "__name__", str(DetectedType))- # Determine confidence- confidence = _determine_confidence(detected_type, data)+ # Build object using the type's from_string (expect classmethod)+ try:+ obj = DetectedType.from_string(row_value)+ except Exception:+ # On failure converting to object, return None to skip the row+ return Nonereturn EntityPreview(- row_index=row_idx,- data=data,- detected_type=detected_type,- primary_value=primary_value,- confidence=confidence,- )---def _determine_confidence(entity_type: str, data: Dict[str, Any]) -> str:- """- Determine confidence level for entity type detection.-- Args:- entity_type: Detected entity type- data: Row data-- Returns:- "high", "medium", or "low"- """- if entity_type == "Unknown":- return "low"-- # Normalize keys- keys = [k.lower().strip() for k in data.keys()]-- # Check if column names strongly indicate the type- type_keywords = {- "Email": ["email", "mail", "e-mail"],- "Ip": ["ip", "ip_address", "ipv4", "ipv6"],- "Domain": ["domain", "hostname"],- "Website": ["url", "website", "web"],- "Phone": ["phone", "telephone", "tel", "mobile"],- "ASN": ["asn", "as_number"],- "Username": ["username", "handle", "user"],- "Organization": ["organization", "org", "company"],- "Individual": ["name", "person", "individual"],- }-- keywords = type_keywords.get(entity_type, [])- if any(keyword in key for key in keys for keyword in keywords):- return "high"-- # Medium confidence if type was detected but no explicit column names- if entity_type != "Unknown":- return "medium"-- return "low"---def _create_result(- entities: List[EntityPreview], columns: List[str]-) -> FileParseResult:- """Create FileParseResult from entities."""- # Calculate type distribution- type_distribution: Dict[str, int] = {}- for entity in entities:- type_name = entity.detected_type- type_distribution[type_name] = type_distribution.get(type_name, 0) + 1-- return FileParseResult(- entities=entities,- total_entities=len(entities),- type_distribution=type_distribution,- columns=columns,+ obj=obj,+ detected_type=detected_name,)diff --git a/flowsint-core/src/flowsint_core/imports/type_matcher.py b/flowsint-core/src/flowsint_core/imports/type_matcher.pydeleted file mode 100644index b5d5f4e3..00000000--- a/flowsint-core/src/flowsint_core/imports/type_matcher.py+++ /dev/null@@ -1,291 +0,0 @@-"""-Entity type matching based on flowsint_types.-Determines which entity type best matches the row data.-"""--from typing import Optional, Dict, Any, List-import re-import ipaddress--# Import all entity types from flowsint_types-try:- from flowsint_types import (- Email, Domain, Ip, ASN, Phone, Username, Website,- Organization, Individual, SocialAccount, Device,- Credential, Malware, SSLCertificate, Document, File- )- FLOWSINT_TYPES_AVAILABLE = True-except ImportError:- FLOWSINT_TYPES_AVAILABLE = False---# Registry of entity type matchers-# Each entry: (type_name, class_name, identifier_fields, matcher_function)-TYPE_REGISTRY: List[tuple] = [- ("Email", "Email", ["email", "mail", "e-mail", "address"], lambda d: _match_email(d)),- ("Ip", "Ip", ["ip", "ip_address", "ipv4", "ipv6", "address"], lambda d: _match_ip(d)),- ("Domain", "Domain", ["domain", "hostname", "host"], lambda d: _match_domain(d)),- ("Website", "Website", ["url", "website", "web", "link"], lambda d: _match_website(d)),- ("Phone", "Phone", ["phone", "telephone", "tel", "mobile", "cell"], lambda d: _match_phone(d)),- ("ASN", "ASN", ["asn", "as_number"], lambda d: _match_asn(d)),- ("Username", "Username", ["username", "user", "handle", "login"], lambda d: _match_username(d)),- ("Organization", "Organization", ["organization", "org", "company", "name"], lambda d: _match_organization(d)),- ("Individual", "Individual", ["name", "person", "individual", "fullname"], lambda d: _match_individual(d)),- ("SocialAccount", "SocialAccount", ["social", "profile", "account"], lambda d: _match_social(d)),- ("Credential", "Credential", ["credential", "password", "creds"], lambda d: _match_credential(d)),-]---def detect_entity_type_from_row(row_data: Dict[str, Any]) -> Optional[str]:- """- Detect the most likely entity type for a row of data.-- Args:- row_data: Dictionary of column_name: value pairs-- Returns:- Entity type name (e.g., "Email", "Domain", "Ip") or None- """- if not row_data:- return None-- # Normalize keys to lowercase for comparison, skip None keys- normalized_data = {- k.lower().strip(): v- for k, v in row_data.items()- if v and k is not None- }-- # Try to match each type- scores = {}- for type_name, class_name, identifier_fields, matcher_func in TYPE_REGISTRY:- score = 0-- # Check if any identifier field matches- for field in identifier_fields:- if field in normalized_data:- score += 10-- # Use matcher function to validate- try:- if matcher_func(normalized_data):- score += 5- except Exception:- pass-- if score > 0:- scores[type_name] = score-- # Return type with highest score- if scores:- return max(scores, key=scores.get)-- return None---def get_primary_identifier(row_data: Dict[str, Any], entity_type: str) -> Optional[str]:- """- Get the primary identifier value for an entity based on its type.-- Args:- row_data: Dictionary of column_name: value pairs- entity_type: The detected entity type-- Returns:- The primary identifier value (e.g., email address, domain name, IP)- """- normalized_data = {k.lower().strip(): v for k, v in row_data.items() if v}-- # Find the registry entry for this type- for type_name, class_name, identifier_fields, _ in TYPE_REGISTRY:- if type_name == entity_type:- # Look for identifier fields in order of priority- for field in identifier_fields:- if field in normalized_data:- return str(normalized_data[field]).strip()-- # Fallback: try common value fields- for key in ["value", "name", "identifier", "id"]:- if key in normalized_data:- return str(normalized_data[key]).strip()-- # Last resort: return first non-empty value- for value in row_data.values():- if value and str(value).strip():- return str(value).strip()-- return None---# Matcher functions for each entity type--def _match_email(data: Dict[str, Any]) -> bool:- """Check if data looks like an email entity."""- email_pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'-- for key, value in data.items():- if any(k in key for k in ["email", "mail"]):- return bool(re.match(email_pattern, str(value)))-- # Check if any value looks like an email- for value in data.values():- if re.match(email_pattern, str(value)):- return True-- return False---def _match_ip(data: Dict[str, Any]) -> bool:- """Check if data looks like an IP entity."""- for key, value in data.items():- if any(k in key for k in ["ip", "address"]) and "email" not in key:- try:- ipaddress.ip_address(str(value))- return True- except ValueError:- pass-- # Check if any value looks like an IP- for value in data.values():- try:- ipaddress.ip_address(str(value))- return True- except (ValueError, AttributeError):- pass-- return False---def _match_domain(data: Dict[str, Any]) -> bool:- """Check if data looks like a domain entity."""- domain_pattern = r'^(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}$'-- for key, value in data.items():- if any(k in key for k in ["domain", "hostname", "host"]):- return bool(re.match(domain_pattern, str(value)))-- # Check if any value looks like a domain (but not email)- for value in data.values():- val_str = str(value)- if re.match(domain_pattern, val_str) and "@" not in val_str and not val_str.startswith("http"):- return True-- return False---def _match_website(data: Dict[str, Any]) -> bool:- """Check if data looks like a website entity."""- url_pattern = r'^https?://'-- for key, value in data.items():- if any(k in key for k in ["url", "website", "web", "link"]):- return bool(re.match(url_pattern, str(value), re.IGNORECASE))-- # Check if any value looks like a URL- for value in data.values():- if re.match(url_pattern, str(value), re.IGNORECASE):- return True-- return False---def _match_phone(data: Dict[str, Any]) -> bool:- """Check if data looks like a phone entity."""- for key, value in data.items():- if any(k in key for k in ["phone", "tel", "mobile", "cell"]):- # Remove common separators- cleaned = re.sub(r'[\s\-\(\)\.]', '', str(value))- phone_pattern = r'^\+?[0-9]{7,15}$'- return bool(re.match(phone_pattern, cleaned))-- return False---def _match_asn(data: Dict[str, Any]) -> bool:- """Check if data looks like an ASN entity."""- asn_pattern = r'^AS\d+$'-- for key, value in data.items():- if "asn" in key or "as_number" in key or "as_name" in key:- return bool(re.match(asn_pattern, str(value), re.IGNORECASE))-- # Check if any value looks like an ASN- for value in data.values():- if re.match(asn_pattern, str(value), re.IGNORECASE):- return True-- return False---def _match_username(data: Dict[str, Any]) -> bool:- """Check if data looks like a username entity."""- username_pattern = r'^@?[a-zA-Z0-9_]{3,30}$'-- for key, value in data.items():- if any(k in key for k in ["username", "user", "handle"]):- val_str = str(value)- if re.match(username_pattern, val_str):- return val_str.startswith('@') or not val_str.isdigit()-- # Fallback: check if any value looks like a username- # (but not if it looks like other types)- for key, value in data.items():- val_str = str(value)- if re.match(username_pattern, val_str):- # Make sure it's not an email or domain- if '@' not in val_str and '.' not in val_str:- # If it starts with @ or is not purely numeric, likely a username- if val_str.startswith('@') or not val_str.isdigit():- return True-- return False---def _match_organization(data: Dict[str, Any]) -> bool:- """Check if data looks like an organization entity."""- # Look for organization-specific fields- org_fields = ["organization", "org", "company", "siren", "siret", "company_name"]-- for key in data.keys():- if any(field in key for field in org_fields):- return True-- # If has "name" but no individual-specific fields- if "name" in data:- individual_fields = ["firstname", "lastname", "first_name", "last_name", "email"]- has_individual_fields = any(field in data for field in individual_fields)- return not has_individual_fields-- return False---def _match_individual(data: Dict[str, Any]) -> bool:- """Check if data looks like an individual entity."""- # Look for individual-specific fields- individual_fields = ["firstname", "lastname", "first_name", "last_name", "fullname", "person"]-- for key in data.keys():- if any(field in key for field in individual_fields):- return True-- return False---def _match_social(data: Dict[str, Any]) -> bool:- """Check if data looks like a social profile entity."""- social_fields = ["twitter", "linkedin", "facebook", "instagram", "github", "social"]-- for key in data.keys():- if any(field in key for field in social_fields):- return True-- return False---def _match_credential(data: Dict[str, Any]) -> bool:- """Check if data looks like a credential entity."""- # Look for credential-specific fields- if ("password" in data or "pass" in data) and ("username" in data or "user" in data or "email" in data):- return True-- return Falsediff --git a/flowsint-core/tests/import/test_entity_detection.py b/flowsint-core/tests/import/test_entity_detection.pyindex 529d0134..e0951c06 100644--- a/flowsint-core/tests/import/test_entity_detection.py+++ b/flowsint-core/tests/import/test_entity_detection.py@@ -1,299 +1,61 @@-"""Unit tests for entity_detection module."""-import pytest-from flowsint_core.imports.entity_detection import (- detect_entity_type,- is_email,- is_domain,- is_ip_address,- is_website,- is_phone,- is_asn,- is_username,- get_default_label,-)---class TestEmailDetection:- """Tests for email detection."""-- def test_valid_emails(self):- """Test detection of valid email addresses."""- valid_emails = [- "[email protected]",- "[email protected]",- "[email protected]",- "[email protected]",- "[email protected]",- ]- for email in valid_emails:- assert is_email(email), f"Failed to detect valid email: {email}"- assert detect_entity_type(email) == "Email"-- def test_invalid_emails(self):- """Test rejection of invalid email addresses."""- invalid_emails = [- "not-an-email",- "@example.com",- "user@",- "user@domain",- "user [email protected]",- "",- ]- for email in invalid_emails:- assert not is_email(email), f"Incorrectly detected as email: {email}"---class TestDomainDetection:- """Tests for domain detection."""-- def test_valid_domains(self):- """Test detection of valid domains."""- valid_domains = [- "example.com",- "sub.domain.com",- "my-domain.co.uk",- "test123.org",- "a.b.c.d.example.com",- ]- for domain in valid_domains:- assert is_domain(domain), f"Failed to detect valid domain: {domain}"- assert detect_entity_type(domain) == "Domain"-- def test_invalid_domains(self):- """Test rejection of invalid domains."""- invalid_domains = [- "not-a-domain",- ".com",- "domain.",- "http://example.com", # This is a website, not a domain- "",- "192.168.1.1", # This is an IP- ]- for domain in invalid_domains:- assert not is_domain(domain), f"Incorrectly detected as domain: {domain}"---class TestIPAddressDetection:- """Tests for IP address detection."""-- def test_valid_ipv4(self):- """Test detection of valid IPv4 addresses."""- valid_ips = [- "192.168.1.1",- "10.0.0.1",- "172.16.0.1",- "8.8.8.8",- "255.255.255.255",- ]- for ip in valid_ips:- assert is_ip_address(ip), f"Failed to detect valid IPv4: {ip}"- assert detect_entity_type(ip) == "IP"-- def test_valid_ipv6(self):- """Test detection of valid IPv6 addresses."""- valid_ips = [- "2001:0db8:85a3:0000:0000:8a2e:0370:7334",- "2001:db8::1",- "::1",- "fe80::1",- ]- for ip in valid_ips:- assert is_ip_address(ip), f"Failed to detect valid IPv6: {ip}"- assert detect_entity_type(ip) == "IP"-- def test_invalid_ips(self):- """Test rejection of invalid IP addresses."""- invalid_ips = [- "256.256.256.256",- "192.168.1",- "not-an-ip",- "",- ]- for ip in invalid_ips:- assert not is_ip_address(ip), f"Incorrectly detected as IP: {ip}"---class TestWebsiteDetection:- """Tests for website/URL detection."""-- def test_valid_websites(self):- """Test detection of valid URLs."""- valid_urls = [- "http://example.com",- "https://www.example.com",- "http://sub.domain.com/path",- "https://example.com:8080/path?query=value",- ]- for url in valid_urls:- assert is_website(url), f"Failed to detect valid URL: {url}"- assert detect_entity_type(url) == "Website"-- def test_invalid_websites(self):- """Test rejection of invalid URLs."""- invalid_urls = [- "example.com",- "ftp://example.com",- "not-a-url",- "",- ]- for url in invalid_urls:- assert not is_website(url), f"Incorrectly detected as website: {url}"---class TestPhoneDetection:- """Tests for phone number detection."""-- def test_valid_phones(self):- """Test detection of valid phone numbers."""- valid_phones = [- "+1234567890",- "123-456-7890",- "(123) 456-7890",- "+44 20 7123 4567",- "555.123.4567",- "5551234567",- ]- for phone in valid_phones:- assert is_phone(phone), f"Failed to detect valid phone: {phone}"- assert detect_entity_type(phone) == "Phone"-- def test_invalid_phones(self):- """Test rejection of invalid phone numbers."""- invalid_phones = [- "123", # Too short- "abc-def-ghij",- "",- "1234567890123456", # Too long- ]- for phone in invalid_phones:- assert not is_phone(phone), f"Incorrectly detected as phone: {phone}"---class TestASNDetection:- """Tests for ASN detection."""-- def test_valid_asns(self):- """Test detection of valid ASNs."""- valid_asns = [- "AS13335",- "as64512",- "AS1",- "AS4294967295",- ]- for asn in valid_asns:- assert is_asn(asn), f"Failed to detect valid ASN: {asn}"- assert detect_entity_type(asn) == "ASN"-- def test_invalid_asns(self):- """Test rejection of invalid ASNs."""- invalid_asns = [- "13335",- "ASN13335",- "AS",- "",- ]- for asn in invalid_asns:- assert not is_asn(asn), f"Incorrectly detected as ASN: {asn}"---class TestUsernameDetection:- """Tests for username detection."""-- def test_valid_usernames(self):- """Test detection of valid usernames."""- valid_usernames = [- "@john_doe",- "@user123",- "username",- "user_name_123",- ]- for username in valid_usernames:- assert is_username(username), f"Failed to detect valid username: {username}"- assert detect_entity_type(username) == "Username"-- def test_invalid_usernames(self):- """Test rejection of invalid usernames."""- invalid_usernames = [- "ab", # Too short- "@",- "123456789", # Purely numeric- "[email protected]", # This is an email- "",- "a" * 31, # Too long- ]- for username in invalid_usernames:- assert not is_username(username), f"Incorrectly detected as username: {username}"---class TestDetectionPriority:- """Tests for detection priority and edge cases."""-- def test_email_vs_username(self):- """Email should take priority over username."""- assert detect_entity_type("[email protected]") == "Email"-- def test_url_vs_domain(self):- """URL should take priority over domain."""- assert detect_entity_type("http://example.com") == "Website"- assert detect_entity_type("example.com") == "Domain"-- def test_ip_vs_domain(self):- """IP should be correctly distinguished from domain."""- assert detect_entity_type("192.168.1.1") == "IP"- assert detect_entity_type("example.com") == "Domain"-- def test_whitespace_handling(self):- """Test that whitespace is properly handled."""- assert detect_entity_type(" [email protected] ") == "Email"- assert detect_entity_type(" example.com ") == "Domain"-- def test_none_and_empty(self):- """Test handling of None and empty strings."""- assert detect_entity_type("") is None- assert detect_entity_type(" ") is None- assert detect_entity_type(None) is None---class TestGetDefaultLabel:- """Tests for get_default_label function."""-- def test_email_label(self):- """Test label generation for email."""- assert get_default_label("Email", "[email protected]") == "[email protected]"-- def test_domain_label(self):- """Test label generation for domain."""- assert get_default_label("Domain", "example.com") == "example.com"-- def test_ip_label(self):- """Test label generation for IP."""- assert get_default_label("IP", "192.168.1.1") == "192.168.1.1"-- def test_username_label(self):- """Test label generation for username (strips @ prefix)."""- assert get_default_label("Username", "@john_doe") == "john_doe"- assert get_default_label("Username", "john_doe") == "john_doe"-- def test_phone_label(self):- """Test label generation for phone."""- assert get_default_label("Phone", "+1234567890") == "+1234567890"-- def test_asn_label(self):- """Test label generation for ASN."""- assert get_default_label("ASN", "AS13335") == "AS13335"-- def test_website_label(self):- """Test label generation for website."""- assert get_default_label("Website", "https://example.com") == "https://example.com"-- def test_organization_label(self):- """Test label generation for organization."""- assert get_default_label("Organization", "ACME Corp") == "ACME Corp"-- def test_individual_label(self):- """Test label generation for individual."""- assert get_default_label("Individual", "John Doe") == "John Doe"-- def test_unknown_type(self):- """Test label generation for unknown type."""- assert get_default_label("UnknownType", "some value") == "some value"+from flowsint_types import Domain, Ip, Phone+from flowsint_core.imports import detect_type+++def test_detection_domains():+ raw_domains = ["mydomain.com", "myseconddomain.com", "blog.subdomain.uk"]+ results = []+ for domain in raw_domains:+ DetectedType = detect_type(domain)+ if DetectedType:+ assert DetectedType is Domain+ py_obj = DetectedType.from_string(domain)+ assert isinstance(py_obj, Domain)+ results.append(py_obj)+ # domain 1+ assert results[0].domain == "mydomain.com"+ assert results[0].root == True+ # domain 2+ assert results[1].domain == "myseconddomain.com"+ assert results[1].root == True+ # domain 3+ assert results[2].domain == "blog.subdomain.uk"+ assert results[2].root == False+++def test_detection_ips():+ raw_ips = ["240.123.123.234", "12.43.23.12"]+ results = []+ for ip in raw_ips:+ DetectedType = detect_type(ip)+ if DetectedType:+ assert DetectedType is Ip+ py_obj = DetectedType.from_string(ip)+ assert isinstance(py_obj, Ip)+ results.append(py_obj)+ # ip 1+ assert results[0].address == "240.123.123.234"+ # ip 2+ assert results[1].address == "12.43.23.12"+++def test_detection_mixed():+ raw_obj = ["240.123.123.234", "my.domain.io", "+33632233223"]+ results = []+ for obj in raw_obj:+ DetectedType = detect_type(obj)+ print(DetectedType)+ if DetectedType:+ py_obj = DetectedType.from_string(obj)+ results.append(py_obj)++ # ip+ assert isinstance(results[0], Ip)+ assert results[0].address == "240.123.123.234"+ # domain+ assert isinstance(results[1], Domain)+ assert results[1].domain == "my.domain.io"+ assert results[1].root == False+ # phone+ assert isinstance(results[2], Phone)+ assert results[2].number == "+33632233223"diff --git a/flowsint-core/tests/import/test_file_parser.py b/flowsint-core/tests/import/test_file_parser.pynew file mode 100644index 00000000..6fe3559c--- /dev/null+++ b/flowsint-core/tests/import/test_file_parser.py@@ -0,0 +1,42 @@+from flowsint_types import Domain, Ip, Username+from unittest.mock import patch++from flowsint_core.imports import (+ parse_import_file,+ FileParseResult,+ EntityPreview,+)+++def test_parse_import_file_txt_basic_with_mocked_detect_type():++ txt_bytes = b"domain.com\nblog.domain.com\n12.34.56.78\nmy_super_username"++ result = parse_import_file(txt_bytes, "values.txt", max_preview_rows=100)++ assert isinstance(result, FileParseResult)+ assert result.total_entities == 4++ first = result.entities["Domain"].results[0]+ assert isinstance(first, EntityPreview)+ assert isinstance(first.obj, Domain)+ assert first.detected_type == "Domain"+ assert first.obj.domain == "domain.com"++ second = result.entities["Domain"].results[1]+ assert isinstance(second, EntityPreview)+ assert isinstance(second.obj, Domain)+ assert second.detected_type == "Domain"+ assert second.obj.domain == "blog.domain.com"++ third = result.entities["Ip"].results[0]+ assert isinstance(third, EntityPreview)+ assert isinstance(third.obj, Ip)+ assert third.detected_type == "Ip"+ assert third.obj.address == "12.34.56.78"++ fourth = result.entities["Username"].results[0]+ assert isinstance(fourth, EntityPreview)+ assert isinstance(fourth.obj, Username)+ assert fourth.detected_type == "Username"+ assert fourth.obj.value == "my_super_username"diff --git a/flowsint-core/tests/import/test_import_integration.py b/flowsint-core/tests/import/test_import_integration.pydeleted file mode 100644index 0612a6b6..00000000--- a/flowsint-core/tests/import/test_import_integration.py+++ /dev/null@@ -1,294 +0,0 @@-"""Integration tests for import functionality (entity-based)."""-import pytest-from flowsint_core.imports import parse_file---class TestEntityBasedImportWorkflow:- """Integration tests for entity-based import workflow."""-- def test_full_entity_workflow(self):- """Test complete workflow: parse file -> detect entities -> prepare for import."""- # Step 1: User uploads a CSV file (each row = one entity)- csv_content = b"""email,name,department[email protected],John Admin,IT[email protected],Jane User,HR[email protected],Bob Security,Security-"""-- # Step 2: Parse and analyze the file- result = parse_file(csv_content, "test.csv")-- # Verify parsing - entities instead of rows- assert result.total_entities == 3- assert len(result.entities) == 3-- # Verify entity detection- assert all(e.detected_type == "Email" for e in result.entities)-- # Step 3: User reviews entities (can change type/label)- entity_mappings = []- for entity in result.entities:- entity_mappings.append({- "row_index": entity.row_index,- "entity_type": entity.detected_type,- "label": entity.primary_value,- "include": True- })-- # Verify mappings- assert len(entity_mappings) == 3- assert all(m["entity_type"] == "Email" for m in entity_mappings)-- # Step 4: Verify all data is preserved in entity- first_entity = result.entities[0]- assert "email" in first_entity.data- assert "name" in first_entity.data- assert "department" in first_entity.data-- def test_txt_entity_workflow(self):- """Test workflow for TXT file where each line is an entity."""- txt_content = b"""example.com-192.168.1.1-AS13335-"""-- result = parse_file(txt_content, "entities.txt")-- assert result.total_entities == 4- assert len(result.entities) == 4-- # Each line is detected as different type- types = {e.detected_type for e in result.entities}- assert "Domain" in types- assert "Ip" in types- assert "Email" in types- assert "ASN" in types-- def test_user_corrections_workflow(self):- """Test workflow where user corrects detected types."""- csv_content = b"""identifier,category,notes-john_doe,username,Social media handle-example.com,domain,Main website-192.168.1.1,server,Production server-"""-- # Parse file- result = parse_file(csv_content, "data.csv")-- # Simulate user reviewing and correcting types- entity_mappings = []- for entity in result.entities:- # User can override detected type- if entity.data.get("category") == "server":- # User wants custom type instead of detected "Ip"- entity_type = "Device"- else:- entity_type = entity.detected_type-- entity_mappings.append({- "row_index": entity.row_index,- "entity_type": entity_type,- "label": entity.primary_value,- "include": True- })-- # Verify overrides- assert entity_mappings[0]["entity_type"] == "Username"- assert entity_mappings[1]["entity_type"] == "Domain"- assert entity_mappings[2]["entity_type"] == "Device" # User override-- def test_label_customization(self):- """Test customizing labels for entities."""- csv_content = b"""email,full_name,role[email protected],John Administrator,Admin[email protected],Jane Security,SecOps-"""-- result = parse_file(csv_content, "contacts.csv")-- # User customizes labels with additional context- entity_mappings = []- for entity in result.entities:- # Combine email with full name for label- custom_label = f"{entity.data['email']} ({entity.data['full_name']})"- entity_mappings.append({- "row_index": entity.row_index,- "entity_type": entity.detected_type,- "label": custom_label,- "include": True- })-- # Verify customized labels- assert "John Administrator" in entity_mappings[0]["label"]- assert "Jane Security" in entity_mappings[1]["label"]-- def test_selective_import(self):- """Test excluding specific entities from import."""- csv_content = b"""email,status[email protected],active[email protected],inactive[email protected],active-"""-- result = parse_file(csv_content, "users.csv")-- # User only wants to import active users- entity_mappings = []- for entity in result.entities:- include = entity.data.get("status") == "active"- entity_mappings.append({- "row_index": entity.row_index,- "entity_type": entity.detected_type,- "label": entity.primary_value,- "include": include- })-- # Verify selective inclusion- included = [m for m in entity_mappings if m["include"]]- assert len(included) == 2---class TestRealWorldEntityScenarios:- """Real-world scenario tests with entity-based approach."""-- def test_organization_import(self):- """Test importing organizations with metadata."""- csv_content = b"""name,industry,country,employees-Google,Technology,USA,150000-Microsoft,Software,USA,220000-Apple,Technology,USA,164000-"""-- result = parse_file(csv_content, "orgs.csv")-- assert result.total_entities == 3- assert all(e.detected_type == "Organization" for e in result.entities)-- # Verify all metadata preserved- for entity in result.entities:- assert "name" in entity.data- assert "industry" in entity.data- assert "country" in entity.data- assert "employees" in entity.data-- def test_threat_intel_import(self):- """Test importing threat intelligence data."""- csv_content = b"""domain,severity,first_seen,description,malware_family-malicious.com,high,2024-01-15,Phishing domain,TrickBot-evil.net,critical,2024-01-16,C2 server,Cobalt Strike-threat.xyz,high,2024-01-18,APT infrastructure,Unknown-"""-- result = parse_file(csv_content, "iocs.csv")-- assert result.total_entities == 3- assert all(e.detected_type == "Domain" for e in result.entities)-- # Verify threat metadata preserved in entity data- first_entity = result.entities[0]- assert first_entity.data["severity"] == "high"- assert first_entity.data["malware_family"] == "TrickBot"- assert "description" in first_entity.data-- def test_infrastructure_mapping(self):- """Test importing infrastructure with multiple related fields."""- csv_content = b"""domain,ip,asn,country,provider-cloudflare.com,1.1.1.1,AS13335,USA,Cloudflare-google.com,8.8.8.8,AS15169,USA,Google-"""-- result = parse_file(csv_content, "infrastructure.csv")-- assert result.total_entities == 2-- # All related data preserved in entity- for entity in result.entities:- assert "domain" in entity.data- assert "ip" in entity.data- assert "asn" in entity.data- assert "country" in entity.data- assert "provider" in entity.data---class TestTypeDistribution:- """Tests for type distribution in entity-based approach."""-- def test_mixed_entity_types(self):- """Test file with multiple entity types."""- txt_content = b"""example.com-test.org-192.168.1.1-10.0.0.1-"""-- result = parse_file(txt_content, "mixed.txt")-- # Check type distribution- assert result.type_distribution["Domain"] == 2- assert result.type_distribution["Email"] == 2- assert result.type_distribution["Ip"] == 2-- def test_predominant_type_detection(self):- """Test that predominant type is correctly identified."""- csv_content = b"""email,notes[email protected],User account[email protected],Admin account[email protected],Guest account[email protected],Service account-"""-- result = parse_file(csv_content, "emails.csv")-- # Should have high confidence for Email type- assert all(e.detected_type == "Email" for e in result.entities)- assert all(e.confidence == "high" for e in result.entities)---class TestEdgeCases:- """Edge case tests for entity-based import."""-- def test_single_column_csv(self):- """Test CSV with single column."""- csv_content = b"""email-"""-- result = parse_file(csv_content, "single_column.csv")-- assert result.total_entities == 2- assert all(e.detected_type == "Email" for e in result.entities)-- def test_many_columns_csv(self):- """Test CSV with many columns (all become entity data)."""- csv_content = b"""email,name,dept,role,location,phone,manager,start_date[email protected],John,IT,Dev,NYC,555-1234,[email protected],2024-01-01-"""-- result = parse_file(csv_content, "detailed.csv")-- assert result.total_entities == 1- entity = result.entities[0]-- # All 8 columns should be in entity data- assert len(entity.data) == 8- assert "email" in entity.data- assert "manager" in entity.data- assert "start_date" in entity.data-- def test_unknown_type_handling(self):- """Test handling of unknown/undetectable types."""- txt_content = b"""random text string-another random value-unstructured data-"""-- result = parse_file(txt_content, "unknown.txt")-- # Should still parse but mark as Unknown- assert result.total_entities == 3- assert all(e.detected_type == "Unknown" for e in result.entities)- assert all(e.confidence == "low" for e in result.entities)