Skip to content

A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.

feat(core): import data + tests

dextmorgn0f39813c9c26
diff --git a/flowsint-core/poetry.lock b/flowsint-core/poetry.lock
index c1221cc5..6cd3be32 100644
--- a/flowsint-core/poetry.lock
+++ b/flowsint-core/poetry.lock
@@ -6469,4 +6469,4 @@ propcache = ">=0.2.1"
[metadata]
lock-version = "2.1"
python-versions = ">=3.12,<4.0"
-content-hash = "4833b73e3dedd38d32f82bae3c5ede7e6ceaec7e403dc1ca2ded467d6fe2acbf"
+content-hash = "36475c223c1801a494e022f8cfb7aa18c12405405b849b84c92b403189994bcc"
diff --git a/flowsint-core/pyproject.toml b/flowsint-core/pyproject.toml
index 72fa73ab..9f908431 100644
--- a/flowsint-core/pyproject.toml
+++ b/flowsint-core/pyproject.toml
@@ -28,6 +28,7 @@ python-multipart = "^0.0.20"
docker = "^7.1.0"
pytest = "^8.4.2"
cryptography = "^45.0.7"
+openpyxl = "^3.1"
[tool.poetry.group.dev.dependencies]
pytest-asyncio = "^0.21"
diff --git a/flowsint-core/src/flowsint_core/imports/__init__.py b/flowsint-core/src/flowsint_core/imports/__init__.py
new file mode 100644
index 00000000..0c77f5cf
--- /dev/null
+++ b/flowsint-core/src/flowsint_core/imports/__init__.py
@@ -0,0 +1,17 @@
+"""
+Import utilities for entity parsing and type detection.
+"""
+
+from .entity_detection import detect_entity_type, get_default_label
+from .file_parser import parse_file, FileParseResult, EntityPreview
+from .type_matcher import detect_entity_type_from_row, get_primary_identifier
+
+__all__ = [
+ "detect_entity_type",
+ "get_default_label",
+ "parse_file",
+ "FileParseResult",
+ "EntityPreview",
+ "detect_entity_type_from_row",
+ "get_primary_identifier",
+]
diff --git a/flowsint-core/src/flowsint_core/imports/entity_detection.py b/flowsint-core/src/flowsint_core/imports/entity_detection.py
new file mode 100644
index 00000000..0cbedfdc
--- /dev/null
+++ b/flowsint-core/src/flowsint_core/imports/entity_detection.py
@@ -0,0 +1,131 @@
+"""
+Entity type detection utilities for import feature.
+Provides basic pattern matching for common entity types.
+"""
+
+import re
+from typing import Optional
+import ipaddress
+
+
+# Ordered registry of detectors. Order matters to avoid false positives
+# (e.g., URL should be checked before Domain).
+# Each entry is a tuple of (entity_type, predicate_function)
+DETECTORS = [
+ ("Email", lambda v: is_email(v)),
+ ("IP", lambda v: is_ip_address(v)),
+ ("Website", lambda v: is_website(v)),
+ ("Domain", lambda v: is_domain(v)),
+ ("Phone", lambda v: is_phone(v)),
+ ("ASN", lambda v: is_asn(v)),
+ ("Username", lambda v: is_username(v)),
+]
+
+
+def detect_entity_type(value: str) -> Optional[str]:
+ """
+ Detect entity type based on value pattern.
+ Returns the detected entity type or None if no match.
+
+ Args:
+ value: The string value to analyze
+
+ Returns:
+ Entity type string (e.g., "Email", "Domain", "IP") or None
+ """
+ if not value or not isinstance(value, str):
+ return None
+
+ value = value.strip()
+
+ # Iterate through ordered detectors to find the first match
+ for entity_type, predicate in DETECTORS:
+ try:
+ if predicate(value):
+ return entity_type
+ except Exception:
+ # If a predicate raises, skip it to avoid breaking detection
+ # (predicates should generally be safe and return bool)
+ continue
+
+ return None
+
+
+def is_email(value: str) -> bool:
+ """Check if value matches email pattern."""
+ email_pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
+ return bool(re.match(email_pattern, value))
+
+
+def is_domain(value: str) -> bool:
+ """Check if value matches domain pattern."""
+ # Basic domain pattern: contains dots and valid characters
+ domain_pattern = r'^(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}$'
+ return bool(re.match(domain_pattern, value))
+
+
+def is_ip_address(value: str) -> bool:
+ """Check if value is a valid IPv4 or IPv6 address."""
+ try:
+ ipaddress.ip_address(value)
+ return True
+ except ValueError:
+ return False
+
+
+def is_website(value: str) -> bool:
+ """Check if value matches URL/website pattern."""
+ url_pattern = r'^https?://'
+ return bool(re.match(url_pattern, value, re.IGNORECASE))
+
+
+def is_phone(value: str) -> bool:
+ """Check if value matches phone number pattern."""
+ # Remove common separators for checking
+ cleaned = re.sub(r'[\s\-\(\)\.]', '', value)
+ # Check if it's mostly digits and has reasonable length
+ phone_pattern = r'^\+?[0-9]{7,15}$'
+ return bool(re.match(phone_pattern, cleaned))
+
+
+def is_asn(value: str) -> bool:
+ """Check if value matches ASN pattern."""
+ asn_pattern = r'^AS\d+$'
+ return bool(re.match(asn_pattern, value, re.IGNORECASE))
+
+
+def is_username(value: str) -> bool:
+ """Check if value matches username pattern (social media style)."""
+ # Matches @username format or simple alphanumeric with underscores
+ username_pattern = r'^@?[a-zA-Z0-9_]{3,30}$'
+ if re.match(username_pattern, value):
+ # Additional check: starts with @ or is not purely numeric
+ return value.startswith('@') or not value.lstrip('@').isdigit()
+ return False
+
+
+def get_default_label(entity_type: str, value: str) -> str:
+ """
+ Get default label for an entity based on its type and value.
+
+ Args:
+ entity_type: The detected or selected entity type
+ value: The entity value
+
+ Returns:
+ Default label string
+ """
+ # For most types, the value itself is a good label
+ type_defaults = {
+ "Email": value,
+ "Domain": value,
+ "IP": value,
+ "Website": value,
+ "Phone": value,
+ "ASN": value,
+ "Username": value.lstrip('@'), # Remove @ prefix for label
+ "Organization": value,
+ "Individual": value,
+ }
+
+ return type_defaults.get(entity_type, value)
diff --git a/flowsint-core/src/flowsint_core/imports/file_parser.py b/flowsint-core/src/flowsint_core/imports/file_parser.py
new file mode 100644
index 00000000..84f85596
--- /dev/null
+++ b/flowsint-core/src/flowsint_core/imports/file_parser.py
@@ -0,0 +1,333 @@
+"""
+File parsing utilities for entity imports.
+Handles CSV, TXT, and XLSX file formats.
+Each row represents ONE entity with all columns as data properties.
+"""
+
+import csv
+import io
+from dataclasses import dataclass
+from typing import List, Dict, Any, BinaryIO, Union, Optional
+from pathlib import Path
+
+try:
+ from openpyxl import load_workbook
+
+ OPENPYXL_AVAILABLE = True
+except ImportError:
+ OPENPYXL_AVAILABLE = False
+
+from .type_matcher import detect_entity_type_from_row, get_primary_identifier
+
+
+@dataclass
+class EntityPreview:
+ """Preview of a single entity to be imported."""
+
+ row_index: int
+ """Index of the row (0-based)."""
+
+ data: Dict[str, Any]
+ """All column data as key-value pairs."""
+
+ detected_type: str
+ """Detected entity type (e.g., "Email", "Domain")."""
+
+ primary_value: str
+ """Primary identifier value for the entity."""
+
+ confidence: str
+ """Confidence level: high, medium, low."""
+
+
+@dataclass
+class FileParseResult:
+ """Result of parsing an import file."""
+
+ entities: List[EntityPreview]
+ """List of detected entities from the file."""
+
+ total_entities: int
+ """Total number of entities (rows)."""
+
+ type_distribution: Dict[str, int]
+ """Distribution of detected types: {type_name: count}."""
+
+ columns: List[str]
+ """Column names found in the file (for reference)."""
+
+
+def parse_file(
+ file_content: Union[bytes, BinaryIO],
+ filename: str,
+ max_preview_rows: int = 100,
+) -> FileParseResult:
+ """
+ Parse an uploaded file and analyze its contents.
+ Each row is treated as a single entity.
+
+ Args:
+ file_content: File content as bytes or file-like object
+ filename: Original filename (used to determine file type)
+ max_preview_rows: Maximum number of rows to parse for preview
+
+ Returns:
+ FileParseResult containing parsed entities and analysis
+
+ Raises:
+ ValueError: If file format is unsupported or parsing fails
+ """
+ file_ext = Path(filename).suffix.lower()
+
+ # Convert file content to bytes if it's a file-like object
+ if hasattr(file_content, "read"):
+ file_bytes = file_content.read()
+ else:
+ file_bytes = file_content
+
+ # Parse based on file extension
+ if file_ext == ".csv":
+ return _parse_csv(file_bytes, max_preview_rows)
+ elif file_ext == ".txt":
+ return _parse_txt(file_bytes, max_preview_rows)
+ elif file_ext in [".xlsx", ".xls"]:
+ if not OPENPYXL_AVAILABLE:
+ raise ValueError(
+ "openpyxl library is required for Excel file support. "
+ "Install with: pip install openpyxl"
+ )
+ return _parse_excel(file_bytes, max_preview_rows)
+ else:
+ raise ValueError(f"Unsupported file format: {file_ext}")
+
+
+def _parse_csv(
+ file_bytes: bytes,
+ max_preview_rows: int,
+) -> FileParseResult:
+ """Parse a CSV file where each row is an entity."""
+ try:
+ # Try to decode as UTF-8, fall back to latin-1
+ try:
+ text_content = file_bytes.decode("utf-8")
+ except UnicodeDecodeError:
+ text_content = file_bytes.decode("latin-1")
+
+ # Parse CSV
+ csv_reader = csv.DictReader(io.StringIO(text_content))
+
+ # Extract columns
+ columns = csv_reader.fieldnames or []
+ if not columns:
+ raise ValueError("No columns found in CSV file")
+
+ # Parse rows as entities
+ entities: List[EntityPreview] = []
+ for row_idx, row in enumerate(csv_reader):
+ if row_idx >= max_preview_rows:
+ break
+
+ # Skip empty rows
+ if not any(v and str(v).strip() for v in row.values()):
+ continue
+
+ entity = _create_entity_preview(row_idx, row, columns)
+ if entity:
+ entities.append(entity)
+
+ return _create_result(entities, columns)
+
+ except Exception as e:
+ raise ValueError(f"Failed to parse CSV file: {str(e)}")
+
+
+def _parse_txt(
+ file_bytes: bytes,
+ max_preview_rows: int,
+) -> FileParseResult:
+ """Parse a TXT file where each line is an entity with a single 'value' column."""
+ try:
+ # Try to decode as UTF-8, fall back to latin-1
+ try:
+ text_content = file_bytes.decode("utf-8")
+ except UnicodeDecodeError:
+ text_content = file_bytes.decode("latin-1")
+
+ # Split by lines and filter empty lines
+ lines = [line.strip() for line in text_content.split("\n")]
+ lines = [line for line in lines if line]
+
+ if not lines:
+ raise ValueError("File is empty")
+
+ # For TXT files, single column named "value"
+ columns = ["value"]
+
+ # Convert lines to entities
+ entities: List[EntityPreview] = []
+ for row_idx, line in enumerate(lines[:max_preview_rows]):
+ row_data = {"value": line}
+ entity = _create_entity_preview(row_idx, row_data, columns)
+ if entity:
+ entities.append(entity)
+
+ return _create_result(entities, columns)
+
+ except Exception as e:
+ raise ValueError(f"Failed to parse TXT file: {str(e)}")
+
+
+def _parse_excel(
+ file_bytes: bytes,
+ max_preview_rows: int,
+) -> FileParseResult:
+ """Parse an Excel file where each row is an entity."""
+ try:
+ # Load workbook from bytes
+ workbook = load_workbook(filename=io.BytesIO(file_bytes), read_only=True)
+
+ # Use the first sheet
+ sheet = workbook.active
+ if sheet is None:
+ raise ValueError("No active sheet found in Excel file")
+
+ # Read all rows
+ all_rows = list(sheet.iter_rows(values_only=True))
+
+ if not all_rows:
+ raise ValueError("Excel file is empty")
+
+ # First row is header
+ header_row = all_rows[0]
+ data_rows = all_rows[1:]
+
+ # Convert header to strings and handle None values
+ columns = [
+ str(cell) if cell is not None else f"Column_{i+1}"
+ for i, cell in enumerate(header_row)
+ ]
+
+ # Convert data rows to entities
+ entities: List[EntityPreview] = []
+ for row_idx, row_values in enumerate(data_rows[:max_preview_rows]):
+ row_dict = {}
+ for i, value in enumerate(row_values):
+ if i < len(columns):
+ row_dict[columns[i]] = str(value) if value is not None else ""
+
+ # Skip empty rows
+ if not any(v and str(v).strip() for v in row_dict.values()):
+ continue
+
+ entity = _create_entity_preview(row_idx, row_dict, columns)
+ if entity:
+ entities.append(entity)
+
+ workbook.close()
+
+ return _create_result(entities, columns)
+
+ except Exception as e:
+ raise ValueError(f"Failed to parse Excel file: {str(e)}")
+
+
+def _create_entity_preview(
+ row_idx: int, row_data: Dict[str, Any], columns: List[str]
+) -> Optional[EntityPreview]:
+ """
+ Create an EntityPreview from a row of data.
+
+ Args:
+ row_idx: Row index
+ row_data: Dictionary of column: value pairs
+ columns: List of column names
+
+ Returns:
+ EntityPreview or None if row is invalid
+ """
+ # Filter out empty values
+ data = {k: v for k, v in row_data.items() if v and str(v).strip()}
+
+ if not data:
+ return None
+
+ # Detect entity type
+ detected_type = detect_entity_type_from_row(data)
+ if not detected_type:
+ detected_type = "Unknown"
+
+ # Get primary identifier
+ primary_value = get_primary_identifier(data, detected_type)
+ if not primary_value:
+ # Fallback to first value
+ primary_value = str(next(iter(data.values())))
+
+ # Determine confidence
+ confidence = _determine_confidence(detected_type, data)
+
+ return EntityPreview(
+ row_index=row_idx,
+ data=data,
+ detected_type=detected_type,
+ primary_value=primary_value,
+ confidence=confidence,
+ )
+
+
+def _determine_confidence(entity_type: str, data: Dict[str, Any]) -> str:
+ """
+ Determine confidence level for entity type detection.
+
+ Args:
+ entity_type: Detected entity type
+ data: Row data
+
+ Returns:
+ "high", "medium", or "low"
+ """
+ if entity_type == "Unknown":
+ return "low"
+
+ # Normalize keys
+ keys = [k.lower().strip() for k in data.keys()]
+
+ # Check if column names strongly indicate the type
+ type_keywords = {
+ "Email": ["email", "mail", "e-mail"],
+ "Ip": ["ip", "ip_address", "ipv4", "ipv6"],
+ "Domain": ["domain", "hostname"],
+ "Website": ["url", "website", "web"],
+ "Phone": ["phone", "telephone", "tel", "mobile"],
+ "ASN": ["asn", "as_number"],
+ "Username": ["username", "handle", "user"],
+ "Organization": ["organization", "org", "company"],
+ "Individual": ["name", "person", "individual"],
+ }
+
+ keywords = type_keywords.get(entity_type, [])
+ if any(keyword in key for key in keys for keyword in keywords):
+ return "high"
+
+ # Medium confidence if type was detected but no explicit column names
+ if entity_type != "Unknown":
+ return "medium"
+
+ return "low"
+
+
+def _create_result(
+ entities: List[EntityPreview], columns: List[str]
+) -> FileParseResult:
+ """Create FileParseResult from entities."""
+ # Calculate type distribution
+ type_distribution: Dict[str, int] = {}
+ for entity in entities:
+ type_name = entity.detected_type
+ type_distribution[type_name] = type_distribution.get(type_name, 0) + 1
+
+ return FileParseResult(
+ entities=entities,
+ total_entities=len(entities),
+ type_distribution=type_distribution,
+ columns=columns,
+ )
diff --git a/flowsint-core/src/flowsint_core/imports/type_matcher.py b/flowsint-core/src/flowsint_core/imports/type_matcher.py
new file mode 100644
index 00000000..67cb8126
--- /dev/null
+++ b/flowsint-core/src/flowsint_core/imports/type_matcher.py
@@ -0,0 +1,291 @@
+"""
+Entity type matching based on flowsint_types.
+Determines which entity type best matches the row data.
+"""
+
+from typing import Optional, Dict, Any, List
+import re
+import ipaddress
+
+# Import all entity types from flowsint_types
+try:
+ from flowsint_types import (
+ Email, Domain, Ip, ASN, Phone, Username, Website,
+ Organization, Individual, SocialProfile, Device,
+ Credential, Malware, SSLCertificate, Document, File
+ )
+ FLOWSINT_TYPES_AVAILABLE = True
+except ImportError:
+ FLOWSINT_TYPES_AVAILABLE = False
+
+
+# Registry of entity type matchers
+# Each entry: (type_name, class_name, identifier_fields, matcher_function)
+TYPE_REGISTRY: List[tuple] = [
+ ("Email", "Email", ["email", "mail", "e-mail", "address"], lambda d: _match_email(d)),
+ ("Ip", "Ip", ["ip", "ip_address", "ipv4", "ipv6", "address"], lambda d: _match_ip(d)),
+ ("Domain", "Domain", ["domain", "hostname", "host"], lambda d: _match_domain(d)),
+ ("Website", "Website", ["url", "website", "web", "link"], lambda d: _match_website(d)),
+ ("Phone", "Phone", ["phone", "telephone", "tel", "mobile", "cell"], lambda d: _match_phone(d)),
+ ("ASN", "ASN", ["asn", "as_number"], lambda d: _match_asn(d)),
+ ("Username", "Username", ["username", "user", "handle", "login"], lambda d: _match_username(d)),
+ ("Organization", "Organization", ["organization", "org", "company", "name"], lambda d: _match_organization(d)),
+ ("Individual", "Individual", ["name", "person", "individual", "fullname"], lambda d: _match_individual(d)),
+ ("SocialProfile", "SocialProfile", ["social", "profile", "account"], lambda d: _match_social(d)),
+ ("Credential", "Credential", ["credential", "password", "creds"], lambda d: _match_credential(d)),
+]
+
+
+def detect_entity_type_from_row(row_data: Dict[str, Any]) -> Optional[str]:
+ """
+ Detect the most likely entity type for a row of data.
+
+ Args:
+ row_data: Dictionary of column_name: value pairs
+
+ Returns:
+ Entity type name (e.g., "Email", "Domain", "Ip") or None
+ """
+ if not row_data:
+ return None
+
+ # Normalize keys to lowercase for comparison, skip None keys
+ normalized_data = {
+ k.lower().strip(): v
+ for k, v in row_data.items()
+ if v and k is not None
+ }
+
+ # Try to match each type
+ scores = {}
+ for type_name, class_name, identifier_fields, matcher_func in TYPE_REGISTRY:
+ score = 0
+
+ # Check if any identifier field matches
+ for field in identifier_fields:
+ if field in normalized_data:
+ score += 10
+
+ # Use matcher function to validate
+ try:
+ if matcher_func(normalized_data):
+ score += 5
+ except Exception:
+ pass
+
+ if score > 0:
+ scores[type_name] = score
+
+ # Return type with highest score
+ if scores:
+ return max(scores, key=scores.get)
+
+ return None
+
+
+def get_primary_identifier(row_data: Dict[str, Any], entity_type: str) -> Optional[str]:
+ """
+ Get the primary identifier value for an entity based on its type.
+
+ Args:
+ row_data: Dictionary of column_name: value pairs
+ entity_type: The detected entity type
+
+ Returns:
+ The primary identifier value (e.g., email address, domain name, IP)
+ """
+ normalized_data = {k.lower().strip(): v for k, v in row_data.items() if v}
+
+ # Find the registry entry for this type
+ for type_name, class_name, identifier_fields, _ in TYPE_REGISTRY:
+ if type_name == entity_type:
+ # Look for identifier fields in order of priority
+ for field in identifier_fields:
+ if field in normalized_data:
+ return str(normalized_data[field]).strip()
+
+ # Fallback: try common value fields
+ for key in ["value", "name", "identifier", "id"]:
+ if key in normalized_data:
+ return str(normalized_data[key]).strip()
+
+ # Last resort: return first non-empty value
+ for value in row_data.values():
+ if value and str(value).strip():
+ return str(value).strip()
+
+ return None
+
+
+# Matcher functions for each entity type
+
+def _match_email(data: Dict[str, Any]) -> bool:
+ """Check if data looks like an email entity."""
+ email_pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
+
+ for key, value in data.items():
+ if any(k in key for k in ["email", "mail"]):
+ return bool(re.match(email_pattern, str(value)))
+
+ # Check if any value looks like an email
+ for value in data.values():
+ if re.match(email_pattern, str(value)):
+ return True
+
+ return False
+
+
+def _match_ip(data: Dict[str, Any]) -> bool:
+ """Check if data looks like an IP entity."""
+ for key, value in data.items():
+ if any(k in key for k in ["ip", "address"]) and "email" not in key:
+ try:
+ ipaddress.ip_address(str(value))
+ return True
+ except ValueError:
+ pass
+
+ # Check if any value looks like an IP
+ for value in data.values():
+ try:
+ ipaddress.ip_address(str(value))
+ return True
+ except (ValueError, AttributeError):
+ pass
+
+ return False
+
+
+def _match_domain(data: Dict[str, Any]) -> bool:
+ """Check if data looks like a domain entity."""
+ domain_pattern = r'^(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}$'
+
+ for key, value in data.items():
+ if any(k in key for k in ["domain", "hostname", "host"]):
+ return bool(re.match(domain_pattern, str(value)))
+
+ # Check if any value looks like a domain (but not email)
+ for value in data.values():
+ val_str = str(value)
+ if re.match(domain_pattern, val_str) and "@" not in val_str and not val_str.startswith("http"):
+ return True
+
+ return False
+
+
+def _match_website(data: Dict[str, Any]) -> bool:
+ """Check if data looks like a website entity."""
+ url_pattern = r'^https?://'
+
+ for key, value in data.items():
+ if any(k in key for k in ["url", "website", "web", "link"]):
+ return bool(re.match(url_pattern, str(value), re.IGNORECASE))
+
+ # Check if any value looks like a URL
+ for value in data.values():
+ if re.match(url_pattern, str(value), re.IGNORECASE):
+ return True
+
+ return False
+
+
+def _match_phone(data: Dict[str, Any]) -> bool:
+ """Check if data looks like a phone entity."""
+ for key, value in data.items():
+ if any(k in key for k in ["phone", "tel", "mobile", "cell"]):
+ # Remove common separators
+ cleaned = re.sub(r'[\s\-\(\)\.]', '', str(value))
+ phone_pattern = r'^\+?[0-9]{7,15}$'
+ return bool(re.match(phone_pattern, cleaned))
+
+ return False
+
+
+def _match_asn(data: Dict[str, Any]) -> bool:
+ """Check if data looks like an ASN entity."""
+ asn_pattern = r'^AS\d+$'
+
+ for key, value in data.items():
+ if "asn" in key or "as_number" in key or "as_name" in key:
+ return bool(re.match(asn_pattern, str(value), re.IGNORECASE))
+
+ # Check if any value looks like an ASN
+ for value in data.values():
+ if re.match(asn_pattern, str(value), re.IGNORECASE):
+ return True
+
+ return False
+
+
+def _match_username(data: Dict[str, Any]) -> bool:
+ """Check if data looks like a username entity."""
+ username_pattern = r'^@?[a-zA-Z0-9_]{3,30}$'
+
+ for key, value in data.items():
+ if any(k in key for k in ["username", "user", "handle"]):
+ val_str = str(value)
+ if re.match(username_pattern, val_str):
+ return val_str.startswith('@') or not val_str.isdigit()
+
+ # Fallback: check if any value looks like a username
+ # (but not if it looks like other types)
+ for key, value in data.items():
+ val_str = str(value)
+ if re.match(username_pattern, val_str):
+ # Make sure it's not an email or domain
+ if '@' not in val_str and '.' not in val_str:
+ # If it starts with @ or is not purely numeric, likely a username
+ if val_str.startswith('@') or not val_str.isdigit():
+ return True
+
+ return False
+
+
+def _match_organization(data: Dict[str, Any]) -> bool:
+ """Check if data looks like an organization entity."""
+ # Look for organization-specific fields
+ org_fields = ["organization", "org", "company", "siren", "siret", "company_name"]
+
+ for key in data.keys():
+ if any(field in key for field in org_fields):
+ return True
+
+ # If has "name" but no individual-specific fields
+ if "name" in data:
+ individual_fields = ["firstname", "lastname", "first_name", "last_name", "email"]
+ has_individual_fields = any(field in data for field in individual_fields)
+ return not has_individual_fields
+
+ return False
+
+
+def _match_individual(data: Dict[str, Any]) -> bool:
+ """Check if data looks like an individual entity."""
+ # Look for individual-specific fields
+ individual_fields = ["firstname", "lastname", "first_name", "last_name", "fullname", "person"]
+
+ for key in data.keys():
+ if any(field in key for field in individual_fields):
+ return True
+
+ return False
+
+
+def _match_social(data: Dict[str, Any]) -> bool:
+ """Check if data looks like a social profile entity."""
+ social_fields = ["twitter", "linkedin", "facebook", "instagram", "github", "social"]
+
+ for key in data.keys():
+ if any(field in key for field in social_fields):
+ return True
+
+ return False
+
+
+def _match_credential(data: Dict[str, Any]) -> bool:
+ """Check if data looks like a credential entity."""
+ # Look for credential-specific fields
+ if ("password" in data or "pass" in data) and ("username" in data or "user" in data or "email" in data):
+ return True
+
+ return False
diff --git a/flowsint-core/tests/import/test_entity_detection.py b/flowsint-core/tests/import/test_entity_detection.py
new file mode 100644
index 00000000..529d0134
--- /dev/null
+++ b/flowsint-core/tests/import/test_entity_detection.py
@@ -0,0 +1,299 @@
+"""Unit tests for entity_detection module."""
+import pytest
+from flowsint_core.imports.entity_detection import (
+ detect_entity_type,
+ is_email,
+ is_domain,
+ is_ip_address,
+ is_website,
+ is_phone,
+ is_asn,
+ is_username,
+ get_default_label,
+)
+
+
+class TestEmailDetection:
+ """Tests for email detection."""
+
+ def test_valid_emails(self):
+ """Test detection of valid email addresses."""
+ valid_emails = [
+ ]
+ for email in valid_emails:
+ assert is_email(email), f"Failed to detect valid email: {email}"
+ assert detect_entity_type(email) == "Email"
+
+ def test_invalid_emails(self):
+ """Test rejection of invalid email addresses."""
+ invalid_emails = [
+ "not-an-email",
+ "@example.com",
+ "user@",
+ "user@domain",
+ "",
+ ]
+ for email in invalid_emails:
+ assert not is_email(email), f"Incorrectly detected as email: {email}"
+
+
+class TestDomainDetection:
+ """Tests for domain detection."""
+
+ def test_valid_domains(self):
+ """Test detection of valid domains."""
+ valid_domains = [
+ "example.com",
+ "sub.domain.com",
+ "my-domain.co.uk",
+ "test123.org",
+ "a.b.c.d.example.com",
+ ]
+ for domain in valid_domains:
+ assert is_domain(domain), f"Failed to detect valid domain: {domain}"
+ assert detect_entity_type(domain) == "Domain"
+
+ def test_invalid_domains(self):
+ """Test rejection of invalid domains."""
+ invalid_domains = [
+ "not-a-domain",
+ ".com",
+ "domain.",
+ "http://example.com", # This is a website, not a domain
+ "",
+ "192.168.1.1", # This is an IP
+ ]
+ for domain in invalid_domains:
+ assert not is_domain(domain), f"Incorrectly detected as domain: {domain}"
+
+
+class TestIPAddressDetection:
+ """Tests for IP address detection."""
+
+ def test_valid_ipv4(self):
+ """Test detection of valid IPv4 addresses."""
+ valid_ips = [
+ "192.168.1.1",
+ "10.0.0.1",
+ "172.16.0.1",
+ "8.8.8.8",
+ "255.255.255.255",
+ ]
+ for ip in valid_ips:
+ assert is_ip_address(ip), f"Failed to detect valid IPv4: {ip}"
+ assert detect_entity_type(ip) == "IP"
+
+ def test_valid_ipv6(self):
+ """Test detection of valid IPv6 addresses."""
+ valid_ips = [
+ "2001:0db8:85a3:0000:0000:8a2e:0370:7334",
+ "2001:db8::1",
+ "::1",
+ "fe80::1",
+ ]
+ for ip in valid_ips:
+ assert is_ip_address(ip), f"Failed to detect valid IPv6: {ip}"
+ assert detect_entity_type(ip) == "IP"
+
+ def test_invalid_ips(self):
+ """Test rejection of invalid IP addresses."""
+ invalid_ips = [
+ "256.256.256.256",
+ "192.168.1",
+ "not-an-ip",
+ "",
+ ]
+ for ip in invalid_ips:
+ assert not is_ip_address(ip), f"Incorrectly detected as IP: {ip}"
+
+
+class TestWebsiteDetection:
+ """Tests for website/URL detection."""
+
+ def test_valid_websites(self):
+ """Test detection of valid URLs."""
+ valid_urls = [
+ "http://example.com",
+ "https://www.example.com",
+ "http://sub.domain.com/path",
+ "https://example.com:8080/path?query=value",
+ ]
+ for url in valid_urls:
+ assert is_website(url), f"Failed to detect valid URL: {url}"
+ assert detect_entity_type(url) == "Website"
+
+ def test_invalid_websites(self):
+ """Test rejection of invalid URLs."""
+ invalid_urls = [
+ "example.com",
+ "ftp://example.com",
+ "not-a-url",
+ "",
+ ]
+ for url in invalid_urls:
+ assert not is_website(url), f"Incorrectly detected as website: {url}"
+
+
+class TestPhoneDetection:
+ """Tests for phone number detection."""
+
+ def test_valid_phones(self):
+ """Test detection of valid phone numbers."""
+ valid_phones = [
+ "+1234567890",
+ "123-456-7890",
+ "(123) 456-7890",
+ "+44 20 7123 4567",
+ "555.123.4567",
+ "5551234567",
+ ]
+ for phone in valid_phones:
+ assert is_phone(phone), f"Failed to detect valid phone: {phone}"
+ assert detect_entity_type(phone) == "Phone"
+
+ def test_invalid_phones(self):
+ """Test rejection of invalid phone numbers."""
+ invalid_phones = [
+ "123", # Too short
+ "abc-def-ghij",
+ "",
+ "1234567890123456", # Too long
+ ]
+ for phone in invalid_phones:
+ assert not is_phone(phone), f"Incorrectly detected as phone: {phone}"
+
+
+class TestASNDetection:
+ """Tests for ASN detection."""
+
+ def test_valid_asns(self):
+ """Test detection of valid ASNs."""
+ valid_asns = [
+ "AS13335",
+ "as64512",
+ "AS1",
+ "AS4294967295",
+ ]
+ for asn in valid_asns:
+ assert is_asn(asn), f"Failed to detect valid ASN: {asn}"
+ assert detect_entity_type(asn) == "ASN"
+
+ def test_invalid_asns(self):
+ """Test rejection of invalid ASNs."""
+ invalid_asns = [
+ "13335",
+ "ASN13335",
+ "AS",
+ "",
+ ]
+ for asn in invalid_asns:
+ assert not is_asn(asn), f"Incorrectly detected as ASN: {asn}"
+
+
+class TestUsernameDetection:
+ """Tests for username detection."""
+
+ def test_valid_usernames(self):
+ """Test detection of valid usernames."""
+ valid_usernames = [
+ "@john_doe",
+ "@user123",
+ "username",
+ "user_name_123",
+ ]
+ for username in valid_usernames:
+ assert is_username(username), f"Failed to detect valid username: {username}"
+ assert detect_entity_type(username) == "Username"
+
+ def test_invalid_usernames(self):
+ """Test rejection of invalid usernames."""
+ invalid_usernames = [
+ "ab", # Too short
+ "@",
+ "123456789", # Purely numeric
+ "[email protected]", # This is an email
+ "",
+ "a" * 31, # Too long
+ ]
+ for username in invalid_usernames:
+ assert not is_username(username), f"Incorrectly detected as username: {username}"
+
+
+class TestDetectionPriority:
+ """Tests for detection priority and edge cases."""
+
+ def test_email_vs_username(self):
+ """Email should take priority over username."""
+ assert detect_entity_type("[email protected]") == "Email"
+
+ def test_url_vs_domain(self):
+ """URL should take priority over domain."""
+ assert detect_entity_type("http://example.com") == "Website"
+ assert detect_entity_type("example.com") == "Domain"
+
+ def test_ip_vs_domain(self):
+ """IP should be correctly distinguished from domain."""
+ assert detect_entity_type("192.168.1.1") == "IP"
+ assert detect_entity_type("example.com") == "Domain"
+
+ def test_whitespace_handling(self):
+ """Test that whitespace is properly handled."""
+ assert detect_entity_type(" [email protected] ") == "Email"
+ assert detect_entity_type(" example.com ") == "Domain"
+
+ def test_none_and_empty(self):
+ """Test handling of None and empty strings."""
+ assert detect_entity_type("") is None
+ assert detect_entity_type(" ") is None
+ assert detect_entity_type(None) is None
+
+
+class TestGetDefaultLabel:
+ """Tests for get_default_label function."""
+
+ def test_email_label(self):
+ """Test label generation for email."""
+ assert get_default_label("Email", "[email protected]") == "[email protected]"
+
+ def test_domain_label(self):
+ """Test label generation for domain."""
+ assert get_default_label("Domain", "example.com") == "example.com"
+
+ def test_ip_label(self):
+ """Test label generation for IP."""
+ assert get_default_label("IP", "192.168.1.1") == "192.168.1.1"
+
+ def test_username_label(self):
+ """Test label generation for username (strips @ prefix)."""
+ assert get_default_label("Username", "@john_doe") == "john_doe"
+ assert get_default_label("Username", "john_doe") == "john_doe"
+
+ def test_phone_label(self):
+ """Test label generation for phone."""
+ assert get_default_label("Phone", "+1234567890") == "+1234567890"
+
+ def test_asn_label(self):
+ """Test label generation for ASN."""
+ assert get_default_label("ASN", "AS13335") == "AS13335"
+
+ def test_website_label(self):
+ """Test label generation for website."""
+ assert get_default_label("Website", "https://example.com") == "https://example.com"
+
+ def test_organization_label(self):
+ """Test label generation for organization."""
+ assert get_default_label("Organization", "ACME Corp") == "ACME Corp"
+
+ def test_individual_label(self):
+ """Test label generation for individual."""
+ assert get_default_label("Individual", "John Doe") == "John Doe"
+
+ def test_unknown_type(self):
+ """Test label generation for unknown type."""
+ assert get_default_label("UnknownType", "some value") == "some value"
diff --git a/flowsint-core/tests/import/test_import_integration.py b/flowsint-core/tests/import/test_import_integration.py
new file mode 100644
index 00000000..0612a6b6
--- /dev/null
+++ b/flowsint-core/tests/import/test_import_integration.py
@@ -0,0 +1,294 @@
+"""Integration tests for import functionality (entity-based)."""
+import pytest
+from flowsint_core.imports import parse_file
+
+
+class TestEntityBasedImportWorkflow:
+ """Integration tests for entity-based import workflow."""
+
+ def test_full_entity_workflow(self):
+ """Test complete workflow: parse file -> detect entities -> prepare for import."""
+ # Step 1: User uploads a CSV file (each row = one entity)
+ csv_content = b"""email,name,department
[email protected],John Admin,IT
[email protected],Jane User,HR
[email protected],Bob Security,Security
+"""
+
+ # Step 2: Parse and analyze the file
+ result = parse_file(csv_content, "test.csv")
+
+ # Verify parsing - entities instead of rows
+ assert result.total_entities == 3
+ assert len(result.entities) == 3
+
+ # Verify entity detection
+ assert all(e.detected_type == "Email" for e in result.entities)
+
+ # Step 3: User reviews entities (can change type/label)
+ entity_mappings = []
+ for entity in result.entities:
+ entity_mappings.append({
+ "row_index": entity.row_index,
+ "entity_type": entity.detected_type,
+ "label": entity.primary_value,
+ "include": True
+ })
+
+ # Verify mappings
+ assert len(entity_mappings) == 3
+ assert all(m["entity_type"] == "Email" for m in entity_mappings)
+
+ # Step 4: Verify all data is preserved in entity
+ first_entity = result.entities[0]
+ assert "email" in first_entity.data
+ assert "name" in first_entity.data
+ assert "department" in first_entity.data
+
+ def test_txt_entity_workflow(self):
+ """Test workflow for TXT file where each line is an entity."""
+ txt_content = b"""example.com
+192.168.1.1
+AS13335
+"""
+
+ result = parse_file(txt_content, "entities.txt")
+
+ assert result.total_entities == 4
+ assert len(result.entities) == 4
+
+ # Each line is detected as different type
+ types = {e.detected_type for e in result.entities}
+ assert "Domain" in types
+ assert "Ip" in types
+ assert "Email" in types
+ assert "ASN" in types
+
+ def test_user_corrections_workflow(self):
+ """Test workflow where user corrects detected types."""
+ csv_content = b"""identifier,category,notes
+john_doe,username,Social media handle
+example.com,domain,Main website
+192.168.1.1,server,Production server
+"""
+
+ # Parse file
+ result = parse_file(csv_content, "data.csv")
+
+ # Simulate user reviewing and correcting types
+ entity_mappings = []
+ for entity in result.entities:
+ # User can override detected type
+ if entity.data.get("category") == "server":
+ # User wants custom type instead of detected "Ip"
+ entity_type = "Device"
+ else:
+ entity_type = entity.detected_type
+
+ entity_mappings.append({
+ "row_index": entity.row_index,
+ "entity_type": entity_type,
+ "label": entity.primary_value,
+ "include": True
+ })
+
+ # Verify overrides
+ assert entity_mappings[0]["entity_type"] == "Username"
+ assert entity_mappings[1]["entity_type"] == "Domain"
+ assert entity_mappings[2]["entity_type"] == "Device" # User override
+
+ def test_label_customization(self):
+ """Test customizing labels for entities."""
+ csv_content = b"""email,full_name,role
[email protected],John Administrator,Admin
[email protected],Jane Security,SecOps
+"""
+
+ result = parse_file(csv_content, "contacts.csv")
+
+ # User customizes labels with additional context
+ entity_mappings = []
+ for entity in result.entities:
+ # Combine email with full name for label
+ custom_label = f"{entity.data['email']} ({entity.data['full_name']})"
+ entity_mappings.append({
+ "row_index": entity.row_index,
+ "entity_type": entity.detected_type,
+ "label": custom_label,
+ "include": True
+ })
+
+ # Verify customized labels
+ assert "John Administrator" in entity_mappings[0]["label"]
+ assert "Jane Security" in entity_mappings[1]["label"]
+
+ def test_selective_import(self):
+ """Test excluding specific entities from import."""
+ csv_content = b"""email,status
+"""
+
+ result = parse_file(csv_content, "users.csv")
+
+ # User only wants to import active users
+ entity_mappings = []
+ for entity in result.entities:
+ include = entity.data.get("status") == "active"
+ entity_mappings.append({
+ "row_index": entity.row_index,
+ "entity_type": entity.detected_type,
+ "label": entity.primary_value,
+ "include": include
+ })
+
+ # Verify selective inclusion
+ included = [m for m in entity_mappings if m["include"]]
+ assert len(included) == 2
+
+
+class TestRealWorldEntityScenarios:
+ """Real-world scenario tests with entity-based approach."""
+
+ def test_organization_import(self):
+ """Test importing organizations with metadata."""
+ csv_content = b"""name,industry,country,employees
+Google,Technology,USA,150000
+Microsoft,Software,USA,220000
+Apple,Technology,USA,164000
+"""
+
+ result = parse_file(csv_content, "orgs.csv")
+
+ assert result.total_entities == 3
+ assert all(e.detected_type == "Organization" for e in result.entities)
+
+ # Verify all metadata preserved
+ for entity in result.entities:
+ assert "name" in entity.data
+ assert "industry" in entity.data
+ assert "country" in entity.data
+ assert "employees" in entity.data
+
+ def test_threat_intel_import(self):
+ """Test importing threat intelligence data."""
+ csv_content = b"""domain,severity,first_seen,description,malware_family
+malicious.com,high,2024-01-15,Phishing domain,TrickBot
+evil.net,critical,2024-01-16,C2 server,Cobalt Strike
+threat.xyz,high,2024-01-18,APT infrastructure,Unknown
+"""
+
+ result = parse_file(csv_content, "iocs.csv")
+
+ assert result.total_entities == 3
+ assert all(e.detected_type == "Domain" for e in result.entities)
+
+ # Verify threat metadata preserved in entity data
+ first_entity = result.entities[0]
+ assert first_entity.data["severity"] == "high"
+ assert first_entity.data["malware_family"] == "TrickBot"
+ assert "description" in first_entity.data
+
+ def test_infrastructure_mapping(self):
+ """Test importing infrastructure with multiple related fields."""
+ csv_content = b"""domain,ip,asn,country,provider
+cloudflare.com,1.1.1.1,AS13335,USA,Cloudflare
+google.com,8.8.8.8,AS15169,USA,Google
+"""
+
+ result = parse_file(csv_content, "infrastructure.csv")
+
+ assert result.total_entities == 2
+
+ # All related data preserved in entity
+ for entity in result.entities:
+ assert "domain" in entity.data
+ assert "ip" in entity.data
+ assert "asn" in entity.data
+ assert "country" in entity.data
+ assert "provider" in entity.data
+
+
+class TestTypeDistribution:
+ """Tests for type distribution in entity-based approach."""
+
+ def test_mixed_entity_types(self):
+ """Test file with multiple entity types."""
+ txt_content = b"""example.com
+test.org
+192.168.1.1
+10.0.0.1
+"""
+
+ result = parse_file(txt_content, "mixed.txt")
+
+ # Check type distribution
+ assert result.type_distribution["Domain"] == 2
+ assert result.type_distribution["Email"] == 2
+ assert result.type_distribution["Ip"] == 2
+
+ def test_predominant_type_detection(self):
+ """Test that predominant type is correctly identified."""
+ csv_content = b"""email,notes
[email protected],User account
[email protected],Admin account
[email protected],Guest account
[email protected],Service account
+"""
+
+ result = parse_file(csv_content, "emails.csv")
+
+ # Should have high confidence for Email type
+ assert all(e.detected_type == "Email" for e in result.entities)
+ assert all(e.confidence == "high" for e in result.entities)
+
+
+class TestEdgeCases:
+ """Edge case tests for entity-based import."""
+
+ def test_single_column_csv(self):
+ """Test CSV with single column."""
+ csv_content = b"""email
+"""
+
+ result = parse_file(csv_content, "single_column.csv")
+
+ assert result.total_entities == 2
+ assert all(e.detected_type == "Email" for e in result.entities)
+
+ def test_many_columns_csv(self):
+ """Test CSV with many columns (all become entity data)."""
+ csv_content = b"""email,name,dept,role,location,phone,manager,start_date
[email protected],John,IT,Dev,NYC,555-1234,[email protected],2024-01-01
+"""
+
+ result = parse_file(csv_content, "detailed.csv")
+
+ assert result.total_entities == 1
+ entity = result.entities[0]
+
+ # All 8 columns should be in entity data
+ assert len(entity.data) == 8
+ assert "email" in entity.data
+ assert "manager" in entity.data
+ assert "start_date" in entity.data
+
+ def test_unknown_type_handling(self):
+ """Test handling of unknown/undetectable types."""
+ txt_content = b"""random text string
+another random value
+unstructured data
+"""
+
+ result = parse_file(txt_content, "unknown.txt")
+
+ # Should still parse but mark as Unknown
+ assert result.total_entities == 3
+ assert all(e.detected_type == "Unknown" for e in result.entities)
+ assert all(e.confidence == "low" for e in result.entities)