A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat(core): import data + tests
dextmorgn0f39813c9c26
diff --git a/flowsint-core/poetry.lock b/flowsint-core/poetry.lockindex c1221cc5..6cd3be32 100644--- a/flowsint-core/poetry.lock+++ b/flowsint-core/poetry.lock@@ -6469,4 +6469,4 @@ propcache = ">=0.2.1"[metadata]lock-version = "2.1"python-versions = ">=3.12,<4.0"-content-hash = "4833b73e3dedd38d32f82bae3c5ede7e6ceaec7e403dc1ca2ded467d6fe2acbf"+content-hash = "36475c223c1801a494e022f8cfb7aa18c12405405b849b84c92b403189994bcc"diff --git a/flowsint-core/pyproject.toml b/flowsint-core/pyproject.tomlindex 72fa73ab..9f908431 100644--- a/flowsint-core/pyproject.toml+++ b/flowsint-core/pyproject.toml@@ -28,6 +28,7 @@ python-multipart = "^0.0.20"docker = "^7.1.0"pytest = "^8.4.2"cryptography = "^45.0.7"+openpyxl = "^3.1"[tool.poetry.group.dev.dependencies]pytest-asyncio = "^0.21"diff --git a/flowsint-core/src/flowsint_core/imports/__init__.py b/flowsint-core/src/flowsint_core/imports/__init__.pynew file mode 100644index 00000000..0c77f5cf--- /dev/null+++ b/flowsint-core/src/flowsint_core/imports/__init__.py@@ -0,0 +1,17 @@+"""+Import utilities for entity parsing and type detection.+"""++from .entity_detection import detect_entity_type, get_default_label+from .file_parser import parse_file, FileParseResult, EntityPreview+from .type_matcher import detect_entity_type_from_row, get_primary_identifier++__all__ = [+ "detect_entity_type",+ "get_default_label",+ "parse_file",+ "FileParseResult",+ "EntityPreview",+ "detect_entity_type_from_row",+ "get_primary_identifier",+]diff --git a/flowsint-core/src/flowsint_core/imports/entity_detection.py b/flowsint-core/src/flowsint_core/imports/entity_detection.pynew file mode 100644index 00000000..0cbedfdc--- /dev/null+++ b/flowsint-core/src/flowsint_core/imports/entity_detection.py@@ -0,0 +1,131 @@+"""+Entity type detection utilities for import feature.+Provides basic pattern matching for common entity types.+"""++import re+from typing import Optional+import ipaddress+++# Ordered registry of detectors. Order matters to avoid false positives+# (e.g., URL should be checked before Domain).+# Each entry is a tuple of (entity_type, predicate_function)+DETECTORS = [+ ("Email", lambda v: is_email(v)),+ ("IP", lambda v: is_ip_address(v)),+ ("Website", lambda v: is_website(v)),+ ("Domain", lambda v: is_domain(v)),+ ("Phone", lambda v: is_phone(v)),+ ("ASN", lambda v: is_asn(v)),+ ("Username", lambda v: is_username(v)),+]+++def detect_entity_type(value: str) -> Optional[str]:+ """+ Detect entity type based on value pattern.+ Returns the detected entity type or None if no match.++ Args:+ value: The string value to analyze++ Returns:+ Entity type string (e.g., "Email", "Domain", "IP") or None+ """+ if not value or not isinstance(value, str):+ return None++ value = value.strip()++ # Iterate through ordered detectors to find the first match+ for entity_type, predicate in DETECTORS:+ try:+ if predicate(value):+ return entity_type+ except Exception:+ # If a predicate raises, skip it to avoid breaking detection+ # (predicates should generally be safe and return bool)+ continue++ return None+++def is_email(value: str) -> bool:+ """Check if value matches email pattern."""+ email_pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'+ return bool(re.match(email_pattern, value))+++def is_domain(value: str) -> bool:+ """Check if value matches domain pattern."""+ # Basic domain pattern: contains dots and valid characters+ domain_pattern = r'^(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}$'+ return bool(re.match(domain_pattern, value))+++def is_ip_address(value: str) -> bool:+ """Check if value is a valid IPv4 or IPv6 address."""+ try:+ ipaddress.ip_address(value)+ return True+ except ValueError:+ return False+++def is_website(value: str) -> bool:+ """Check if value matches URL/website pattern."""+ url_pattern = r'^https?://'+ return bool(re.match(url_pattern, value, re.IGNORECASE))+++def is_phone(value: str) -> bool:+ """Check if value matches phone number pattern."""+ # Remove common separators for checking+ cleaned = re.sub(r'[\s\-\(\)\.]', '', value)+ # Check if it's mostly digits and has reasonable length+ phone_pattern = r'^\+?[0-9]{7,15}$'+ return bool(re.match(phone_pattern, cleaned))+++def is_asn(value: str) -> bool:+ """Check if value matches ASN pattern."""+ asn_pattern = r'^AS\d+$'+ return bool(re.match(asn_pattern, value, re.IGNORECASE))+++def is_username(value: str) -> bool:+ """Check if value matches username pattern (social media style)."""+ # Matches @username format or simple alphanumeric with underscores+ username_pattern = r'^@?[a-zA-Z0-9_]{3,30}$'+ if re.match(username_pattern, value):+ # Additional check: starts with @ or is not purely numeric+ return value.startswith('@') or not value.lstrip('@').isdigit()+ return False+++def get_default_label(entity_type: str, value: str) -> str:+ """+ Get default label for an entity based on its type and value.++ Args:+ entity_type: The detected or selected entity type+ value: The entity value++ Returns:+ Default label string+ """+ # For most types, the value itself is a good label+ type_defaults = {+ "Email": value,+ "Domain": value,+ "IP": value,+ "Website": value,+ "Phone": value,+ "ASN": value,+ "Username": value.lstrip('@'), # Remove @ prefix for label+ "Organization": value,+ "Individual": value,+ }++ return type_defaults.get(entity_type, value)diff --git a/flowsint-core/src/flowsint_core/imports/file_parser.py b/flowsint-core/src/flowsint_core/imports/file_parser.pynew file mode 100644index 00000000..84f85596--- /dev/null+++ b/flowsint-core/src/flowsint_core/imports/file_parser.py@@ -0,0 +1,333 @@+"""+File parsing utilities for entity imports.+Handles CSV, TXT, and XLSX file formats.+Each row represents ONE entity with all columns as data properties.+"""++import csv+import io+from dataclasses import dataclass+from typing import List, Dict, Any, BinaryIO, Union, Optional+from pathlib import Path++try:+ from openpyxl import load_workbook++ OPENPYXL_AVAILABLE = True+except ImportError:+ OPENPYXL_AVAILABLE = False++from .type_matcher import detect_entity_type_from_row, get_primary_identifier+++@dataclass+class EntityPreview:+ """Preview of a single entity to be imported."""++ row_index: int+ """Index of the row (0-based)."""++ data: Dict[str, Any]+ """All column data as key-value pairs."""++ detected_type: str+ """Detected entity type (e.g., "Email", "Domain")."""++ primary_value: str+ """Primary identifier value for the entity."""++ confidence: str+ """Confidence level: high, medium, low."""+++@dataclass+class FileParseResult:+ """Result of parsing an import file."""++ entities: List[EntityPreview]+ """List of detected entities from the file."""++ total_entities: int+ """Total number of entities (rows)."""++ type_distribution: Dict[str, int]+ """Distribution of detected types: {type_name: count}."""++ columns: List[str]+ """Column names found in the file (for reference)."""+++def parse_file(+ file_content: Union[bytes, BinaryIO],+ filename: str,+ max_preview_rows: int = 100,+) -> FileParseResult:+ """+ Parse an uploaded file and analyze its contents.+ Each row is treated as a single entity.++ Args:+ file_content: File content as bytes or file-like object+ filename: Original filename (used to determine file type)+ max_preview_rows: Maximum number of rows to parse for preview++ Returns:+ FileParseResult containing parsed entities and analysis++ Raises:+ ValueError: If file format is unsupported or parsing fails+ """+ file_ext = Path(filename).suffix.lower()++ # Convert file content to bytes if it's a file-like object+ if hasattr(file_content, "read"):+ file_bytes = file_content.read()+ else:+ file_bytes = file_content++ # Parse based on file extension+ if file_ext == ".csv":+ return _parse_csv(file_bytes, max_preview_rows)+ elif file_ext == ".txt":+ return _parse_txt(file_bytes, max_preview_rows)+ elif file_ext in [".xlsx", ".xls"]:+ if not OPENPYXL_AVAILABLE:+ raise ValueError(+ "openpyxl library is required for Excel file support. "+ "Install with: pip install openpyxl"+ )+ return _parse_excel(file_bytes, max_preview_rows)+ else:+ raise ValueError(f"Unsupported file format: {file_ext}")+++def _parse_csv(+ file_bytes: bytes,+ max_preview_rows: int,+) -> FileParseResult:+ """Parse a CSV file where each row is an entity."""+ try:+ # Try to decode as UTF-8, fall back to latin-1+ try:+ text_content = file_bytes.decode("utf-8")+ except UnicodeDecodeError:+ text_content = file_bytes.decode("latin-1")++ # Parse CSV+ csv_reader = csv.DictReader(io.StringIO(text_content))++ # Extract columns+ columns = csv_reader.fieldnames or []+ if not columns:+ raise ValueError("No columns found in CSV file")++ # Parse rows as entities+ entities: List[EntityPreview] = []+ for row_idx, row in enumerate(csv_reader):+ if row_idx >= max_preview_rows:+ break++ # Skip empty rows+ if not any(v and str(v).strip() for v in row.values()):+ continue++ entity = _create_entity_preview(row_idx, row, columns)+ if entity:+ entities.append(entity)++ return _create_result(entities, columns)++ except Exception as e:+ raise ValueError(f"Failed to parse CSV file: {str(e)}")+++def _parse_txt(+ file_bytes: bytes,+ max_preview_rows: int,+) -> FileParseResult:+ """Parse a TXT file where each line is an entity with a single 'value' column."""+ try:+ # Try to decode as UTF-8, fall back to latin-1+ try:+ text_content = file_bytes.decode("utf-8")+ except UnicodeDecodeError:+ text_content = file_bytes.decode("latin-1")++ # Split by lines and filter empty lines+ lines = [line.strip() for line in text_content.split("\n")]+ lines = [line for line in lines if line]++ if not lines:+ raise ValueError("File is empty")++ # For TXT files, single column named "value"+ columns = ["value"]++ # Convert lines to entities+ entities: List[EntityPreview] = []+ for row_idx, line in enumerate(lines[:max_preview_rows]):+ row_data = {"value": line}+ entity = _create_entity_preview(row_idx, row_data, columns)+ if entity:+ entities.append(entity)++ return _create_result(entities, columns)++ except Exception as e:+ raise ValueError(f"Failed to parse TXT file: {str(e)}")+++def _parse_excel(+ file_bytes: bytes,+ max_preview_rows: int,+) -> FileParseResult:+ """Parse an Excel file where each row is an entity."""+ try:+ # Load workbook from bytes+ workbook = load_workbook(filename=io.BytesIO(file_bytes), read_only=True)++ # Use the first sheet+ sheet = workbook.active+ if sheet is None:+ raise ValueError("No active sheet found in Excel file")++ # Read all rows+ all_rows = list(sheet.iter_rows(values_only=True))++ if not all_rows:+ raise ValueError("Excel file is empty")++ # First row is header+ header_row = all_rows[0]+ data_rows = all_rows[1:]++ # Convert header to strings and handle None values+ columns = [+ str(cell) if cell is not None else f"Column_{i+1}"+ for i, cell in enumerate(header_row)+ ]++ # Convert data rows to entities+ entities: List[EntityPreview] = []+ for row_idx, row_values in enumerate(data_rows[:max_preview_rows]):+ row_dict = {}+ for i, value in enumerate(row_values):+ if i < len(columns):+ row_dict[columns[i]] = str(value) if value is not None else ""++ # Skip empty rows+ if not any(v and str(v).strip() for v in row_dict.values()):+ continue++ entity = _create_entity_preview(row_idx, row_dict, columns)+ if entity:+ entities.append(entity)++ workbook.close()++ return _create_result(entities, columns)++ except Exception as e:+ raise ValueError(f"Failed to parse Excel file: {str(e)}")+++def _create_entity_preview(+ row_idx: int, row_data: Dict[str, Any], columns: List[str]+) -> Optional[EntityPreview]:+ """+ Create an EntityPreview from a row of data.++ Args:+ row_idx: Row index+ row_data: Dictionary of column: value pairs+ columns: List of column names++ Returns:+ EntityPreview or None if row is invalid+ """+ # Filter out empty values+ data = {k: v for k, v in row_data.items() if v and str(v).strip()}++ if not data:+ return None++ # Detect entity type+ detected_type = detect_entity_type_from_row(data)+ if not detected_type:+ detected_type = "Unknown"++ # Get primary identifier+ primary_value = get_primary_identifier(data, detected_type)+ if not primary_value:+ # Fallback to first value+ primary_value = str(next(iter(data.values())))++ # Determine confidence+ confidence = _determine_confidence(detected_type, data)++ return EntityPreview(+ row_index=row_idx,+ data=data,+ detected_type=detected_type,+ primary_value=primary_value,+ confidence=confidence,+ )+++def _determine_confidence(entity_type: str, data: Dict[str, Any]) -> str:+ """+ Determine confidence level for entity type detection.++ Args:+ entity_type: Detected entity type+ data: Row data++ Returns:+ "high", "medium", or "low"+ """+ if entity_type == "Unknown":+ return "low"++ # Normalize keys+ keys = [k.lower().strip() for k in data.keys()]++ # Check if column names strongly indicate the type+ type_keywords = {+ "Email": ["email", "mail", "e-mail"],+ "Ip": ["ip", "ip_address", "ipv4", "ipv6"],+ "Domain": ["domain", "hostname"],+ "Website": ["url", "website", "web"],+ "Phone": ["phone", "telephone", "tel", "mobile"],+ "ASN": ["asn", "as_number"],+ "Username": ["username", "handle", "user"],+ "Organization": ["organization", "org", "company"],+ "Individual": ["name", "person", "individual"],+ }++ keywords = type_keywords.get(entity_type, [])+ if any(keyword in key for key in keys for keyword in keywords):+ return "high"++ # Medium confidence if type was detected but no explicit column names+ if entity_type != "Unknown":+ return "medium"++ return "low"+++def _create_result(+ entities: List[EntityPreview], columns: List[str]+) -> FileParseResult:+ """Create FileParseResult from entities."""+ # Calculate type distribution+ type_distribution: Dict[str, int] = {}+ for entity in entities:+ type_name = entity.detected_type+ type_distribution[type_name] = type_distribution.get(type_name, 0) + 1++ return FileParseResult(+ entities=entities,+ total_entities=len(entities),+ type_distribution=type_distribution,+ columns=columns,+ )diff --git a/flowsint-core/src/flowsint_core/imports/type_matcher.py b/flowsint-core/src/flowsint_core/imports/type_matcher.pynew file mode 100644index 00000000..67cb8126--- /dev/null+++ b/flowsint-core/src/flowsint_core/imports/type_matcher.py@@ -0,0 +1,291 @@+"""+Entity type matching based on flowsint_types.+Determines which entity type best matches the row data.+"""++from typing import Optional, Dict, Any, List+import re+import ipaddress++# Import all entity types from flowsint_types+try:+ from flowsint_types import (+ Email, Domain, Ip, ASN, Phone, Username, Website,+ Organization, Individual, SocialProfile, Device,+ Credential, Malware, SSLCertificate, Document, File+ )+ FLOWSINT_TYPES_AVAILABLE = True+except ImportError:+ FLOWSINT_TYPES_AVAILABLE = False+++# Registry of entity type matchers+# Each entry: (type_name, class_name, identifier_fields, matcher_function)+TYPE_REGISTRY: List[tuple] = [+ ("Email", "Email", ["email", "mail", "e-mail", "address"], lambda d: _match_email(d)),+ ("Ip", "Ip", ["ip", "ip_address", "ipv4", "ipv6", "address"], lambda d: _match_ip(d)),+ ("Domain", "Domain", ["domain", "hostname", "host"], lambda d: _match_domain(d)),+ ("Website", "Website", ["url", "website", "web", "link"], lambda d: _match_website(d)),+ ("Phone", "Phone", ["phone", "telephone", "tel", "mobile", "cell"], lambda d: _match_phone(d)),+ ("ASN", "ASN", ["asn", "as_number"], lambda d: _match_asn(d)),+ ("Username", "Username", ["username", "user", "handle", "login"], lambda d: _match_username(d)),+ ("Organization", "Organization", ["organization", "org", "company", "name"], lambda d: _match_organization(d)),+ ("Individual", "Individual", ["name", "person", "individual", "fullname"], lambda d: _match_individual(d)),+ ("SocialProfile", "SocialProfile", ["social", "profile", "account"], lambda d: _match_social(d)),+ ("Credential", "Credential", ["credential", "password", "creds"], lambda d: _match_credential(d)),+]+++def detect_entity_type_from_row(row_data: Dict[str, Any]) -> Optional[str]:+ """+ Detect the most likely entity type for a row of data.++ Args:+ row_data: Dictionary of column_name: value pairs++ Returns:+ Entity type name (e.g., "Email", "Domain", "Ip") or None+ """+ if not row_data:+ return None++ # Normalize keys to lowercase for comparison, skip None keys+ normalized_data = {+ k.lower().strip(): v+ for k, v in row_data.items()+ if v and k is not None+ }++ # Try to match each type+ scores = {}+ for type_name, class_name, identifier_fields, matcher_func in TYPE_REGISTRY:+ score = 0++ # Check if any identifier field matches+ for field in identifier_fields:+ if field in normalized_data:+ score += 10++ # Use matcher function to validate+ try:+ if matcher_func(normalized_data):+ score += 5+ except Exception:+ pass++ if score > 0:+ scores[type_name] = score++ # Return type with highest score+ if scores:+ return max(scores, key=scores.get)++ return None+++def get_primary_identifier(row_data: Dict[str, Any], entity_type: str) -> Optional[str]:+ """+ Get the primary identifier value for an entity based on its type.++ Args:+ row_data: Dictionary of column_name: value pairs+ entity_type: The detected entity type++ Returns:+ The primary identifier value (e.g., email address, domain name, IP)+ """+ normalized_data = {k.lower().strip(): v for k, v in row_data.items() if v}++ # Find the registry entry for this type+ for type_name, class_name, identifier_fields, _ in TYPE_REGISTRY:+ if type_name == entity_type:+ # Look for identifier fields in order of priority+ for field in identifier_fields:+ if field in normalized_data:+ return str(normalized_data[field]).strip()++ # Fallback: try common value fields+ for key in ["value", "name", "identifier", "id"]:+ if key in normalized_data:+ return str(normalized_data[key]).strip()++ # Last resort: return first non-empty value+ for value in row_data.values():+ if value and str(value).strip():+ return str(value).strip()++ return None+++# Matcher functions for each entity type++def _match_email(data: Dict[str, Any]) -> bool:+ """Check if data looks like an email entity."""+ email_pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'++ for key, value in data.items():+ if any(k in key for k in ["email", "mail"]):+ return bool(re.match(email_pattern, str(value)))++ # Check if any value looks like an email+ for value in data.values():+ if re.match(email_pattern, str(value)):+ return True++ return False+++def _match_ip(data: Dict[str, Any]) -> bool:+ """Check if data looks like an IP entity."""+ for key, value in data.items():+ if any(k in key for k in ["ip", "address"]) and "email" not in key:+ try:+ ipaddress.ip_address(str(value))+ return True+ except ValueError:+ pass++ # Check if any value looks like an IP+ for value in data.values():+ try:+ ipaddress.ip_address(str(value))+ return True+ except (ValueError, AttributeError):+ pass++ return False+++def _match_domain(data: Dict[str, Any]) -> bool:+ """Check if data looks like a domain entity."""+ domain_pattern = r'^(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}$'++ for key, value in data.items():+ if any(k in key for k in ["domain", "hostname", "host"]):+ return bool(re.match(domain_pattern, str(value)))++ # Check if any value looks like a domain (but not email)+ for value in data.values():+ val_str = str(value)+ if re.match(domain_pattern, val_str) and "@" not in val_str and not val_str.startswith("http"):+ return True++ return False+++def _match_website(data: Dict[str, Any]) -> bool:+ """Check if data looks like a website entity."""+ url_pattern = r'^https?://'++ for key, value in data.items():+ if any(k in key for k in ["url", "website", "web", "link"]):+ return bool(re.match(url_pattern, str(value), re.IGNORECASE))++ # Check if any value looks like a URL+ for value in data.values():+ if re.match(url_pattern, str(value), re.IGNORECASE):+ return True++ return False+++def _match_phone(data: Dict[str, Any]) -> bool:+ """Check if data looks like a phone entity."""+ for key, value in data.items():+ if any(k in key for k in ["phone", "tel", "mobile", "cell"]):+ # Remove common separators+ cleaned = re.sub(r'[\s\-\(\)\.]', '', str(value))+ phone_pattern = r'^\+?[0-9]{7,15}$'+ return bool(re.match(phone_pattern, cleaned))++ return False+++def _match_asn(data: Dict[str, Any]) -> bool:+ """Check if data looks like an ASN entity."""+ asn_pattern = r'^AS\d+$'++ for key, value in data.items():+ if "asn" in key or "as_number" in key or "as_name" in key:+ return bool(re.match(asn_pattern, str(value), re.IGNORECASE))++ # Check if any value looks like an ASN+ for value in data.values():+ if re.match(asn_pattern, str(value), re.IGNORECASE):+ return True++ return False+++def _match_username(data: Dict[str, Any]) -> bool:+ """Check if data looks like a username entity."""+ username_pattern = r'^@?[a-zA-Z0-9_]{3,30}$'++ for key, value in data.items():+ if any(k in key for k in ["username", "user", "handle"]):+ val_str = str(value)+ if re.match(username_pattern, val_str):+ return val_str.startswith('@') or not val_str.isdigit()++ # Fallback: check if any value looks like a username+ # (but not if it looks like other types)+ for key, value in data.items():+ val_str = str(value)+ if re.match(username_pattern, val_str):+ # Make sure it's not an email or domain+ if '@' not in val_str and '.' not in val_str:+ # If it starts with @ or is not purely numeric, likely a username+ if val_str.startswith('@') or not val_str.isdigit():+ return True++ return False+++def _match_organization(data: Dict[str, Any]) -> bool:+ """Check if data looks like an organization entity."""+ # Look for organization-specific fields+ org_fields = ["organization", "org", "company", "siren", "siret", "company_name"]++ for key in data.keys():+ if any(field in key for field in org_fields):+ return True++ # If has "name" but no individual-specific fields+ if "name" in data:+ individual_fields = ["firstname", "lastname", "first_name", "last_name", "email"]+ has_individual_fields = any(field in data for field in individual_fields)+ return not has_individual_fields++ return False+++def _match_individual(data: Dict[str, Any]) -> bool:+ """Check if data looks like an individual entity."""+ # Look for individual-specific fields+ individual_fields = ["firstname", "lastname", "first_name", "last_name", "fullname", "person"]++ for key in data.keys():+ if any(field in key for field in individual_fields):+ return True++ return False+++def _match_social(data: Dict[str, Any]) -> bool:+ """Check if data looks like a social profile entity."""+ social_fields = ["twitter", "linkedin", "facebook", "instagram", "github", "social"]++ for key in data.keys():+ if any(field in key for field in social_fields):+ return True++ return False+++def _match_credential(data: Dict[str, Any]) -> bool:+ """Check if data looks like a credential entity."""+ # Look for credential-specific fields+ if ("password" in data or "pass" in data) and ("username" in data or "user" in data or "email" in data):+ return True++ return Falsediff --git a/flowsint-core/tests/import/test_entity_detection.py b/flowsint-core/tests/import/test_entity_detection.pynew file mode 100644index 00000000..529d0134--- /dev/null+++ b/flowsint-core/tests/import/test_entity_detection.py@@ -0,0 +1,299 @@+"""Unit tests for entity_detection module."""+import pytest+from flowsint_core.imports.entity_detection import (+ detect_entity_type,+ is_email,+ is_domain,+ is_ip_address,+ is_website,+ is_phone,+ is_asn,+ is_username,+ get_default_label,+)+++class TestEmailDetection:+ """Tests for email detection."""++ def test_valid_emails(self):+ """Test detection of valid email addresses."""+ valid_emails = [+ "[email protected]",+ "[email protected]",+ "[email protected]",+ "[email protected]",+ "[email protected]",+ ]+ for email in valid_emails:+ assert is_email(email), f"Failed to detect valid email: {email}"+ assert detect_entity_type(email) == "Email"++ def test_invalid_emails(self):+ """Test rejection of invalid email addresses."""+ invalid_emails = [+ "not-an-email",+ "@example.com",+ "user@",+ "user@domain",+ "user [email protected]",+ "",+ ]+ for email in invalid_emails:+ assert not is_email(email), f"Incorrectly detected as email: {email}"+++class TestDomainDetection:+ """Tests for domain detection."""++ def test_valid_domains(self):+ """Test detection of valid domains."""+ valid_domains = [+ "example.com",+ "sub.domain.com",+ "my-domain.co.uk",+ "test123.org",+ "a.b.c.d.example.com",+ ]+ for domain in valid_domains:+ assert is_domain(domain), f"Failed to detect valid domain: {domain}"+ assert detect_entity_type(domain) == "Domain"++ def test_invalid_domains(self):+ """Test rejection of invalid domains."""+ invalid_domains = [+ "not-a-domain",+ ".com",+ "domain.",+ "http://example.com", # This is a website, not a domain+ "",+ "192.168.1.1", # This is an IP+ ]+ for domain in invalid_domains:+ assert not is_domain(domain), f"Incorrectly detected as domain: {domain}"+++class TestIPAddressDetection:+ """Tests for IP address detection."""++ def test_valid_ipv4(self):+ """Test detection of valid IPv4 addresses."""+ valid_ips = [+ "192.168.1.1",+ "10.0.0.1",+ "172.16.0.1",+ "8.8.8.8",+ "255.255.255.255",+ ]+ for ip in valid_ips:+ assert is_ip_address(ip), f"Failed to detect valid IPv4: {ip}"+ assert detect_entity_type(ip) == "IP"++ def test_valid_ipv6(self):+ """Test detection of valid IPv6 addresses."""+ valid_ips = [+ "2001:0db8:85a3:0000:0000:8a2e:0370:7334",+ "2001:db8::1",+ "::1",+ "fe80::1",+ ]+ for ip in valid_ips:+ assert is_ip_address(ip), f"Failed to detect valid IPv6: {ip}"+ assert detect_entity_type(ip) == "IP"++ def test_invalid_ips(self):+ """Test rejection of invalid IP addresses."""+ invalid_ips = [+ "256.256.256.256",+ "192.168.1",+ "not-an-ip",+ "",+ ]+ for ip in invalid_ips:+ assert not is_ip_address(ip), f"Incorrectly detected as IP: {ip}"+++class TestWebsiteDetection:+ """Tests for website/URL detection."""++ def test_valid_websites(self):+ """Test detection of valid URLs."""+ valid_urls = [+ "http://example.com",+ "https://www.example.com",+ "http://sub.domain.com/path",+ "https://example.com:8080/path?query=value",+ ]+ for url in valid_urls:+ assert is_website(url), f"Failed to detect valid URL: {url}"+ assert detect_entity_type(url) == "Website"++ def test_invalid_websites(self):+ """Test rejection of invalid URLs."""+ invalid_urls = [+ "example.com",+ "ftp://example.com",+ "not-a-url",+ "",+ ]+ for url in invalid_urls:+ assert not is_website(url), f"Incorrectly detected as website: {url}"+++class TestPhoneDetection:+ """Tests for phone number detection."""++ def test_valid_phones(self):+ """Test detection of valid phone numbers."""+ valid_phones = [+ "+1234567890",+ "123-456-7890",+ "(123) 456-7890",+ "+44 20 7123 4567",+ "555.123.4567",+ "5551234567",+ ]+ for phone in valid_phones:+ assert is_phone(phone), f"Failed to detect valid phone: {phone}"+ assert detect_entity_type(phone) == "Phone"++ def test_invalid_phones(self):+ """Test rejection of invalid phone numbers."""+ invalid_phones = [+ "123", # Too short+ "abc-def-ghij",+ "",+ "1234567890123456", # Too long+ ]+ for phone in invalid_phones:+ assert not is_phone(phone), f"Incorrectly detected as phone: {phone}"+++class TestASNDetection:+ """Tests for ASN detection."""++ def test_valid_asns(self):+ """Test detection of valid ASNs."""+ valid_asns = [+ "AS13335",+ "as64512",+ "AS1",+ "AS4294967295",+ ]+ for asn in valid_asns:+ assert is_asn(asn), f"Failed to detect valid ASN: {asn}"+ assert detect_entity_type(asn) == "ASN"++ def test_invalid_asns(self):+ """Test rejection of invalid ASNs."""+ invalid_asns = [+ "13335",+ "ASN13335",+ "AS",+ "",+ ]+ for asn in invalid_asns:+ assert not is_asn(asn), f"Incorrectly detected as ASN: {asn}"+++class TestUsernameDetection:+ """Tests for username detection."""++ def test_valid_usernames(self):+ """Test detection of valid usernames."""+ valid_usernames = [+ "@john_doe",+ "@user123",+ "username",+ "user_name_123",+ ]+ for username in valid_usernames:+ assert is_username(username), f"Failed to detect valid username: {username}"+ assert detect_entity_type(username) == "Username"++ def test_invalid_usernames(self):+ """Test rejection of invalid usernames."""+ invalid_usernames = [+ "ab", # Too short+ "@",+ "123456789", # Purely numeric+ "[email protected]", # This is an email+ "",+ "a" * 31, # Too long+ ]+ for username in invalid_usernames:+ assert not is_username(username), f"Incorrectly detected as username: {username}"+++class TestDetectionPriority:+ """Tests for detection priority and edge cases."""++ def test_email_vs_username(self):+ """Email should take priority over username."""+ assert detect_entity_type("[email protected]") == "Email"++ def test_url_vs_domain(self):+ """URL should take priority over domain."""+ assert detect_entity_type("http://example.com") == "Website"+ assert detect_entity_type("example.com") == "Domain"++ def test_ip_vs_domain(self):+ """IP should be correctly distinguished from domain."""+ assert detect_entity_type("192.168.1.1") == "IP"+ assert detect_entity_type("example.com") == "Domain"++ def test_whitespace_handling(self):+ """Test that whitespace is properly handled."""+ assert detect_entity_type(" [email protected] ") == "Email"+ assert detect_entity_type(" example.com ") == "Domain"++ def test_none_and_empty(self):+ """Test handling of None and empty strings."""+ assert detect_entity_type("") is None+ assert detect_entity_type(" ") is None+ assert detect_entity_type(None) is None+++class TestGetDefaultLabel:+ """Tests for get_default_label function."""++ def test_email_label(self):+ """Test label generation for email."""+ assert get_default_label("Email", "[email protected]") == "[email protected]"++ def test_domain_label(self):+ """Test label generation for domain."""+ assert get_default_label("Domain", "example.com") == "example.com"++ def test_ip_label(self):+ """Test label generation for IP."""+ assert get_default_label("IP", "192.168.1.1") == "192.168.1.1"++ def test_username_label(self):+ """Test label generation for username (strips @ prefix)."""+ assert get_default_label("Username", "@john_doe") == "john_doe"+ assert get_default_label("Username", "john_doe") == "john_doe"++ def test_phone_label(self):+ """Test label generation for phone."""+ assert get_default_label("Phone", "+1234567890") == "+1234567890"++ def test_asn_label(self):+ """Test label generation for ASN."""+ assert get_default_label("ASN", "AS13335") == "AS13335"++ def test_website_label(self):+ """Test label generation for website."""+ assert get_default_label("Website", "https://example.com") == "https://example.com"++ def test_organization_label(self):+ """Test label generation for organization."""+ assert get_default_label("Organization", "ACME Corp") == "ACME Corp"++ def test_individual_label(self):+ """Test label generation for individual."""+ assert get_default_label("Individual", "John Doe") == "John Doe"++ def test_unknown_type(self):+ """Test label generation for unknown type."""+ assert get_default_label("UnknownType", "some value") == "some value"diff --git a/flowsint-core/tests/import/test_import_integration.py b/flowsint-core/tests/import/test_import_integration.pynew file mode 100644index 00000000..0612a6b6--- /dev/null+++ b/flowsint-core/tests/import/test_import_integration.py@@ -0,0 +1,294 @@+"""Integration tests for import functionality (entity-based)."""+import pytest+from flowsint_core.imports import parse_file+++class TestEntityBasedImportWorkflow:+ """Integration tests for entity-based import workflow."""++ def test_full_entity_workflow(self):+ """Test complete workflow: parse file -> detect entities -> prepare for import."""+ # Step 1: User uploads a CSV file (each row = one entity)+ csv_content = b"""email,name,department[email protected],John Admin,IT[email protected],Jane User,HR[email protected],Bob Security,Security+"""++ # Step 2: Parse and analyze the file+ result = parse_file(csv_content, "test.csv")++ # Verify parsing - entities instead of rows+ assert result.total_entities == 3+ assert len(result.entities) == 3++ # Verify entity detection+ assert all(e.detected_type == "Email" for e in result.entities)++ # Step 3: User reviews entities (can change type/label)+ entity_mappings = []+ for entity in result.entities:+ entity_mappings.append({+ "row_index": entity.row_index,+ "entity_type": entity.detected_type,+ "label": entity.primary_value,+ "include": True+ })++ # Verify mappings+ assert len(entity_mappings) == 3+ assert all(m["entity_type"] == "Email" for m in entity_mappings)++ # Step 4: Verify all data is preserved in entity+ first_entity = result.entities[0]+ assert "email" in first_entity.data+ assert "name" in first_entity.data+ assert "department" in first_entity.data++ def test_txt_entity_workflow(self):+ """Test workflow for TXT file where each line is an entity."""+ txt_content = b"""example.com+192.168.1.1+AS13335+"""++ result = parse_file(txt_content, "entities.txt")++ assert result.total_entities == 4+ assert len(result.entities) == 4++ # Each line is detected as different type+ types = {e.detected_type for e in result.entities}+ assert "Domain" in types+ assert "Ip" in types+ assert "Email" in types+ assert "ASN" in types++ def test_user_corrections_workflow(self):+ """Test workflow where user corrects detected types."""+ csv_content = b"""identifier,category,notes+john_doe,username,Social media handle+example.com,domain,Main website+192.168.1.1,server,Production server+"""++ # Parse file+ result = parse_file(csv_content, "data.csv")++ # Simulate user reviewing and correcting types+ entity_mappings = []+ for entity in result.entities:+ # User can override detected type+ if entity.data.get("category") == "server":+ # User wants custom type instead of detected "Ip"+ entity_type = "Device"+ else:+ entity_type = entity.detected_type++ entity_mappings.append({+ "row_index": entity.row_index,+ "entity_type": entity_type,+ "label": entity.primary_value,+ "include": True+ })++ # Verify overrides+ assert entity_mappings[0]["entity_type"] == "Username"+ assert entity_mappings[1]["entity_type"] == "Domain"+ assert entity_mappings[2]["entity_type"] == "Device" # User override++ def test_label_customization(self):+ """Test customizing labels for entities."""+ csv_content = b"""email,full_name,role[email protected],John Administrator,Admin[email protected],Jane Security,SecOps+"""++ result = parse_file(csv_content, "contacts.csv")++ # User customizes labels with additional context+ entity_mappings = []+ for entity in result.entities:+ # Combine email with full name for label+ custom_label = f"{entity.data['email']} ({entity.data['full_name']})"+ entity_mappings.append({+ "row_index": entity.row_index,+ "entity_type": entity.detected_type,+ "label": custom_label,+ "include": True+ })++ # Verify customized labels+ assert "John Administrator" in entity_mappings[0]["label"]+ assert "Jane Security" in entity_mappings[1]["label"]++ def test_selective_import(self):+ """Test excluding specific entities from import."""+ csv_content = b"""email,status[email protected],active[email protected],inactive[email protected],active+"""++ result = parse_file(csv_content, "users.csv")++ # User only wants to import active users+ entity_mappings = []+ for entity in result.entities:+ include = entity.data.get("status") == "active"+ entity_mappings.append({+ "row_index": entity.row_index,+ "entity_type": entity.detected_type,+ "label": entity.primary_value,+ "include": include+ })++ # Verify selective inclusion+ included = [m for m in entity_mappings if m["include"]]+ assert len(included) == 2+++class TestRealWorldEntityScenarios:+ """Real-world scenario tests with entity-based approach."""++ def test_organization_import(self):+ """Test importing organizations with metadata."""+ csv_content = b"""name,industry,country,employees+Google,Technology,USA,150000+Microsoft,Software,USA,220000+Apple,Technology,USA,164000+"""++ result = parse_file(csv_content, "orgs.csv")++ assert result.total_entities == 3+ assert all(e.detected_type == "Organization" for e in result.entities)++ # Verify all metadata preserved+ for entity in result.entities:+ assert "name" in entity.data+ assert "industry" in entity.data+ assert "country" in entity.data+ assert "employees" in entity.data++ def test_threat_intel_import(self):+ """Test importing threat intelligence data."""+ csv_content = b"""domain,severity,first_seen,description,malware_family+malicious.com,high,2024-01-15,Phishing domain,TrickBot+evil.net,critical,2024-01-16,C2 server,Cobalt Strike+threat.xyz,high,2024-01-18,APT infrastructure,Unknown+"""++ result = parse_file(csv_content, "iocs.csv")++ assert result.total_entities == 3+ assert all(e.detected_type == "Domain" for e in result.entities)++ # Verify threat metadata preserved in entity data+ first_entity = result.entities[0]+ assert first_entity.data["severity"] == "high"+ assert first_entity.data["malware_family"] == "TrickBot"+ assert "description" in first_entity.data++ def test_infrastructure_mapping(self):+ """Test importing infrastructure with multiple related fields."""+ csv_content = b"""domain,ip,asn,country,provider+cloudflare.com,1.1.1.1,AS13335,USA,Cloudflare+google.com,8.8.8.8,AS15169,USA,Google+"""++ result = parse_file(csv_content, "infrastructure.csv")++ assert result.total_entities == 2++ # All related data preserved in entity+ for entity in result.entities:+ assert "domain" in entity.data+ assert "ip" in entity.data+ assert "asn" in entity.data+ assert "country" in entity.data+ assert "provider" in entity.data+++class TestTypeDistribution:+ """Tests for type distribution in entity-based approach."""++ def test_mixed_entity_types(self):+ """Test file with multiple entity types."""+ txt_content = b"""example.com+test.org+192.168.1.1+10.0.0.1+"""++ result = parse_file(txt_content, "mixed.txt")++ # Check type distribution+ assert result.type_distribution["Domain"] == 2+ assert result.type_distribution["Email"] == 2+ assert result.type_distribution["Ip"] == 2++ def test_predominant_type_detection(self):+ """Test that predominant type is correctly identified."""+ csv_content = b"""email,notes[email protected],User account[email protected],Admin account[email protected],Guest account[email protected],Service account+"""++ result = parse_file(csv_content, "emails.csv")++ # Should have high confidence for Email type+ assert all(e.detected_type == "Email" for e in result.entities)+ assert all(e.confidence == "high" for e in result.entities)+++class TestEdgeCases:+ """Edge case tests for entity-based import."""++ def test_single_column_csv(self):+ """Test CSV with single column."""+ csv_content = b"""email+"""++ result = parse_file(csv_content, "single_column.csv")++ assert result.total_entities == 2+ assert all(e.detected_type == "Email" for e in result.entities)++ def test_many_columns_csv(self):+ """Test CSV with many columns (all become entity data)."""+ csv_content = b"""email,name,dept,role,location,phone,manager,start_date[email protected],John,IT,Dev,NYC,555-1234,[email protected],2024-01-01+"""++ result = parse_file(csv_content, "detailed.csv")++ assert result.total_entities == 1+ entity = result.entities[0]++ # All 8 columns should be in entity data+ assert len(entity.data) == 8+ assert "email" in entity.data+ assert "manager" in entity.data+ assert "start_date" in entity.data++ def test_unknown_type_handling(self):+ """Test handling of unknown/undetectable types."""+ txt_content = b"""random text string+another random value+unstructured data+"""++ result = parse_file(txt_content, "unknown.txt")++ # Should still parse but mark as Unknown+ assert result.total_entities == 3+ assert all(e.detected_type == "Unknown" for e in result.entities)+ assert all(e.confidence == "low" for e in result.entities)