Skip to content

A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.

feat(core): type registry + better import

dextmorgn95add9c4fd79
diff --git a/flowsint-core/src/flowsint_core/core/graph_repository.py b/flowsint-core/src/flowsint_core/core/graph_repository.py
index 8377e223..e12ae02b 100644
--- a/flowsint-core/src/flowsint_core/core/graph_repository.py
+++ b/flowsint-core/src/flowsint_core/core/graph_repository.py
@@ -9,6 +9,7 @@ from typing import Dict, Any, List, Optional, Tuple
from datetime import datetime, timezone
from .graph_db import Neo4jConnection
from .graph_serializer import GraphSerializer
+from flowsint_types.registry import get_type
class GraphRepository:
@@ -31,6 +32,36 @@ class GraphRepository:
self._batch_operations: List[Tuple[str, Dict[str, Any]]] = []
self._batch_size = 100
+ @staticmethod
+ def _get_primary_field_for_type(type_name: str) -> Optional[str]:
+ """
+ Get the primary field name for a given type.
+
+ Args:
+ type_name: The type name (e.g., "Ip", "Domain", or lowercase variants)
+
+ Returns:
+ The primary field name, or None if type not found
+ """
+ # Get the type class from registry
+ type_class = get_type(type_name, case_sensitive=False)
+ if not type_class:
+ return None
+
+ # Find the primary field (marked with json_schema_extra={"primary": True})
+ model_fields = type_class.model_fields
+ for field_name, field_info in model_fields.items():
+ if field_info.json_schema_extra and field_info.json_schema_extra.get("primary"):
+ return field_name
+
+ # Fallback: use first required field or first field
+ for field_name, field_info in model_fields.items():
+ if field_info.is_required():
+ return field_name
+
+ # Last resort: first field
+ return next(iter(model_fields.keys())) if model_fields else None
+
def create_node(
self,
node_type: str,
@@ -264,14 +295,14 @@ class GraphRepository:
**properties: Any
) -> Optional[str]:
"""
- Create a node from import operation (MERGE on sketch_id + label).
+ Create a node from import operation using the type's primary key.
- This method is specifically designed for bulk imports where nodes
- are identified by their label within a sketch, rather than by
- a type-specific key property.
+ This method uses the same MERGE strategy as enrichers: it identifies nodes
+ by their primary key field (e.g., "address" for IP, "domain" for Domain)
+ combined with sketch_id, preventing duplicates between imports and enrichers.
Args:
- node_type: Node label (e.g., "Domain", "Ip")
+ node_type: Node label (e.g., "Domain", "Ip") - will be normalized to lowercase
label: Human-readable label for the node
sketch_id: Investigation sketch ID
**properties: All additional node properties
@@ -282,25 +313,47 @@ class GraphRepository:
if not self._connection:
return None
+ # Normalize node_type to lowercase (same as enrichers)
+ node_type = node_type.lower()
+
+ # Get the primary key field for this type
+ key_prop = self._get_primary_field_for_type(node_type)
+ if not key_prop:
+ # Fallback: if type not found in registry, use "label" as key
+ key_prop = "label"
+ key_value = label
+ else:
+ # Extract the primary key value from properties
+ key_value = properties.get(key_prop)
+ if not key_value:
+ # Fallback: if primary key not in properties, use label
+ key_value = label
+
# Serialize and prepare all properties
serialized_props = GraphSerializer.serialize_properties(properties)
serialized_props["type"] = node_type.lower()
+ serialized_props["sketch_id"] = sketch_id
serialized_props["label"] = label
serialized_props["caption"] = label
- serialized_props["created_at"] = datetime.now(timezone.utc).isoformat()
- query = f"""
- MERGE (n:`{node_type}` {{sketch_id: $sketch_id, label: $label}})
- ON CREATE SET n += $props
- RETURN elementId(n) as id
- """
+ # Build SET clauses (exclude sketch_id as it's in MERGE, but include key_prop)
+ set_clauses = [f"n.{prop} = ${prop}" for prop in serialized_props.keys()
+ if prop != "sketch_id"]
params = {
- "sketch_id": sketch_id,
- "label": label,
- "props": serialized_props
+ key_prop: key_value,
+ "created_at": datetime.now(timezone.utc).isoformat(),
+ **serialized_props
}
+ # MERGE on primary key + sketch_id (same as enrichers)
+ query = f"""
+ MERGE (n:{node_type} {{{key_prop}: ${key_prop}, sketch_id: $sketch_id}})
+ ON CREATE SET n.created_at = $created_at
+ SET {', '.join(set_clauses)}
+ RETURN elementId(n) as id
+ """
+
result = self._connection.query(query, params)
return result[0]["id"] if result else None
diff --git a/flowsint-core/src/flowsint_core/core/registry.py b/flowsint-core/src/flowsint_core/core/registry.py
index 59972bfe..cff123e0 100644
--- a/flowsint-core/src/flowsint_core/core/registry.py
+++ b/flowsint-core/src/flowsint_core/core/registry.py
@@ -58,6 +58,10 @@ from flowsint_enrichers.individual.to_org import IndividualToOrgEnricher
# Integration enrichers
from flowsint_enrichers.n8n.connector import N8nConnector
+# Dummy
+from flowsint_enrichers.ip.to_dummy_domains import IpToDummyDomainsEnricher
+from flowsint_enrichers.domain.to_dummy_ip import DomainToDummyIpEnricher
+
class EnricherRegistry:
@@ -197,3 +201,7 @@ EnricherRegistry.register(IndividualToDomainsEnricher)
# Integration enrichers
EnricherRegistry.register(N8nConnector)
+
+# Dummy
+EnricherRegistry.register(IpToDummyDomainsEnricher)
+EnricherRegistry.register(DomainToDummyIpEnricher)
diff --git a/flowsint-core/src/flowsint_core/imports/__init__.py b/flowsint-core/src/flowsint_core/imports/__init__.py
index 0c77f5cf..5638a9ab 100644
--- a/flowsint-core/src/flowsint_core/imports/__init__.py
+++ b/flowsint-core/src/flowsint_core/imports/__init__.py
@@ -2,16 +2,13 @@
Import utilities for entity parsing and type detection.
"""
-from .entity_detection import detect_entity_type, get_default_label
-from .file_parser import parse_file, FileParseResult, EntityPreview
-from .type_matcher import detect_entity_type_from_row, get_primary_identifier
+from .entity_detection import detect_type
+from .file_parser import parse_import_file, FileParseResult, EntityPreview, _parse_txt
__all__ = [
- "detect_entity_type",
- "get_default_label",
- "parse_file",
+ "detect_type",
+ "parse_import_file",
"FileParseResult",
"EntityPreview",
- "detect_entity_type_from_row",
- "get_primary_identifier",
+ "_parse_txt",
]
diff --git a/flowsint-core/src/flowsint_core/imports/entity_detection.py b/flowsint-core/src/flowsint_core/imports/entity_detection.py
index 0cbedfdc..b5fc90cf 100644
--- a/flowsint-core/src/flowsint_core/imports/entity_detection.py
+++ b/flowsint-core/src/flowsint_core/imports/entity_detection.py
@@ -2,130 +2,12 @@
Entity type detection utilities for import feature.
Provides basic pattern matching for common entity types.
"""
+from typing import Optional, Type
+from flowsint_types import TYPE_REGISTRY, FlowsintType
-import re
-from typing import Optional
-import ipaddress
-
-
-# Ordered registry of detectors. Order matters to avoid false positives
-# (e.g., URL should be checked before Domain).
-# Each entry is a tuple of (entity_type, predicate_function)
-DETECTORS = [
- ("Email", lambda v: is_email(v)),
- ("IP", lambda v: is_ip_address(v)),
- ("Website", lambda v: is_website(v)),
- ("Domain", lambda v: is_domain(v)),
- ("Phone", lambda v: is_phone(v)),
- ("ASN", lambda v: is_asn(v)),
- ("Username", lambda v: is_username(v)),
-]
-
-
-def detect_entity_type(value: str) -> Optional[str]:
- """
- Detect entity type based on value pattern.
- Returns the detected entity type or None if no match.
-
- Args:
- value: The string value to analyze
-
- Returns:
- Entity type string (e.g., "Email", "Domain", "IP") or None
- """
- if not value or not isinstance(value, str):
- return None
-
- value = value.strip()
-
- # Iterate through ordered detectors to find the first match
- for entity_type, predicate in DETECTORS:
- try:
- if predicate(value):
- return entity_type
- except Exception:
- # If a predicate raises, skip it to avoid breaking detection
- # (predicates should generally be safe and return bool)
- continue
+def detect_type(value: str) -> Optional[Type[FlowsintType]]:
+ for model in TYPE_REGISTRY.all_types().values():
+ if hasattr(model, "detect") and model.detect(value):
+ return model
return None
-
-
-def is_email(value: str) -> bool:
- """Check if value matches email pattern."""
- email_pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
- return bool(re.match(email_pattern, value))
-
-
-def is_domain(value: str) -> bool:
- """Check if value matches domain pattern."""
- # Basic domain pattern: contains dots and valid characters
- domain_pattern = r'^(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}$'
- return bool(re.match(domain_pattern, value))
-
-
-def is_ip_address(value: str) -> bool:
- """Check if value is a valid IPv4 or IPv6 address."""
- try:
- ipaddress.ip_address(value)
- return True
- except ValueError:
- return False
-
-
-def is_website(value: str) -> bool:
- """Check if value matches URL/website pattern."""
- url_pattern = r'^https?://'
- return bool(re.match(url_pattern, value, re.IGNORECASE))
-
-
-def is_phone(value: str) -> bool:
- """Check if value matches phone number pattern."""
- # Remove common separators for checking
- cleaned = re.sub(r'[\s\-\(\)\.]', '', value)
- # Check if it's mostly digits and has reasonable length
- phone_pattern = r'^\+?[0-9]{7,15}$'
- return bool(re.match(phone_pattern, cleaned))
-
-
-def is_asn(value: str) -> bool:
- """Check if value matches ASN pattern."""
- asn_pattern = r'^AS\d+$'
- return bool(re.match(asn_pattern, value, re.IGNORECASE))
-
-
-def is_username(value: str) -> bool:
- """Check if value matches username pattern (social media style)."""
- # Matches @username format or simple alphanumeric with underscores
- username_pattern = r'^@?[a-zA-Z0-9_]{3,30}$'
- if re.match(username_pattern, value):
- # Additional check: starts with @ or is not purely numeric
- return value.startswith('@') or not value.lstrip('@').isdigit()
- return False
-
-
-def get_default_label(entity_type: str, value: str) -> str:
- """
- Get default label for an entity based on its type and value.
-
- Args:
- entity_type: The detected or selected entity type
- value: The entity value
-
- Returns:
- Default label string
- """
- # For most types, the value itself is a good label
- type_defaults = {
- "Email": value,
- "Domain": value,
- "IP": value,
- "Website": value,
- "Phone": value,
- "ASN": value,
- "Username": value.lstrip('@'), # Remove @ prefix for label
- "Organization": value,
- "Individual": value,
- }
-
- return type_defaults.get(entity_type, value)
diff --git a/flowsint-core/src/flowsint_core/imports/file_parser.py b/flowsint-core/src/flowsint_core/imports/file_parser.py
index 84f85596..e443c77b 100644
--- a/flowsint-core/src/flowsint_core/imports/file_parser.py
+++ b/flowsint-core/src/flowsint_core/imports/file_parser.py
@@ -1,81 +1,46 @@
"""
File parsing utilities for entity imports.
-Handles CSV, TXT, and XLSX file formats.
-Each row represents ONE entity with all columns as data properties.
+Handles TXT file format only.
+Each line represents ONE entity with a single value.
"""
-import csv
-import io
from dataclasses import dataclass
-from typing import List, Dict, Any, BinaryIO, Union, Optional
+from typing import Any, Dict, List, BinaryIO, Union, Optional
from pathlib import Path
-
-try:
- from openpyxl import load_workbook
-
- OPENPYXL_AVAILABLE = True
-except ImportError:
- OPENPYXL_AVAILABLE = False
-
-from .type_matcher import detect_entity_type_from_row, get_primary_identifier
+from .entity_detection import detect_type
@dataclass
class EntityPreview:
"""Preview of a single entity to be imported."""
- row_index: int
- """Index of the row (0-based)."""
-
- data: Dict[str, Any]
- """All column data as key-value pairs."""
-
+ obj: Dict[str, Any]
detected_type: str
- """Detected entity type (e.g., "Email", "Domain")."""
- primary_value: str
- """Primary identifier value for the entity."""
- confidence: str
- """Confidence level: high, medium, low."""
+@dataclass
+class Entity:
+ """Entity class"""
+
+ type: str
+ results: List[EntityPreview]
@dataclass
class FileParseResult:
"""Result of parsing an import file."""
- entities: List[EntityPreview]
- """List of detected entities from the file."""
-
+ entities: Dict[str, Entity]
total_entities: int
- """Total number of entities (rows)."""
-
- type_distribution: Dict[str, int]
- """Distribution of detected types: {type_name: count}."""
-
- columns: List[str]
- """Column names found in the file (for reference)."""
-def parse_file(
+def parse_import_file(
file_content: Union[bytes, BinaryIO],
filename: str,
max_preview_rows: int = 100,
) -> FileParseResult:
"""
- Parse an uploaded file and analyze its contents.
- Each row is treated as a single entity.
-
- Args:
- file_content: File content as bytes or file-like object
- filename: Original filename (used to determine file type)
- max_preview_rows: Maximum number of rows to parse for preview
-
- Returns:
- FileParseResult containing parsed entities and analysis
-
- Raises:
- ValueError: If file format is unsupported or parsing fails
+ Parse an uploaded TXT file and analyze its contents.
"""
file_ext = Path(filename).suffix.lower()
@@ -85,67 +50,20 @@ def parse_file(
else:
file_bytes = file_content
- # Parse based on file extension
- if file_ext == ".csv":
- return _parse_csv(file_bytes, max_preview_rows)
- elif file_ext == ".txt":
- return _parse_txt(file_bytes, max_preview_rows)
- elif file_ext in [".xlsx", ".xls"]:
- if not OPENPYXL_AVAILABLE:
- raise ValueError(
- "openpyxl library is required for Excel file support. "
- "Install with: pip install openpyxl"
- )
- return _parse_excel(file_bytes, max_preview_rows)
- else:
- raise ValueError(f"Unsupported file format: {file_ext}")
-
-
-def _parse_csv(
- file_bytes: bytes,
- max_preview_rows: int,
-) -> FileParseResult:
- """Parse a CSV file where each row is an entity."""
- try:
- # Try to decode as UTF-8, fall back to latin-1
- try:
- text_content = file_bytes.decode("utf-8")
- except UnicodeDecodeError:
- text_content = file_bytes.decode("latin-1")
-
- # Parse CSV
- csv_reader = csv.DictReader(io.StringIO(text_content))
-
- # Extract columns
- columns = csv_reader.fieldnames or []
- if not columns:
- raise ValueError("No columns found in CSV file")
-
- # Parse rows as entities
- entities: List[EntityPreview] = []
- for row_idx, row in enumerate(csv_reader):
- if row_idx >= max_preview_rows:
- break
+ # Only support TXT files
+ if file_ext != ".txt":
+ raise ValueError(
+ f"Unsupported file format: {file_ext}. Only .txt files are supported."
+ )
- # Skip empty rows
- if not any(v and str(v).strip() for v in row.values()):
- continue
-
- entity = _create_entity_preview(row_idx, row, columns)
- if entity:
- entities.append(entity)
-
- return _create_result(entities, columns)
-
- except Exception as e:
- raise ValueError(f"Failed to parse CSV file: {str(e)}")
+ return _parse_txt(file_bytes, max_preview_rows)
def _parse_txt(
file_bytes: bytes,
max_preview_rows: int,
) -> FileParseResult:
- """Parse a TXT file where each line is an entity with a single 'value' column."""
+ """Parse a TXT file where each line is an entity with a single string value."""
try:
# Try to decode as UTF-8, fall back to latin-1
try:
@@ -160,174 +78,64 @@ def _parse_txt(
if not lines:
raise ValueError("File is empty")
- # For TXT files, single column named "value"
- columns = ["value"]
+ total_rows = len(lines)
- # Convert lines to entities
- entities: List[EntityPreview] = []
- for row_idx, line in enumerate(lines[:max_preview_rows]):
- row_data = {"value": line}
- entity = _create_entity_preview(row_idx, row_data, columns)
- if entity:
- entities.append(entity)
-
- return _create_result(entities, columns)
-
- except Exception as e:
- raise ValueError(f"Failed to parse TXT file: {str(e)}")
+ entities: Dict[str, Entity] = {}
-
-def _parse_excel(
- file_bytes: bytes,
- max_preview_rows: int,
-) -> FileParseResult:
- """Parse an Excel file where each row is an entity."""
- try:
- # Load workbook from bytes
- workbook = load_workbook(filename=io.BytesIO(file_bytes), read_only=True)
-
- # Use the first sheet
- sheet = workbook.active
- if sheet is None:
- raise ValueError("No active sheet found in Excel file")
-
- # Read all rows
- all_rows = list(sheet.iter_rows(values_only=True))
-
- if not all_rows:
- raise ValueError("Excel file is empty")
-
- # First row is header
- header_row = all_rows[0]
- data_rows = all_rows[1:]
-
- # Convert header to strings and handle None values
- columns = [
- str(cell) if cell is not None else f"Column_{i+1}"
- for i, cell in enumerate(header_row)
- ]
-
- # Convert data rows to entities
- entities: List[EntityPreview] = []
- for row_idx, row_values in enumerate(data_rows[:max_preview_rows]):
- row_dict = {}
- for i, value in enumerate(row_values):
- if i < len(columns):
- row_dict[columns[i]] = str(value) if value is not None else ""
-
- # Skip empty rows
- if not any(v and str(v).strip() for v in row_dict.values()):
- continue
-
- entity = _create_entity_preview(row_idx, row_dict, columns)
+ for _, line in enumerate(lines[:max_preview_rows]):
+ entity = _create_entity_preview(line)
if entity:
- entities.append(entity)
-
- workbook.close()
-
- return _create_result(entities, columns)
+ if entity.detected_type in entities:
+ entities[entity.detected_type].results.append(entity)
+ else:
+ entities[entity.detected_type] = Entity(
+ type=entity.detected_type, results=[entity]
+ )
+ return FileParseResult(
+ entities=entities,
+ total_entities=total_rows,
+ )
except Exception as e:
- raise ValueError(f"Failed to parse Excel file: {str(e)}")
+ # Normalize exceptions to ValueError for callers/tests
+ raise ValueError(f"Failed to parse TXT file: {str(e)}")
-def _create_entity_preview(
- row_idx: int, row_data: Dict[str, Any], columns: List[str]
-) -> Optional[EntityPreview]:
+def _create_entity_preview(row_value: str) -> Optional[EntityPreview]:
"""
Create an EntityPreview from a row of data.
Args:
- row_idx: Row index
- row_data: Dictionary of column: value pairs
- columns: List of column names
-
+ row_value: string value of the entry
Returns:
EntityPreview or None if row is invalid
"""
- # Filter out empty values
- data = {k: v for k, v in row_data.items() if v and str(v).strip()}
+ # Detect entity type (detect_type may return a class with from_string or None)
+ DetectedType = detect_type(row_value)
- if not data:
- return None
+ # If detection failed (None), provide a minimal Unknown type with from_string
+ if not DetectedType:
- # Detect entity type
- detected_type = detect_entity_type_from_row(data)
- if not detected_type:
- detected_type = "Unknown"
+ class UnknownType:
+ @classmethod
+ def from_string(cls, value: str):
+ # fallback: return raw string or a minimal wrapper
+ return value
- # Get primary identifier
- primary_value = get_primary_identifier(data, detected_type)
- if not primary_value:
- # Fallback to first value
- primary_value = str(next(iter(data.values())))
+ DetectedType = UnknownType
+ detected_name = "Unknown"
+ else:
+ # If detect_type returned a class/type, try to get a readable name
+ detected_name = getattr(DetectedType, "__name__", str(DetectedType))
- # Determine confidence
- confidence = _determine_confidence(detected_type, data)
+ # Build object using the type's from_string (expect classmethod)
+ try:
+ obj = DetectedType.from_string(row_value)
+ except Exception:
+ # On failure converting to object, return None to skip the row
+ return None
return EntityPreview(
- row_index=row_idx,
- data=data,
- detected_type=detected_type,
- primary_value=primary_value,
- confidence=confidence,
- )
-
-
-def _determine_confidence(entity_type: str, data: Dict[str, Any]) -> str:
- """
- Determine confidence level for entity type detection.
-
- Args:
- entity_type: Detected entity type
- data: Row data
-
- Returns:
- "high", "medium", or "low"
- """
- if entity_type == "Unknown":
- return "low"
-
- # Normalize keys
- keys = [k.lower().strip() for k in data.keys()]
-
- # Check if column names strongly indicate the type
- type_keywords = {
- "Email": ["email", "mail", "e-mail"],
- "Ip": ["ip", "ip_address", "ipv4", "ipv6"],
- "Domain": ["domain", "hostname"],
- "Website": ["url", "website", "web"],
- "Phone": ["phone", "telephone", "tel", "mobile"],
- "ASN": ["asn", "as_number"],
- "Username": ["username", "handle", "user"],
- "Organization": ["organization", "org", "company"],
- "Individual": ["name", "person", "individual"],
- }
-
- keywords = type_keywords.get(entity_type, [])
- if any(keyword in key for key in keys for keyword in keywords):
- return "high"
-
- # Medium confidence if type was detected but no explicit column names
- if entity_type != "Unknown":
- return "medium"
-
- return "low"
-
-
-def _create_result(
- entities: List[EntityPreview], columns: List[str]
-) -> FileParseResult:
- """Create FileParseResult from entities."""
- # Calculate type distribution
- type_distribution: Dict[str, int] = {}
- for entity in entities:
- type_name = entity.detected_type
- type_distribution[type_name] = type_distribution.get(type_name, 0) + 1
-
- return FileParseResult(
- entities=entities,
- total_entities=len(entities),
- type_distribution=type_distribution,
- columns=columns,
+ obj=obj,
+ detected_type=detected_name,
)
diff --git a/flowsint-core/src/flowsint_core/imports/type_matcher.py b/flowsint-core/src/flowsint_core/imports/type_matcher.py
deleted file mode 100644
index b5d5f4e3..00000000
--- a/flowsint-core/src/flowsint_core/imports/type_matcher.py
+++ /dev/null
@@ -1,291 +0,0 @@
-"""
-Entity type matching based on flowsint_types.
-Determines which entity type best matches the row data.
-"""
-
-from typing import Optional, Dict, Any, List
-import re
-import ipaddress
-
-# Import all entity types from flowsint_types
-try:
- from flowsint_types import (
- Email, Domain, Ip, ASN, Phone, Username, Website,
- Organization, Individual, SocialAccount, Device,
- Credential, Malware, SSLCertificate, Document, File
- )
- FLOWSINT_TYPES_AVAILABLE = True
-except ImportError:
- FLOWSINT_TYPES_AVAILABLE = False
-
-
-# Registry of entity type matchers
-# Each entry: (type_name, class_name, identifier_fields, matcher_function)
-TYPE_REGISTRY: List[tuple] = [
- ("Email", "Email", ["email", "mail", "e-mail", "address"], lambda d: _match_email(d)),
- ("Ip", "Ip", ["ip", "ip_address", "ipv4", "ipv6", "address"], lambda d: _match_ip(d)),
- ("Domain", "Domain", ["domain", "hostname", "host"], lambda d: _match_domain(d)),
- ("Website", "Website", ["url", "website", "web", "link"], lambda d: _match_website(d)),
- ("Phone", "Phone", ["phone", "telephone", "tel", "mobile", "cell"], lambda d: _match_phone(d)),
- ("ASN", "ASN", ["asn", "as_number"], lambda d: _match_asn(d)),
- ("Username", "Username", ["username", "user", "handle", "login"], lambda d: _match_username(d)),
- ("Organization", "Organization", ["organization", "org", "company", "name"], lambda d: _match_organization(d)),
- ("Individual", "Individual", ["name", "person", "individual", "fullname"], lambda d: _match_individual(d)),
- ("SocialAccount", "SocialAccount", ["social", "profile", "account"], lambda d: _match_social(d)),
- ("Credential", "Credential", ["credential", "password", "creds"], lambda d: _match_credential(d)),
-]
-
-
-def detect_entity_type_from_row(row_data: Dict[str, Any]) -> Optional[str]:
- """
- Detect the most likely entity type for a row of data.
-
- Args:
- row_data: Dictionary of column_name: value pairs
-
- Returns:
- Entity type name (e.g., "Email", "Domain", "Ip") or None
- """
- if not row_data:
- return None
-
- # Normalize keys to lowercase for comparison, skip None keys
- normalized_data = {
- k.lower().strip(): v
- for k, v in row_data.items()
- if v and k is not None
- }
-
- # Try to match each type
- scores = {}
- for type_name, class_name, identifier_fields, matcher_func in TYPE_REGISTRY:
- score = 0
-
- # Check if any identifier field matches
- for field in identifier_fields:
- if field in normalized_data:
- score += 10
-
- # Use matcher function to validate
- try:
- if matcher_func(normalized_data):
- score += 5
- except Exception:
- pass
-
- if score > 0:
- scores[type_name] = score
-
- # Return type with highest score
- if scores:
- return max(scores, key=scores.get)
-
- return None
-
-
-def get_primary_identifier(row_data: Dict[str, Any], entity_type: str) -> Optional[str]:
- """
- Get the primary identifier value for an entity based on its type.
-
- Args:
- row_data: Dictionary of column_name: value pairs
- entity_type: The detected entity type
-
- Returns:
- The primary identifier value (e.g., email address, domain name, IP)
- """
- normalized_data = {k.lower().strip(): v for k, v in row_data.items() if v}
-
- # Find the registry entry for this type
- for type_name, class_name, identifier_fields, _ in TYPE_REGISTRY:
- if type_name == entity_type:
- # Look for identifier fields in order of priority
- for field in identifier_fields:
- if field in normalized_data:
- return str(normalized_data[field]).strip()
-
- # Fallback: try common value fields
- for key in ["value", "name", "identifier", "id"]:
- if key in normalized_data:
- return str(normalized_data[key]).strip()
-
- # Last resort: return first non-empty value
- for value in row_data.values():
- if value and str(value).strip():
- return str(value).strip()
-
- return None
-
-
-# Matcher functions for each entity type
-
-def _match_email(data: Dict[str, Any]) -> bool:
- """Check if data looks like an email entity."""
- email_pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
-
- for key, value in data.items():
- if any(k in key for k in ["email", "mail"]):
- return bool(re.match(email_pattern, str(value)))
-
- # Check if any value looks like an email
- for value in data.values():
- if re.match(email_pattern, str(value)):
- return True
-
- return False
-
-
-def _match_ip(data: Dict[str, Any]) -> bool:
- """Check if data looks like an IP entity."""
- for key, value in data.items():
- if any(k in key for k in ["ip", "address"]) and "email" not in key:
- try:
- ipaddress.ip_address(str(value))
- return True
- except ValueError:
- pass
-
- # Check if any value looks like an IP
- for value in data.values():
- try:
- ipaddress.ip_address(str(value))
- return True
- except (ValueError, AttributeError):
- pass
-
- return False
-
-
-def _match_domain(data: Dict[str, Any]) -> bool:
- """Check if data looks like a domain entity."""
- domain_pattern = r'^(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}$'
-
- for key, value in data.items():
- if any(k in key for k in ["domain", "hostname", "host"]):
- return bool(re.match(domain_pattern, str(value)))
-
- # Check if any value looks like a domain (but not email)
- for value in data.values():
- val_str = str(value)
- if re.match(domain_pattern, val_str) and "@" not in val_str and not val_str.startswith("http"):
- return True
-
- return False
-
-
-def _match_website(data: Dict[str, Any]) -> bool:
- """Check if data looks like a website entity."""
- url_pattern = r'^https?://'
-
- for key, value in data.items():
- if any(k in key for k in ["url", "website", "web", "link"]):
- return bool(re.match(url_pattern, str(value), re.IGNORECASE))
-
- # Check if any value looks like a URL
- for value in data.values():
- if re.match(url_pattern, str(value), re.IGNORECASE):
- return True
-
- return False
-
-
-def _match_phone(data: Dict[str, Any]) -> bool:
- """Check if data looks like a phone entity."""
- for key, value in data.items():
- if any(k in key for k in ["phone", "tel", "mobile", "cell"]):
- # Remove common separators
- cleaned = re.sub(r'[\s\-\(\)\.]', '', str(value))
- phone_pattern = r'^\+?[0-9]{7,15}$'
- return bool(re.match(phone_pattern, cleaned))
-
- return False
-
-
-def _match_asn(data: Dict[str, Any]) -> bool:
- """Check if data looks like an ASN entity."""
- asn_pattern = r'^AS\d+$'
-
- for key, value in data.items():
- if "asn" in key or "as_number" in key or "as_name" in key:
- return bool(re.match(asn_pattern, str(value), re.IGNORECASE))
-
- # Check if any value looks like an ASN
- for value in data.values():
- if re.match(asn_pattern, str(value), re.IGNORECASE):
- return True
-
- return False
-
-
-def _match_username(data: Dict[str, Any]) -> bool:
- """Check if data looks like a username entity."""
- username_pattern = r'^@?[a-zA-Z0-9_]{3,30}$'
-
- for key, value in data.items():
- if any(k in key for k in ["username", "user", "handle"]):
- val_str = str(value)
- if re.match(username_pattern, val_str):
- return val_str.startswith('@') or not val_str.isdigit()
-
- # Fallback: check if any value looks like a username
- # (but not if it looks like other types)
- for key, value in data.items():
- val_str = str(value)
- if re.match(username_pattern, val_str):
- # Make sure it's not an email or domain
- if '@' not in val_str and '.' not in val_str:
- # If it starts with @ or is not purely numeric, likely a username
- if val_str.startswith('@') or not val_str.isdigit():
- return True
-
- return False
-
-
-def _match_organization(data: Dict[str, Any]) -> bool:
- """Check if data looks like an organization entity."""
- # Look for organization-specific fields
- org_fields = ["organization", "org", "company", "siren", "siret", "company_name"]
-
- for key in data.keys():
- if any(field in key for field in org_fields):
- return True
-
- # If has "name" but no individual-specific fields
- if "name" in data:
- individual_fields = ["firstname", "lastname", "first_name", "last_name", "email"]
- has_individual_fields = any(field in data for field in individual_fields)
- return not has_individual_fields
-
- return False
-
-
-def _match_individual(data: Dict[str, Any]) -> bool:
- """Check if data looks like an individual entity."""
- # Look for individual-specific fields
- individual_fields = ["firstname", "lastname", "first_name", "last_name", "fullname", "person"]
-
- for key in data.keys():
- if any(field in key for field in individual_fields):
- return True
-
- return False
-
-
-def _match_social(data: Dict[str, Any]) -> bool:
- """Check if data looks like a social profile entity."""
- social_fields = ["twitter", "linkedin", "facebook", "instagram", "github", "social"]
-
- for key in data.keys():
- if any(field in key for field in social_fields):
- return True
-
- return False
-
-
-def _match_credential(data: Dict[str, Any]) -> bool:
- """Check if data looks like a credential entity."""
- # Look for credential-specific fields
- if ("password" in data or "pass" in data) and ("username" in data or "user" in data or "email" in data):
- return True
-
- return False
diff --git a/flowsint-core/tests/import/test_entity_detection.py b/flowsint-core/tests/import/test_entity_detection.py
index 529d0134..e0951c06 100644
--- a/flowsint-core/tests/import/test_entity_detection.py
+++ b/flowsint-core/tests/import/test_entity_detection.py
@@ -1,299 +1,61 @@
-"""Unit tests for entity_detection module."""
-import pytest
-from flowsint_core.imports.entity_detection import (
- detect_entity_type,
- is_email,
- is_domain,
- is_ip_address,
- is_website,
- is_phone,
- is_asn,
- is_username,
- get_default_label,
-)
-
-
-class TestEmailDetection:
- """Tests for email detection."""
-
- def test_valid_emails(self):
- """Test detection of valid email addresses."""
- valid_emails = [
- ]
- for email in valid_emails:
- assert is_email(email), f"Failed to detect valid email: {email}"
- assert detect_entity_type(email) == "Email"
-
- def test_invalid_emails(self):
- """Test rejection of invalid email addresses."""
- invalid_emails = [
- "not-an-email",
- "@example.com",
- "user@",
- "user@domain",
- "",
- ]
- for email in invalid_emails:
- assert not is_email(email), f"Incorrectly detected as email: {email}"
-
-
-class TestDomainDetection:
- """Tests for domain detection."""
-
- def test_valid_domains(self):
- """Test detection of valid domains."""
- valid_domains = [
- "example.com",
- "sub.domain.com",
- "my-domain.co.uk",
- "test123.org",
- "a.b.c.d.example.com",
- ]
- for domain in valid_domains:
- assert is_domain(domain), f"Failed to detect valid domain: {domain}"
- assert detect_entity_type(domain) == "Domain"
-
- def test_invalid_domains(self):
- """Test rejection of invalid domains."""
- invalid_domains = [
- "not-a-domain",
- ".com",
- "domain.",
- "http://example.com", # This is a website, not a domain
- "",
- "192.168.1.1", # This is an IP
- ]
- for domain in invalid_domains:
- assert not is_domain(domain), f"Incorrectly detected as domain: {domain}"
-
-
-class TestIPAddressDetection:
- """Tests for IP address detection."""
-
- def test_valid_ipv4(self):
- """Test detection of valid IPv4 addresses."""
- valid_ips = [
- "192.168.1.1",
- "10.0.0.1",
- "172.16.0.1",
- "8.8.8.8",
- "255.255.255.255",
- ]
- for ip in valid_ips:
- assert is_ip_address(ip), f"Failed to detect valid IPv4: {ip}"
- assert detect_entity_type(ip) == "IP"
-
- def test_valid_ipv6(self):
- """Test detection of valid IPv6 addresses."""
- valid_ips = [
- "2001:0db8:85a3:0000:0000:8a2e:0370:7334",
- "2001:db8::1",
- "::1",
- "fe80::1",
- ]
- for ip in valid_ips:
- assert is_ip_address(ip), f"Failed to detect valid IPv6: {ip}"
- assert detect_entity_type(ip) == "IP"
-
- def test_invalid_ips(self):
- """Test rejection of invalid IP addresses."""
- invalid_ips = [
- "256.256.256.256",
- "192.168.1",
- "not-an-ip",
- "",
- ]
- for ip in invalid_ips:
- assert not is_ip_address(ip), f"Incorrectly detected as IP: {ip}"
-
-
-class TestWebsiteDetection:
- """Tests for website/URL detection."""
-
- def test_valid_websites(self):
- """Test detection of valid URLs."""
- valid_urls = [
- "http://example.com",
- "https://www.example.com",
- "http://sub.domain.com/path",
- "https://example.com:8080/path?query=value",
- ]
- for url in valid_urls:
- assert is_website(url), f"Failed to detect valid URL: {url}"
- assert detect_entity_type(url) == "Website"
-
- def test_invalid_websites(self):
- """Test rejection of invalid URLs."""
- invalid_urls = [
- "example.com",
- "ftp://example.com",
- "not-a-url",
- "",
- ]
- for url in invalid_urls:
- assert not is_website(url), f"Incorrectly detected as website: {url}"
-
-
-class TestPhoneDetection:
- """Tests for phone number detection."""
-
- def test_valid_phones(self):
- """Test detection of valid phone numbers."""
- valid_phones = [
- "+1234567890",
- "123-456-7890",
- "(123) 456-7890",
- "+44 20 7123 4567",
- "555.123.4567",
- "5551234567",
- ]
- for phone in valid_phones:
- assert is_phone(phone), f"Failed to detect valid phone: {phone}"
- assert detect_entity_type(phone) == "Phone"
-
- def test_invalid_phones(self):
- """Test rejection of invalid phone numbers."""
- invalid_phones = [
- "123", # Too short
- "abc-def-ghij",
- "",
- "1234567890123456", # Too long
- ]
- for phone in invalid_phones:
- assert not is_phone(phone), f"Incorrectly detected as phone: {phone}"
-
-
-class TestASNDetection:
- """Tests for ASN detection."""
-
- def test_valid_asns(self):
- """Test detection of valid ASNs."""
- valid_asns = [
- "AS13335",
- "as64512",
- "AS1",
- "AS4294967295",
- ]
- for asn in valid_asns:
- assert is_asn(asn), f"Failed to detect valid ASN: {asn}"
- assert detect_entity_type(asn) == "ASN"
-
- def test_invalid_asns(self):
- """Test rejection of invalid ASNs."""
- invalid_asns = [
- "13335",
- "ASN13335",
- "AS",
- "",
- ]
- for asn in invalid_asns:
- assert not is_asn(asn), f"Incorrectly detected as ASN: {asn}"
-
-
-class TestUsernameDetection:
- """Tests for username detection."""
-
- def test_valid_usernames(self):
- """Test detection of valid usernames."""
- valid_usernames = [
- "@john_doe",
- "@user123",
- "username",
- "user_name_123",
- ]
- for username in valid_usernames:
- assert is_username(username), f"Failed to detect valid username: {username}"
- assert detect_entity_type(username) == "Username"
-
- def test_invalid_usernames(self):
- """Test rejection of invalid usernames."""
- invalid_usernames = [
- "ab", # Too short
- "@",
- "123456789", # Purely numeric
- "[email protected]", # This is an email
- "",
- "a" * 31, # Too long
- ]
- for username in invalid_usernames:
- assert not is_username(username), f"Incorrectly detected as username: {username}"
-
-
-class TestDetectionPriority:
- """Tests for detection priority and edge cases."""
-
- def test_email_vs_username(self):
- """Email should take priority over username."""
- assert detect_entity_type("[email protected]") == "Email"
-
- def test_url_vs_domain(self):
- """URL should take priority over domain."""
- assert detect_entity_type("http://example.com") == "Website"
- assert detect_entity_type("example.com") == "Domain"
-
- def test_ip_vs_domain(self):
- """IP should be correctly distinguished from domain."""
- assert detect_entity_type("192.168.1.1") == "IP"
- assert detect_entity_type("example.com") == "Domain"
-
- def test_whitespace_handling(self):
- """Test that whitespace is properly handled."""
- assert detect_entity_type(" [email protected] ") == "Email"
- assert detect_entity_type(" example.com ") == "Domain"
-
- def test_none_and_empty(self):
- """Test handling of None and empty strings."""
- assert detect_entity_type("") is None
- assert detect_entity_type(" ") is None
- assert detect_entity_type(None) is None
-
-
-class TestGetDefaultLabel:
- """Tests for get_default_label function."""
-
- def test_email_label(self):
- """Test label generation for email."""
- assert get_default_label("Email", "[email protected]") == "[email protected]"
-
- def test_domain_label(self):
- """Test label generation for domain."""
- assert get_default_label("Domain", "example.com") == "example.com"
-
- def test_ip_label(self):
- """Test label generation for IP."""
- assert get_default_label("IP", "192.168.1.1") == "192.168.1.1"
-
- def test_username_label(self):
- """Test label generation for username (strips @ prefix)."""
- assert get_default_label("Username", "@john_doe") == "john_doe"
- assert get_default_label("Username", "john_doe") == "john_doe"
-
- def test_phone_label(self):
- """Test label generation for phone."""
- assert get_default_label("Phone", "+1234567890") == "+1234567890"
-
- def test_asn_label(self):
- """Test label generation for ASN."""
- assert get_default_label("ASN", "AS13335") == "AS13335"
-
- def test_website_label(self):
- """Test label generation for website."""
- assert get_default_label("Website", "https://example.com") == "https://example.com"
-
- def test_organization_label(self):
- """Test label generation for organization."""
- assert get_default_label("Organization", "ACME Corp") == "ACME Corp"
-
- def test_individual_label(self):
- """Test label generation for individual."""
- assert get_default_label("Individual", "John Doe") == "John Doe"
-
- def test_unknown_type(self):
- """Test label generation for unknown type."""
- assert get_default_label("UnknownType", "some value") == "some value"
+from flowsint_types import Domain, Ip, Phone
+from flowsint_core.imports import detect_type
+
+
+def test_detection_domains():
+ raw_domains = ["mydomain.com", "myseconddomain.com", "blog.subdomain.uk"]
+ results = []
+ for domain in raw_domains:
+ DetectedType = detect_type(domain)
+ if DetectedType:
+ assert DetectedType is Domain
+ py_obj = DetectedType.from_string(domain)
+ assert isinstance(py_obj, Domain)
+ results.append(py_obj)
+ # domain 1
+ assert results[0].domain == "mydomain.com"
+ assert results[0].root == True
+ # domain 2
+ assert results[1].domain == "myseconddomain.com"
+ assert results[1].root == True
+ # domain 3
+ assert results[2].domain == "blog.subdomain.uk"
+ assert results[2].root == False
+
+
+def test_detection_ips():
+ raw_ips = ["240.123.123.234", "12.43.23.12"]
+ results = []
+ for ip in raw_ips:
+ DetectedType = detect_type(ip)
+ if DetectedType:
+ assert DetectedType is Ip
+ py_obj = DetectedType.from_string(ip)
+ assert isinstance(py_obj, Ip)
+ results.append(py_obj)
+ # ip 1
+ assert results[0].address == "240.123.123.234"
+ # ip 2
+ assert results[1].address == "12.43.23.12"
+
+
+def test_detection_mixed():
+ raw_obj = ["240.123.123.234", "my.domain.io", "+33632233223"]
+ results = []
+ for obj in raw_obj:
+ DetectedType = detect_type(obj)
+ print(DetectedType)
+ if DetectedType:
+ py_obj = DetectedType.from_string(obj)
+ results.append(py_obj)
+
+ # ip
+ assert isinstance(results[0], Ip)
+ assert results[0].address == "240.123.123.234"
+ # domain
+ assert isinstance(results[1], Domain)
+ assert results[1].domain == "my.domain.io"
+ assert results[1].root == False
+ # phone
+ assert isinstance(results[2], Phone)
+ assert results[2].number == "+33632233223"
diff --git a/flowsint-core/tests/import/test_file_parser.py b/flowsint-core/tests/import/test_file_parser.py
new file mode 100644
index 00000000..6fe3559c
--- /dev/null
+++ b/flowsint-core/tests/import/test_file_parser.py
@@ -0,0 +1,42 @@
+from flowsint_types import Domain, Ip, Username
+from unittest.mock import patch
+
+from flowsint_core.imports import (
+ parse_import_file,
+ FileParseResult,
+ EntityPreview,
+)
+
+
+def test_parse_import_file_txt_basic_with_mocked_detect_type():
+
+ txt_bytes = b"domain.com\nblog.domain.com\n12.34.56.78\nmy_super_username"
+
+ result = parse_import_file(txt_bytes, "values.txt", max_preview_rows=100)
+
+ assert isinstance(result, FileParseResult)
+ assert result.total_entities == 4
+
+ first = result.entities["Domain"].results[0]
+ assert isinstance(first, EntityPreview)
+ assert isinstance(first.obj, Domain)
+ assert first.detected_type == "Domain"
+ assert first.obj.domain == "domain.com"
+
+ second = result.entities["Domain"].results[1]
+ assert isinstance(second, EntityPreview)
+ assert isinstance(second.obj, Domain)
+ assert second.detected_type == "Domain"
+ assert second.obj.domain == "blog.domain.com"
+
+ third = result.entities["Ip"].results[0]
+ assert isinstance(third, EntityPreview)
+ assert isinstance(third.obj, Ip)
+ assert third.detected_type == "Ip"
+ assert third.obj.address == "12.34.56.78"
+
+ fourth = result.entities["Username"].results[0]
+ assert isinstance(fourth, EntityPreview)
+ assert isinstance(fourth.obj, Username)
+ assert fourth.detected_type == "Username"
+ assert fourth.obj.value == "my_super_username"
diff --git a/flowsint-core/tests/import/test_import_integration.py b/flowsint-core/tests/import/test_import_integration.py
deleted file mode 100644
index 0612a6b6..00000000
--- a/flowsint-core/tests/import/test_import_integration.py
+++ /dev/null
@@ -1,294 +0,0 @@
-"""Integration tests for import functionality (entity-based)."""
-import pytest
-from flowsint_core.imports import parse_file
-
-
-class TestEntityBasedImportWorkflow:
- """Integration tests for entity-based import workflow."""
-
- def test_full_entity_workflow(self):
- """Test complete workflow: parse file -> detect entities -> prepare for import."""
- # Step 1: User uploads a CSV file (each row = one entity)
- csv_content = b"""email,name,department
[email protected],John Admin,IT
[email protected],Jane User,HR
[email protected],Bob Security,Security
-"""
-
- # Step 2: Parse and analyze the file
- result = parse_file(csv_content, "test.csv")
-
- # Verify parsing - entities instead of rows
- assert result.total_entities == 3
- assert len(result.entities) == 3
-
- # Verify entity detection
- assert all(e.detected_type == "Email" for e in result.entities)
-
- # Step 3: User reviews entities (can change type/label)
- entity_mappings = []
- for entity in result.entities:
- entity_mappings.append({
- "row_index": entity.row_index,
- "entity_type": entity.detected_type,
- "label": entity.primary_value,
- "include": True
- })
-
- # Verify mappings
- assert len(entity_mappings) == 3
- assert all(m["entity_type"] == "Email" for m in entity_mappings)
-
- # Step 4: Verify all data is preserved in entity
- first_entity = result.entities[0]
- assert "email" in first_entity.data
- assert "name" in first_entity.data
- assert "department" in first_entity.data
-
- def test_txt_entity_workflow(self):
- """Test workflow for TXT file where each line is an entity."""
- txt_content = b"""example.com
-192.168.1.1
-AS13335
-"""
-
- result = parse_file(txt_content, "entities.txt")
-
- assert result.total_entities == 4
- assert len(result.entities) == 4
-
- # Each line is detected as different type
- types = {e.detected_type for e in result.entities}
- assert "Domain" in types
- assert "Ip" in types
- assert "Email" in types
- assert "ASN" in types
-
- def test_user_corrections_workflow(self):
- """Test workflow where user corrects detected types."""
- csv_content = b"""identifier,category,notes
-john_doe,username,Social media handle
-example.com,domain,Main website
-192.168.1.1,server,Production server
-"""
-
- # Parse file
- result = parse_file(csv_content, "data.csv")
-
- # Simulate user reviewing and correcting types
- entity_mappings = []
- for entity in result.entities:
- # User can override detected type
- if entity.data.get("category") == "server":
- # User wants custom type instead of detected "Ip"
- entity_type = "Device"
- else:
- entity_type = entity.detected_type
-
- entity_mappings.append({
- "row_index": entity.row_index,
- "entity_type": entity_type,
- "label": entity.primary_value,
- "include": True
- })
-
- # Verify overrides
- assert entity_mappings[0]["entity_type"] == "Username"
- assert entity_mappings[1]["entity_type"] == "Domain"
- assert entity_mappings[2]["entity_type"] == "Device" # User override
-
- def test_label_customization(self):
- """Test customizing labels for entities."""
- csv_content = b"""email,full_name,role
[email protected],John Administrator,Admin
[email protected],Jane Security,SecOps
-"""
-
- result = parse_file(csv_content, "contacts.csv")
-
- # User customizes labels with additional context
- entity_mappings = []
- for entity in result.entities:
- # Combine email with full name for label
- custom_label = f"{entity.data['email']} ({entity.data['full_name']})"
- entity_mappings.append({
- "row_index": entity.row_index,
- "entity_type": entity.detected_type,
- "label": custom_label,
- "include": True
- })
-
- # Verify customized labels
- assert "John Administrator" in entity_mappings[0]["label"]
- assert "Jane Security" in entity_mappings[1]["label"]
-
- def test_selective_import(self):
- """Test excluding specific entities from import."""
- csv_content = b"""email,status
-"""
-
- result = parse_file(csv_content, "users.csv")
-
- # User only wants to import active users
- entity_mappings = []
- for entity in result.entities:
- include = entity.data.get("status") == "active"
- entity_mappings.append({
- "row_index": entity.row_index,
- "entity_type": entity.detected_type,
- "label": entity.primary_value,
- "include": include
- })
-
- # Verify selective inclusion
- included = [m for m in entity_mappings if m["include"]]
- assert len(included) == 2
-
-
-class TestRealWorldEntityScenarios:
- """Real-world scenario tests with entity-based approach."""
-
- def test_organization_import(self):
- """Test importing organizations with metadata."""
- csv_content = b"""name,industry,country,employees
-Google,Technology,USA,150000
-Microsoft,Software,USA,220000
-Apple,Technology,USA,164000
-"""
-
- result = parse_file(csv_content, "orgs.csv")
-
- assert result.total_entities == 3
- assert all(e.detected_type == "Organization" for e in result.entities)
-
- # Verify all metadata preserved
- for entity in result.entities:
- assert "name" in entity.data
- assert "industry" in entity.data
- assert "country" in entity.data
- assert "employees" in entity.data
-
- def test_threat_intel_import(self):
- """Test importing threat intelligence data."""
- csv_content = b"""domain,severity,first_seen,description,malware_family
-malicious.com,high,2024-01-15,Phishing domain,TrickBot
-evil.net,critical,2024-01-16,C2 server,Cobalt Strike
-threat.xyz,high,2024-01-18,APT infrastructure,Unknown
-"""
-
- result = parse_file(csv_content, "iocs.csv")
-
- assert result.total_entities == 3
- assert all(e.detected_type == "Domain" for e in result.entities)
-
- # Verify threat metadata preserved in entity data
- first_entity = result.entities[0]
- assert first_entity.data["severity"] == "high"
- assert first_entity.data["malware_family"] == "TrickBot"
- assert "description" in first_entity.data
-
- def test_infrastructure_mapping(self):
- """Test importing infrastructure with multiple related fields."""
- csv_content = b"""domain,ip,asn,country,provider
-cloudflare.com,1.1.1.1,AS13335,USA,Cloudflare
-google.com,8.8.8.8,AS15169,USA,Google
-"""
-
- result = parse_file(csv_content, "infrastructure.csv")
-
- assert result.total_entities == 2
-
- # All related data preserved in entity
- for entity in result.entities:
- assert "domain" in entity.data
- assert "ip" in entity.data
- assert "asn" in entity.data
- assert "country" in entity.data
- assert "provider" in entity.data
-
-
-class TestTypeDistribution:
- """Tests for type distribution in entity-based approach."""
-
- def test_mixed_entity_types(self):
- """Test file with multiple entity types."""
- txt_content = b"""example.com
-test.org
-192.168.1.1
-10.0.0.1
-"""
-
- result = parse_file(txt_content, "mixed.txt")
-
- # Check type distribution
- assert result.type_distribution["Domain"] == 2
- assert result.type_distribution["Email"] == 2
- assert result.type_distribution["Ip"] == 2
-
- def test_predominant_type_detection(self):
- """Test that predominant type is correctly identified."""
- csv_content = b"""email,notes
[email protected],User account
[email protected],Admin account
[email protected],Guest account
[email protected],Service account
-"""
-
- result = parse_file(csv_content, "emails.csv")
-
- # Should have high confidence for Email type
- assert all(e.detected_type == "Email" for e in result.entities)
- assert all(e.confidence == "high" for e in result.entities)
-
-
-class TestEdgeCases:
- """Edge case tests for entity-based import."""
-
- def test_single_column_csv(self):
- """Test CSV with single column."""
- csv_content = b"""email
-"""
-
- result = parse_file(csv_content, "single_column.csv")
-
- assert result.total_entities == 2
- assert all(e.detected_type == "Email" for e in result.entities)
-
- def test_many_columns_csv(self):
- """Test CSV with many columns (all become entity data)."""
- csv_content = b"""email,name,dept,role,location,phone,manager,start_date
[email protected],John,IT,Dev,NYC,555-1234,[email protected],2024-01-01
-"""
-
- result = parse_file(csv_content, "detailed.csv")
-
- assert result.total_entities == 1
- entity = result.entities[0]
-
- # All 8 columns should be in entity data
- assert len(entity.data) == 8
- assert "email" in entity.data
- assert "manager" in entity.data
- assert "start_date" in entity.data
-
- def test_unknown_type_handling(self):
- """Test handling of unknown/undetectable types."""
- txt_content = b"""random text string
-another random value
-unstructured data
-"""
-
- result = parse_file(txt_content, "unknown.txt")
-
- # Should still parse but mark as Unknown
- assert result.total_entities == 3
- assert all(e.detected_type == "Unknown" for e in result.entities)
- assert all(e.confidence == "low" for e in result.entities)