Skip to content

A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.

feat(types): auto_discovery of types

dextmorgnc3c328f2997c
diff --git a/flowsint-types/src/flowsint_types/__init__.py b/flowsint-types/src/flowsint_types/__init__.py
index 30e6600e..3853aad4 100644
--- a/flowsint-types/src/flowsint_types/__init__.py
+++ b/flowsint-types/src/flowsint_types/__init__.py
@@ -3,8 +3,15 @@ Flowsint Types - Pydantic models for flowsint
"""
# Import registry first to ensure it's ready for auto-registration
-from .registry import TYPE_REGISTRY, flowsint_type, get_type
+from .registry import TYPE_REGISTRY, flowsint_type, get_type, load_all_types
+# Import base class
+from .flowsint_base import FlowsintType
+
+# Auto-discover and register all types
+load_all_types()
+
+# For backward compatibility, explicitly import commonly used types
from .address import Location
from .affiliation import Affiliation
from .alias import Alias
@@ -43,9 +50,6 @@ from .web_tracker import WebTracker
from .website import Website
from .whois import Whois
-# Import base
-from .flowsint_base import FlowsintType
-
from typing import Dict, Type, Any, Optional
from pydantic import BaseModel
@@ -151,7 +155,6 @@ TYPE_TO_MODEL: Dict[str, Type[BaseModel]] = {
"webtracker": WebTracker,
"weapon": Weapon,
"whois": Whois,
- "FlowsintType": FlowsintType,
}
@@ -196,27 +199,21 @@ def clean_neo4j_node_data(node_data: Dict[str, Any]) -> Dict[str, Any]:
# Skip Neo4j-specific fields (including 'type' which is the node type in Neo4j)
if k in ["sketch_id", "created_at", "type", "x", "y", "caption", "color"]:
continue
-
# Skip empty values (empty strings, None, empty lists, etc.)
if v in ("", None, [], {}):
continue
-
cleaned[k] = v
-
return cleaned
def parse_node_to_pydantic(node_data: Dict[str, Any]) -> Optional[BaseModel]:
"""
Parse a Neo4j node's properties into a Pydantic model instance.
-
Args:
node_data: Dictionary containing node properties from Neo4j.
Must include a 'type' field indicating the node type.
-
Returns:
An instance of the appropriate Pydantic model, or None if parsing fails
-
Example:
>>> node_data = {
... 'type': 'domain',
@@ -229,13 +226,10 @@ def parse_node_to_pydantic(node_data: Dict[str, Any]) -> Optional[BaseModel]:
"""
if not node_data or "type" not in node_data:
return None
-
node_type = node_data.get("type")
model_class = get_model_for_type(node_type)
-
if not model_class:
return None
-
try:
# Clean the node data first
cleaned_data = clean_neo4j_node_data(node_data)
diff --git a/flowsint-types/src/flowsint_types/flowsint_base.py b/flowsint-types/src/flowsint_types/flowsint_base.py
index 00e39244..e56ee73c 100644
--- a/flowsint-types/src/flowsint_types/flowsint_base.py
+++ b/flowsint-types/src/flowsint_types/flowsint_base.py
@@ -1,36 +1,20 @@
from pydantic import BaseModel, Field
-from pydantic._internal._model_construction import ModelMetaclass
-from typing import Optional, Type
+from typing import Optional
-class FlowsintTypeMetaclass(ModelMetaclass):
- """
- Metaclass for FlowsintType that automatically registers types in the TYPE_REGISTRY.
-
- This metaclass ensures that any class that inherits from FlowsintType
- is automatically registered in the global type registry.
- """
-
- def __new__(mcs, name: str, bases: tuple, namespace: dict, **kwargs):
- # Create the class using Pydantic's metaclass
- cls = super().__new__(mcs, name, bases, namespace, **kwargs)
-
- # Only register if this is a subclass of FlowsintType (not FlowsintType itself)
- # and if it's not an abstract class
- if name != 'FlowsintType' and bases:
- # Import here to avoid circular dependency
- from .registry import TYPE_REGISTRY
- TYPE_REGISTRY.register(cls)
-
- return cls
-
-
-class FlowsintType(BaseModel, metaclass=FlowsintTypeMetaclass):
+class FlowsintType(BaseModel):
"""Base class for all Flowsint entity types with label support.
Label is optional but computed at definition time.
- All classes that inherit from FlowsintType are automatically registered
- in the global TYPE_REGISTRY and can be accessed by their class name.
+ All classes that inherit from FlowsintType must be decorated with @flowsint_type
+ to be registered in the global TYPE_REGISTRY and accessed by their class name.
+
+ Usage:
+ from flowsint_types.registry import flowsint_type
+
+ @flowsint_type
+ class Domain(FlowsintType):
+ domain: str
"""
label: Optional[str] = Field(
None, description="UI-readable label for this entity, the one used on the graph.", title="Label"
diff --git a/flowsint-types/src/flowsint_types/registry.py b/flowsint-types/src/flowsint_types/registry.py
index dfc6da76..30d47284 100644
--- a/flowsint-types/src/flowsint_types/registry.py
+++ b/flowsint-types/src/flowsint_types/registry.py
@@ -2,10 +2,16 @@
This module provides an automatic type registration system using decorators.
All FlowsintType subclasses can be decorated with @flowsint_type to automatically
register themselves in the global TYPE_REGISTRY.
+
+Auto-discovery is performed by calling load_all_types() which imports all modules
+in the flowsint_types package, triggering the @flowsint_type decorators.
"""
from typing import Dict, Type, TypeVar, Optional
from pydantic import BaseModel
+import importlib
+import pkgutil
+import sys
T = TypeVar("T", bound=BaseModel)
@@ -135,3 +141,56 @@ def get_type(type_name: str, case_sensitive: bool = False) -> Optional[Type[Base
return TYPE_REGISTRY.get(type_name)
else:
return TYPE_REGISTRY.get_lowercase(type_name)
+
+
+# Auto-discovery cache
+_types_loaded = False
+
+
+def load_all_types() -> None:
+ """
+ Automatically discover and import all type modules in the flowsint_types package.
+
+ This function uses importlib to dynamically import all Python modules in the
+ flowsint_types package, which triggers the @flowsint_type decorators and
+ registers all types in TYPE_REGISTRY.
+
+ Features:
+ - Only imports modules once (cached via _types_loaded flag)
+ - Ignores private modules (starting with _)
+ - Only imports .py files
+ - Uses pkgutil for high performance iteration
+
+ This function is idempotent - calling it multiple times is safe and efficient.
+ """
+ global _types_loaded
+
+ # Early return if already loaded
+ if _types_loaded:
+ return
+
+ # Get the flowsint_types package
+ import flowsint_types
+ package = flowsint_types
+ package_path = package.__path__
+ package_name = package.__name__
+
+ # Iterate over all modules in the package
+ for importer, modname, ispkg in pkgutil.iter_modules(package_path, prefix=f"{package_name}."):
+ # Skip private modules
+ if modname.split('.')[-1].startswith('_'):
+ continue
+
+ # Skip if already imported
+ if modname in sys.modules:
+ continue
+
+ # Import the module to trigger @flowsint_type decorators
+ try:
+ importlib.import_module(modname)
+ except Exception as e:
+ # Log but don't fail - some modules might have optional dependencies
+ print(f"Warning: Failed to import {modname}: {e}", file=sys.stderr)
+
+ # Mark as loaded
+ _types_loaded = True
diff --git a/flowsint-types/tests/test_parser.py b/flowsint-types/tests/test_parser.py
index 29f3d68d..e98df467 100644
--- a/flowsint-types/tests/test_parser.py
+++ b/flowsint-types/tests/test_parser.py
@@ -10,7 +10,6 @@ These tests ensure that parse_node_to_pydantic correctly handles:
"""
import pytest
-from pydantic import BaseModel
from flowsint_types import (
parse_node_to_pydantic,
clean_neo4j_node_data,
@@ -21,14 +20,6 @@ from flowsint_types import (
Email,
Phone,
Organization,
- Username,
- Website,
- ASN,
- CIDR,
- SocialAccount,
- Individual,
- Phrase,
- Port,
)
@@ -38,115 +29,115 @@ class TestCleanNeo4jNodeData:
def test_clean_removes_neo4j_fields(self):
"""Test that Neo4j-specific fields are removed."""
node_data = {
- 'type': 'domain', # Neo4j type field, should be removed
- 'domain': 'example.com',
- 'label': 'example.com',
- 'sketch_id': 'should-be-removed',
- 'created_at': 'should-be-removed',
- 'x': 100,
- 'y': 200,
- 'caption': 'should-be-removed',
- 'color': 'should-be-removed',
+ "type": "domain",
+ "domain": "example.com",
+ "label": "example.com",
+ "sketch_id": "should-be-removed",
+ "created_at": "should-be-removed",
+ "x": 100,
+ "y": 200,
+ "caption": "should-be-removed",
+ "color": "should-be-removed",
}
result = clean_neo4j_node_data(node_data)
- assert 'type' not in result # Neo4j 'type' field should be removed
- assert 'domain' in result
- assert 'label' in result
- assert 'sketch_id' not in result
- assert 'created_at' not in result
- assert 'x' not in result
- assert 'y' not in result
- assert 'caption' not in result
- assert 'color' not in result
+ assert "type" not in result # Neo4j 'type' field should be removed
+ assert "domain" in result
+ assert "label" in result
+ assert "sketch_id" not in result
+ assert "created_at" not in result
+ assert "x" not in result
+ assert "y" not in result
+ assert "caption" not in result
+ assert "color" not in result
def test_clean_removes_empty_strings(self):
"""Test that empty strings are removed."""
node_data = {
- 'type': 'ip',
- 'address': '192.168.1.1',
- 'latitude': '',
- 'longitude': '',
- 'country': '',
+ "type": "ip",
+ "address": "192.168.1.1",
+ "latitude": "",
+ "longitude": "",
+ "country": "",
}
result = clean_neo4j_node_data(node_data)
- assert result == {'address': '192.168.1.1'}
+ assert result == {"address": "192.168.1.1"}
def test_clean_removes_none_values(self):
"""Test that None values are removed."""
node_data = {
- 'type': 'domain',
- 'domain': 'example.com',
- 'some_field': None,
+ "type": "domain",
+ "domain": "example.com",
+ "some_field": None,
}
result = clean_neo4j_node_data(node_data)
- assert result == {'domain': 'example.com'}
+ assert result == {"domain": "example.com"}
def test_clean_removes_empty_lists(self):
"""Test that empty lists are removed."""
node_data = {
- 'type': 'domain',
- 'domain': 'example.com',
- 'tags': [],
+ "type": "domain",
+ "domain": "example.com",
+ "tags": [],
}
result = clean_neo4j_node_data(node_data)
- assert result == {'domain': 'example.com'}
+ assert result == {"domain": "example.com"}
def test_clean_removes_empty_dicts(self):
"""Test that empty dicts are removed."""
node_data = {
- 'type': 'domain',
- 'domain': 'example.com',
- 'metadata': {},
+ "type": "domain",
+ "domain": "example.com",
+ "metadata": {},
}
result = clean_neo4j_node_data(node_data)
- assert result == {'domain': 'example.com'}
+ assert result == {"domain": "example.com"}
def test_clean_preserves_valid_data(self):
"""Test that valid data is preserved (except Neo4j 'type' field)."""
node_data = {
- 'type': 'ip',
- 'address': '8.8.8.8',
- 'label': '8.8.8.8',
- 'latitude': 37.386,
- 'longitude': -122.0838,
- 'country': 'US',
+ "type": "ip",
+ "address": "8.8.8.8",
+ "label": "8.8.8.8",
+ "latitude": 37.386,
+ "longitude": -122.0838,
+ "country": "US",
}
result = clean_neo4j_node_data(node_data)
# 'type' is filtered, rest is preserved
expected = {
- 'address': '8.8.8.8',
- 'label': '8.8.8.8',
- 'latitude': 37.386,
- 'longitude': -122.0838,
- 'country': 'US',
+ "address": "8.8.8.8",
+ "label": "8.8.8.8",
+ "latitude": 37.386,
+ "longitude": -122.0838,
+ "country": "US",
}
assert result == expected
def test_clean_preserves_zero_values(self):
"""Test that zero values are preserved (not treated as empty)."""
node_data = {
- 'type': 'ip',
- 'address': '127.0.0.1',
- 'latitude': 0,
- 'longitude': 0,
+ "type": "ip",
+ "address": "127.0.0.1",
+ "latitude": 0,
+ "longitude": 0,
}
result = clean_neo4j_node_data(node_data)
- assert result['latitude'] == 0
- assert result['longitude'] == 0
+ assert result["latitude"] == 0
+ assert result["longitude"] == 0
def test_clean_empty_dict(self):
"""Test cleaning an empty dict."""
@@ -160,44 +151,44 @@ class TestParseNodeToPydantic:
def test_parse_domain_valid(self):
"""Test parsing a valid domain node from Neo4j."""
node_data = {
- 'type': 'domain',
- 'domain': 'example.com',
- 'label': 'example.com',
- 'root': True,
- 'sketch_id': 'abc-123',
- 'created_at': '2024-01-01T00:00:00Z',
- 'x': 100,
- 'y': 200,
+ "type": "domain",
+ "domain": "example.com",
+ "label": "example.com",
+ "root": True,
+ "sketch_id": "abc-123",
+ "created_at": "2024-01-01T00:00:00Z",
+ "x": 100,
+ "y": 200,
}
result = parse_node_to_pydantic(node_data)
assert result is not None
assert isinstance(result, Domain)
- assert result.domain == 'example.com'
- assert result.label == 'example.com'
+ assert result.domain == "example.com"
+ assert result.label == "example.com"
assert result.root == True
def test_parse_ip_with_empty_optional_fields(self):
"""Test parsing IP node with empty strings for optional fields."""
node_data = {
- 'type': 'ip',
- 'address': '192.168.1.1',
- 'label': '192.168.1.1',
- 'latitude': '', # Empty string should be filtered
- 'longitude': '', # Empty string should be filtered
- 'country': '',
- 'city': '',
- 'isp': '',
- 'sketch_id': 'abc-123',
- 'created_at': '2024-01-01T00:00:00Z',
+ "type": "ip",
+ "address": "192.168.1.1",
+ "label": "192.168.1.1",
+ "latitude": "", # Empty string should be filtered
+ "longitude": "", # Empty string should be filtered
+ "country": "",
+ "city": "",
+ "isp": "",
+ "sketch_id": "abc-123",
+ "created_at": "2024-01-01T00:00:00Z",
}
result = parse_node_to_pydantic(node_data)
assert result is not None
assert isinstance(result, Ip)
- assert result.address == '192.168.1.1'
+ assert result.address == "192.168.1.1"
assert result.latitude is None
assert result.longitude is None
assert result.country is None
@@ -207,83 +198,83 @@ class TestParseNodeToPydantic:
def test_parse_ip_with_valid_optional_fields(self):
"""Test parsing IP node with valid optional fields."""
node_data = {
- 'type': 'ip',
- 'address': '8.8.8.8',
- 'label': '8.8.8.8',
- 'latitude': 37.386,
- 'longitude': -122.0838,
- 'country': 'US',
- 'city': 'Mountain View',
- 'isp': 'Google LLC',
+ "type": "ip",
+ "address": "8.8.8.8",
+ "label": "8.8.8.8",
+ "latitude": 37.386,
+ "longitude": -122.0838,
+ "country": "US",
+ "city": "Mountain View",
+ "isp": "Google LLC",
}
result = parse_node_to_pydantic(node_data)
assert result is not None
assert isinstance(result, Ip)
- assert result.address == '8.8.8.8'
+ assert result.address == "8.8.8.8"
assert result.latitude == 37.386
assert result.longitude == -122.0838
- assert result.country == 'US'
- assert result.city == 'Mountain View'
- assert result.isp == 'Google LLC'
+ assert result.country == "US"
+ assert result.city == "Mountain View"
+ assert result.isp == "Google LLC"
def test_parse_email_valid(self):
"""Test parsing a valid email node."""
node_data = {
- 'type': 'email',
- 'email': '[email protected]',
- 'label': '[email protected]',
+ "type": "email",
+ "email": "[email protected]",
+ "label": "[email protected]",
}
result = parse_node_to_pydantic(node_data)
assert result is not None
assert isinstance(result, Email)
- assert result.email == '[email protected]'
+ assert result.email == "[email protected]"
def test_parse_phone_valid(self):
"""Test parsing a valid phone node."""
node_data = {
- 'type': 'phone',
- 'number': '+33612345678',
- 'label': '+33612345678',
- 'country': 'FR',
+ "type": "phone",
+ "number": "+33612345678",
+ "label": "+33612345678",
+ "country": "FR",
}
result = parse_node_to_pydantic(node_data)
assert result is not None
assert isinstance(result, Phone)
- assert result.number == '+33612345678'
- assert result.country == 'FR'
+ assert result.number == "+33612345678"
+ assert result.country == "FR"
def test_parse_organization_valid(self):
"""Test parsing a valid organization node."""
node_data = {
- 'type': 'organization',
- 'name': 'ACME Corp',
- 'label': 'ACME Corp',
+ "type": "organization",
+ "name": "ACME Corp",
+ "label": "ACME Corp",
}
result = parse_node_to_pydantic(node_data)
assert result is not None
assert isinstance(result, Organization)
- assert result.name == 'ACME Corp'
+ assert result.name == "ACME Corp"
def test_parse_node_filters_neo4j_fields(self):
"""Test that Neo4j-specific fields are filtered out."""
node_data = {
- 'type': 'domain',
- 'domain': 'test.com',
- 'label': 'test.com',
- 'sketch_id': 'should-be-filtered',
- 'created_at': 'should-be-filtered',
- 'x': 100,
- 'y': 200,
- 'caption': 'should-be-filtered',
- 'color': 'should-be-filtered',
+ "type": "domain",
+ "domain": "test.com",
+ "label": "test.com",
+ "sketch_id": "should-be-filtered",
+ "created_at": "should-be-filtered",
+ "x": 100,
+ "y": 200,
+ "caption": "should-be-filtered",
+ "color": "should-be-filtered",
}
result = parse_node_to_pydantic(node_data)
@@ -291,13 +282,13 @@ class TestParseNodeToPydantic:
assert result is not None
assert isinstance(result, Domain)
# These fields should not cause errors even though they're not in the Pydantic model
- assert result.domain == 'test.com'
+ assert result.domain == "test.com"
def test_parse_node_missing_type(self):
"""Test that nodes without 'type' field return None."""
node_data = {
- 'domain': 'example.com',
- 'label': 'example.com',
+ "domain": "example.com",
+ "label": "example.com",
}
result = parse_node_to_pydantic(node_data)
@@ -307,8 +298,8 @@ class TestParseNodeToPydantic:
def test_parse_node_unknown_type(self):
"""Test that nodes with unknown type return None."""
node_data = {
- 'type': 'unknown_type_xyz',
- 'some_field': 'some_value',
+ "type": "unknown_type_xyz",
+ "some_field": "some_value",
}
result = parse_node_to_pydantic(node_data)
@@ -318,9 +309,9 @@ class TestParseNodeToPydantic:
def test_parse_node_invalid_email(self):
"""Test that invalid data fails validation and returns None."""
node_data = {
- 'type': 'email',
- 'email': 'not-an-email', # Invalid email
- 'label': 'not-an-email',
+ "type": "email",
+ "email": "not-an-email", # Invalid email
+ "label": "not-an-email",
}
result = parse_node_to_pydantic(node_data)
@@ -330,9 +321,9 @@ class TestParseNodeToPydantic:
def test_parse_node_invalid_ip(self):
"""Test that invalid IP address fails validation and returns None."""
node_data = {
- 'type': 'ip',
- 'address': '999.999.999.999', # Invalid IP
- 'label': '999.999.999.999',
+ "type": "ip",
+ "address": "999.999.999.999", # Invalid IP
+ "label": "999.999.999.999",
}
result = parse_node_to_pydantic(node_data)
@@ -342,9 +333,9 @@ class TestParseNodeToPydantic:
def test_parse_node_missing_required_field(self):
"""Test that missing required fields cause validation to fail."""
node_data = {
- 'type': 'email',
+ "type": "email",
# Missing 'email' field which is required
- 'label': 'test',
+ "label": "test",
}
result = parse_node_to_pydantic(node_data)
@@ -354,9 +345,9 @@ class TestParseNodeToPydantic:
def test_parse_node_empty_string_for_required_field(self):
"""Test that empty string for required field is filtered and causes validation to fail."""
node_data = {
- 'type': 'domain',
- 'domain': '', # Empty string should be filtered, causing validation to fail
- 'label': 'test',
+ "type": "domain",
+ "domain": "", # Empty string should be filtered, causing validation to fail
+ "label": "test",
}
result = parse_node_to_pydantic(node_data)
@@ -366,11 +357,11 @@ class TestParseNodeToPydantic:
def test_parse_node_filters_none_values(self):
"""Test that None values are filtered out."""
node_data = {
- 'type': 'ip',
- 'address': '1.2.3.4',
- 'label': '1.2.3.4',
- 'latitude': None,
- 'longitude': None,
+ "type": "ip",
+ "address": "1.2.3.4",
+ "label": "1.2.3.4",
+ "latitude": None,
+ "longitude": None,
}
result = parse_node_to_pydantic(node_data)
@@ -383,10 +374,10 @@ class TestParseNodeToPydantic:
def test_parse_node_filters_empty_lists(self):
"""Test that empty lists are filtered out."""
node_data = {
- 'type': 'domain',
- 'domain': 'example.com',
- 'label': 'example.com',
- 'some_list_field': [], # Should be filtered
+ "type": "domain",
+ "domain": "example.com",
+ "label": "example.com",
+ "some_list_field": [], # Should be filtered
}
result = parse_node_to_pydantic(node_data)
@@ -397,10 +388,10 @@ class TestParseNodeToPydantic:
def test_parse_node_filters_empty_dicts(self):
"""Test that empty dicts are filtered out."""
node_data = {
- 'type': 'domain',
- 'domain': 'example.com',
- 'label': 'example.com',
- 'some_dict_field': {}, # Should be filtered
+ "type": "domain",
+ "domain": "example.com",
+ "label": "example.com",
+ "some_dict_field": {}, # Should be filtered
}
result = parse_node_to_pydantic(node_data)
@@ -425,11 +416,11 @@ class TestEdgeCases:
def test_ip_with_string_zero_coordinates(self):
"""Test IP with '0' string for coordinates (edge case from real data)."""
node_data = {
- 'type': 'ip',
- 'address': '127.0.0.1',
- 'label': '127.0.0.1',
- 'latitude': '0',
- 'longitude': '0',
+ "type": "ip",
+ "address": "127.0.0.1",
+ "label": "127.0.0.1",
+ "latitude": "0",
+ "longitude": "0",
}
result = parse_node_to_pydantic(node_data)
@@ -443,34 +434,34 @@ class TestEdgeCases:
def test_domain_with_subdomain(self):
"""Test domain parsing with subdomain."""
node_data = {
- 'type': 'domain',
- 'domain': 'sub.example.com',
- 'label': 'sub.example.com',
+ "type": "domain",
+ "domain": "sub.example.com",
+ "label": "sub.example.com",
}
result = parse_node_to_pydantic(node_data)
assert result is not None
assert isinstance(result, Domain)
- assert result.domain == 'sub.example.com'
+ assert result.domain == "sub.example.com"
assert result.root == False # Should be computed as not root
def test_multiple_neo4j_specific_fields(self):
"""Test that all Neo4j-specific fields are properly filtered."""
neo4j_fields = {
- 'sketch_id': 'test',
- 'created_at': '2024-01-01',
- 'type': 'domain',
- 'caption': 'test caption',
- 'x': 100.5,
- 'y': 200.5,
- 'color': '#FF0000',
+ "sketch_id": "test",
+ "created_at": "2024-01-01",
+ "type": "domain",
+ "caption": "test caption",
+ "x": 100.5,
+ "y": 200.5,
+ "color": "#FF0000",
}
node_data = {
**neo4j_fields,
- 'domain': 'example.com',
- 'label': 'example.com',
+ "domain": "example.com",
+ "label": "example.com",
}
result = parse_node_to_pydantic(node_data)
@@ -478,8 +469,8 @@ class TestEdgeCases:
assert result is not None
assert isinstance(result, Domain)
# Verify only valid fields were used
- assert result.domain == 'example.com'
- assert result.label == 'example.com'
+ assert result.domain == "example.com"
+ assert result.label == "example.com"
class TestAllTypes:
@@ -488,78 +479,95 @@ class TestAllTypes:
# Mapping of type names to valid minimal data for testing
# This ensures every type in TYPE_TO_MODEL can be parsed
VALID_TEST_DATA = {
- 'domain': {'domain': 'example.com'},
- 'email': {'email': '[email protected]'},
- 'ip': {'address': '192.168.1.1'},
- 'phone': {'number': '+33612345678'},
- 'username': {'value': 'john_doe'},
- 'organization': {'name': 'ACME Corp'},
- 'individual': {'first_name': 'John', 'last_name': 'Doe'},
- 'socialaccount': {'username': {'value': 'johndoe'}},
- 'asn': {'asn_str': 'AS15169'},
- 'cidr': {'network': '192.168.1.0/24'},
- 'cryptowallet': {'address': '1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa'},
- 'cryptowallettransaction': {'source': {'address': '1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa'}},
- 'cryptonft': {'wallet': {'address': '1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa'}, 'contract_address': '0x' + 'a' * 40, 'token_id': '123'},
- 'website': {'url': 'https://example.com'},
- 'port': {'number': 443},
- 'phrase': {'text': 'test phrase'},
- 'breach': {'name': 'Test Breach'},
- 'credential': {'username': 'user'},
- 'device': {'device_id': 'device-123'},
- 'document': {'title': 'Test Document'},
- 'file': {'filename': 'test.txt'},
- 'malware': {'name': 'TestMalware'},
- 'sslcertificate': {'subject': 'CN=example.com'},
- 'location': {'address': '123 Main St', 'city': 'Paris', 'country': 'France', 'zip': '75001'},
- 'affiliation': {'organization': 'test org'},
- 'alias': {'alias': 'test_alias'},
- 'bankaccount': {'account_number': '123456789'},
- 'creditcard': {'card_number': '4111111111111111'},
- 'dnsrecord': {'name': 'example.com', 'record_type': 'A', 'value': '192.168.1.1'},
- 'gravatar': {'src': 'https://gravatar.com/avatar/abc', 'hash': 'abc123'},
- 'leak': {'name': 'test leak'},
- 'message': {'message_id': 'msg-123', 'content': 'test message'},
- 'reputationscore': {'entity_id': 'entity-123'},
- 'riskprofile': {'entity_id': 'entity-123'},
- 'script': {'script_id': 'script-123'},
- 'session': {'session_id': 'session-123'},
- 'webtracker': {'tracker_id': 'tracker-123'},
- 'weapon': {'name': 'Test Weapon'},
- 'whois': {'domain': {'domain': 'example.com'}},
+ "domain": {"domain": "example.com"},
+ "email": {"email": "[email protected]"},
+ "ip": {"address": "192.168.1.1"},
+ "phone": {"number": "+33612345678"},
+ "username": {"value": "john_doe"},
+ "organization": {"name": "ACME Corp"},
+ "individual": {"first_name": "John", "last_name": "Doe"},
+ "socialaccount": {"username": {"value": "johndoe"}},
+ "asn": {"asn_str": "AS15169"},
+ "cidr": {"network": "192.168.1.0/24"},
+ "cryptowallet": {"address": "1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa"},
+ "cryptowallettransaction": {
+ "source": {"address": "1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa"}
+ },
+ "cryptonft": {
+ "wallet": {"address": "1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa"},
+ "contract_address": "0x" + "a" * 40,
+ "token_id": "123",
+ },
+ "website": {"url": "https://example.com"},
+ "port": {"number": 443},
+ "phrase": {"text": "test phrase"},
+ "breach": {"name": "Test Breach"},
+ "credential": {"username": "user"},
+ "device": {"device_id": "device-123"},
+ "document": {"title": "Test Document"},
+ "file": {"filename": "test.txt"},
+ "malware": {"name": "TestMalware"},
+ "sslcertificate": {"subject": "CN=example.com"},
+ "location": {
+ "address": "123 Main St",
+ "city": "Paris",
+ "country": "France",
+ "zip": "75001",
+ },
+ "affiliation": {"organization": "test org"},
+ "alias": {"alias": "test_alias"},
+ "bankaccount": {"account_number": "123456789"},
+ "creditcard": {"card_number": "4111111111111111"},
+ "dnsrecord": {
+ "name": "example.com",
+ "record_type": "A",
+ "value": "192.168.1.1",
+ },
+ "gravatar": {"src": "https://gravatar.com/avatar/abc", "hash": "abc123"},
+ "leak": {"name": "test leak"},
+ "message": {"message_id": "msg-123", "content": "test message"},
+ "reputationscore": {"entity_id": "entity-123"},
+ "riskprofile": {"entity_id": "entity-123"},
+ "script": {"script_id": "script-123"},
+ "session": {"session_id": "session-123"},
+ "webtracker": {"tracker_id": "tracker-123"},
+ "weapon": {"name": "Test Weapon"},
+ "whois": {"domain": {"domain": "example.com"}},
}
- @pytest.mark.parametrize('type_name', list(TYPE_TO_MODEL.keys()))
+ @pytest.mark.parametrize("type_name", list(TYPE_TO_MODEL.keys()))
def test_type_in_registry_has_test_data(self, type_name):
"""Verify that every type in registry has test data defined."""
- assert type_name in self.VALID_TEST_DATA, \
- f"Type '{type_name}' is in TYPE_TO_MODEL but has no test data in VALID_TEST_DATA. " \
+ assert type_name in self.VALID_TEST_DATA, (
+ f"Type '{type_name}' is in TYPE_TO_MODEL but has no test data in VALID_TEST_DATA. "
f"Please add minimal valid data for this type."
+ )
- @pytest.mark.parametrize('type_name,model_class', TYPE_TO_MODEL.items())
+ @pytest.mark.parametrize("type_name,model_class", TYPE_TO_MODEL.items())
def test_parse_all_types_with_valid_data(self, type_name, model_class):
"""Test parsing each type with valid minimal data."""
if type_name not in self.VALID_TEST_DATA:
pytest.skip(f"No test data for {type_name}")
node_data = {
- 'type': type_name,
+ "type": type_name,
**self.VALID_TEST_DATA[type_name],
- 'label': f'test-{type_name}',
+ "label": f"test-{type_name}",
# Add Neo4j fields that should be filtered
- 'sketch_id': 'test-sketch',
- 'created_at': '2024-01-01T00:00:00Z',
- 'x': 100,
- 'y': 200,
+ "sketch_id": "test-sketch",
+ "created_at": "2024-01-01T00:00:00Z",
+ "x": 100,
+ "y": 200,
}
result = parse_node_to_pydantic(node_data)
assert result is not None, f"Failed to parse valid {type_name} data"
- assert isinstance(result, model_class), \
- f"Expected {model_class.__name__} but got {type(result).__name__}"
+ assert isinstance(
+ result, model_class
+ ), f"Expected {model_class.__name__} but got {type(result).__name__}"
- @pytest.mark.parametrize('type_name,model_class', TYPE_TO_MODEL.items())
+ @pytest.mark.parametrize("type_name,model_class", TYPE_TO_MODEL.items())
def test_parse_all_types_with_empty_optional_fields(self, type_name, model_class):
"""Test that empty strings in optional fields don't break parsing."""
if type_name not in self.VALID_TEST_DATA:
@@ -570,36 +578,39 @@ class TestAllTypes:
# Add empty strings for some potential optional fields
node_data = {
- 'type': type_name,
+ "type": type_name,
**required_data,
- 'label': f'test-{type_name}',
- 'description': '', # Common optional field
- 'metadata': '',
- 'tags': '',
- 'notes': '',
- 'custom_field': '',
+ "label": f"test-{type_name}",
+ "description": "", # Common optional field
+ "metadata": "",
+ "tags": "",
+ "notes": "",
+ "custom_field": "",
}
result = parse_node_to_pydantic(node_data)
- assert result is not None, \
- f"Failed to parse {type_name} with empty optional fields"
+ assert (
+ result is not None
+ ), f"Failed to parse {type_name} with empty optional fields"
assert isinstance(result, model_class)
def test_type_registry_completeness(self):
"""Verify TYPE_TO_MODEL contains all expected types."""
# This is a sanity check to ensure the registry isn't empty
- assert len(TYPE_TO_MODEL) > 30, \
- f"TYPE_TO_MODEL should have 30+ types, found {len(TYPE_TO_MODEL)}"
+ assert (
+ len(TYPE_TO_MODEL) > 30
+ ), f"TYPE_TO_MODEL should have 30+ types, found {len(TYPE_TO_MODEL)}"
# Verify some key types are present
- required_types = ['domain', 'email', 'ip', 'phone', 'username', 'organization']
+ required_types = ["domain", "email", "ip", "phone", "username", "organization"]
for type_name in required_types:
- assert type_name in TYPE_TO_MODEL, \
- f"Required type '{type_name}' missing from TYPE_TO_MODEL"
+ assert (
+ type_name in TYPE_TO_MODEL
+ ), f"Required type '{type_name}' missing from TYPE_TO_MODEL"
def test_get_model_for_type_case_insensitive(self):
"""Test that get_model_for_type is case-insensitive."""
- assert get_model_for_type('domain') == get_model_for_type('Domain')
- assert get_model_for_type('EMAIL') == get_model_for_type('email')
- assert get_model_for_type('Ip') == get_model_for_type('ip')
+ assert get_model_for_type("domain") == get_model_for_type("Domain")
+ assert get_model_for_type("EMAIL") == get_model_for_type("email")
+ assert get_model_for_type("Ip") == get_model_for_type("ip")