A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat(types): auto_discovery of types
dextmorgnc3c328f2997c
diff --git a/flowsint-types/src/flowsint_types/__init__.py b/flowsint-types/src/flowsint_types/__init__.pyindex 30e6600e..3853aad4 100644--- a/flowsint-types/src/flowsint_types/__init__.py+++ b/flowsint-types/src/flowsint_types/__init__.py@@ -3,8 +3,15 @@ Flowsint Types - Pydantic models for flowsint"""# Import registry first to ensure it's ready for auto-registration-from .registry import TYPE_REGISTRY, flowsint_type, get_type+from .registry import TYPE_REGISTRY, flowsint_type, get_type, load_all_types+# Import base class+from .flowsint_base import FlowsintType++# Auto-discover and register all types+load_all_types()++# For backward compatibility, explicitly import commonly used typesfrom .address import Locationfrom .affiliation import Affiliationfrom .alias import Alias@@ -43,9 +50,6 @@ from .web_tracker import WebTrackerfrom .website import Websitefrom .whois import Whois-# Import base-from .flowsint_base import FlowsintType-from typing import Dict, Type, Any, Optionalfrom pydantic import BaseModel@@ -151,7 +155,6 @@ TYPE_TO_MODEL: Dict[str, Type[BaseModel]] = {"webtracker": WebTracker,"weapon": Weapon,"whois": Whois,- "FlowsintType": FlowsintType,}@@ -196,27 +199,21 @@ def clean_neo4j_node_data(node_data: Dict[str, Any]) -> Dict[str, Any]:# Skip Neo4j-specific fields (including 'type' which is the node type in Neo4j)if k in ["sketch_id", "created_at", "type", "x", "y", "caption", "color"]:continue-# Skip empty values (empty strings, None, empty lists, etc.)if v in ("", None, [], {}):continue-cleaned[k] = v-return cleaneddef parse_node_to_pydantic(node_data: Dict[str, Any]) -> Optional[BaseModel]:"""Parse a Neo4j node's properties into a Pydantic model instance.-Args:node_data: Dictionary containing node properties from Neo4j.Must include a 'type' field indicating the node type.-Returns:An instance of the appropriate Pydantic model, or None if parsing fails-Example:>>> node_data = {... 'type': 'domain',@@ -229,13 +226,10 @@ def parse_node_to_pydantic(node_data: Dict[str, Any]) -> Optional[BaseModel]:"""if not node_data or "type" not in node_data:return None-node_type = node_data.get("type")model_class = get_model_for_type(node_type)-if not model_class:return None-try:# Clean the node data firstcleaned_data = clean_neo4j_node_data(node_data)diff --git a/flowsint-types/src/flowsint_types/flowsint_base.py b/flowsint-types/src/flowsint_types/flowsint_base.pyindex 00e39244..e56ee73c 100644--- a/flowsint-types/src/flowsint_types/flowsint_base.py+++ b/flowsint-types/src/flowsint_types/flowsint_base.py@@ -1,36 +1,20 @@from pydantic import BaseModel, Field-from pydantic._internal._model_construction import ModelMetaclass-from typing import Optional, Type+from typing import Optional-class FlowsintTypeMetaclass(ModelMetaclass):- """- Metaclass for FlowsintType that automatically registers types in the TYPE_REGISTRY.-- This metaclass ensures that any class that inherits from FlowsintType- is automatically registered in the global type registry.- """-- def __new__(mcs, name: str, bases: tuple, namespace: dict, **kwargs):- # Create the class using Pydantic's metaclass- cls = super().__new__(mcs, name, bases, namespace, **kwargs)-- # Only register if this is a subclass of FlowsintType (not FlowsintType itself)- # and if it's not an abstract class- if name != 'FlowsintType' and bases:- # Import here to avoid circular dependency- from .registry import TYPE_REGISTRY- TYPE_REGISTRY.register(cls)-- return cls---class FlowsintType(BaseModel, metaclass=FlowsintTypeMetaclass):+class FlowsintType(BaseModel):"""Base class for all Flowsint entity types with label support.Label is optional but computed at definition time.- All classes that inherit from FlowsintType are automatically registered- in the global TYPE_REGISTRY and can be accessed by their class name.+ All classes that inherit from FlowsintType must be decorated with @flowsint_type+ to be registered in the global TYPE_REGISTRY and accessed by their class name.++ Usage:+ from flowsint_types.registry import flowsint_type++ @flowsint_type+ class Domain(FlowsintType):+ domain: str"""label: Optional[str] = Field(None, description="UI-readable label for this entity, the one used on the graph.", title="Label"diff --git a/flowsint-types/src/flowsint_types/registry.py b/flowsint-types/src/flowsint_types/registry.pyindex dfc6da76..30d47284 100644--- a/flowsint-types/src/flowsint_types/registry.py+++ b/flowsint-types/src/flowsint_types/registry.py@@ -2,10 +2,16 @@This module provides an automatic type registration system using decorators.All FlowsintType subclasses can be decorated with @flowsint_type to automaticallyregister themselves in the global TYPE_REGISTRY.++Auto-discovery is performed by calling load_all_types() which imports all modules+in the flowsint_types package, triggering the @flowsint_type decorators."""from typing import Dict, Type, TypeVar, Optionalfrom pydantic import BaseModel+import importlib+import pkgutil+import sysT = TypeVar("T", bound=BaseModel)@@ -135,3 +141,56 @@ def get_type(type_name: str, case_sensitive: bool = False) -> Optional[Type[Basereturn TYPE_REGISTRY.get(type_name)else:return TYPE_REGISTRY.get_lowercase(type_name)+++# Auto-discovery cache+_types_loaded = False+++def load_all_types() -> None:+ """+ Automatically discover and import all type modules in the flowsint_types package.++ This function uses importlib to dynamically import all Python modules in the+ flowsint_types package, which triggers the @flowsint_type decorators and+ registers all types in TYPE_REGISTRY.++ Features:+ - Only imports modules once (cached via _types_loaded flag)+ - Ignores private modules (starting with _)+ - Only imports .py files+ - Uses pkgutil for high performance iteration++ This function is idempotent - calling it multiple times is safe and efficient.+ """+ global _types_loaded++ # Early return if already loaded+ if _types_loaded:+ return++ # Get the flowsint_types package+ import flowsint_types+ package = flowsint_types+ package_path = package.__path__+ package_name = package.__name__++ # Iterate over all modules in the package+ for importer, modname, ispkg in pkgutil.iter_modules(package_path, prefix=f"{package_name}."):+ # Skip private modules+ if modname.split('.')[-1].startswith('_'):+ continue++ # Skip if already imported+ if modname in sys.modules:+ continue++ # Import the module to trigger @flowsint_type decorators+ try:+ importlib.import_module(modname)+ except Exception as e:+ # Log but don't fail - some modules might have optional dependencies+ print(f"Warning: Failed to import {modname}: {e}", file=sys.stderr)++ # Mark as loaded+ _types_loaded = Truediff --git a/flowsint-types/tests/test_parser.py b/flowsint-types/tests/test_parser.pyindex 29f3d68d..e98df467 100644--- a/flowsint-types/tests/test_parser.py+++ b/flowsint-types/tests/test_parser.py@@ -10,7 +10,6 @@ These tests ensure that parse_node_to_pydantic correctly handles:"""import pytest-from pydantic import BaseModelfrom flowsint_types import (parse_node_to_pydantic,clean_neo4j_node_data,@@ -21,14 +20,6 @@ from flowsint_types import (Email,Phone,Organization,- Username,- Website,- ASN,- CIDR,- SocialAccount,- Individual,- Phrase,- Port,)@@ -38,115 +29,115 @@ class TestCleanNeo4jNodeData:def test_clean_removes_neo4j_fields(self):"""Test that Neo4j-specific fields are removed."""node_data = {- 'type': 'domain', # Neo4j type field, should be removed- 'domain': 'example.com',- 'label': 'example.com',- 'sketch_id': 'should-be-removed',- 'created_at': 'should-be-removed',- 'x': 100,- 'y': 200,- 'caption': 'should-be-removed',- 'color': 'should-be-removed',+ "type": "domain",+ "domain": "example.com",+ "label": "example.com",+ "sketch_id": "should-be-removed",+ "created_at": "should-be-removed",+ "x": 100,+ "y": 200,+ "caption": "should-be-removed",+ "color": "should-be-removed",}result = clean_neo4j_node_data(node_data)- assert 'type' not in result # Neo4j 'type' field should be removed- assert 'domain' in result- assert 'label' in result- assert 'sketch_id' not in result- assert 'created_at' not in result- assert 'x' not in result- assert 'y' not in result- assert 'caption' not in result- assert 'color' not in result+ assert "type" not in result # Neo4j 'type' field should be removed+ assert "domain" in result+ assert "label" in result+ assert "sketch_id" not in result+ assert "created_at" not in result+ assert "x" not in result+ assert "y" not in result+ assert "caption" not in result+ assert "color" not in resultdef test_clean_removes_empty_strings(self):"""Test that empty strings are removed."""node_data = {- 'type': 'ip',- 'address': '192.168.1.1',- 'latitude': '',- 'longitude': '',- 'country': '',+ "type": "ip",+ "address": "192.168.1.1",+ "latitude": "",+ "longitude": "",+ "country": "",}result = clean_neo4j_node_data(node_data)- assert result == {'address': '192.168.1.1'}+ assert result == {"address": "192.168.1.1"}def test_clean_removes_none_values(self):"""Test that None values are removed."""node_data = {- 'type': 'domain',- 'domain': 'example.com',- 'some_field': None,+ "type": "domain",+ "domain": "example.com",+ "some_field": None,}result = clean_neo4j_node_data(node_data)- assert result == {'domain': 'example.com'}+ assert result == {"domain": "example.com"}def test_clean_removes_empty_lists(self):"""Test that empty lists are removed."""node_data = {- 'type': 'domain',- 'domain': 'example.com',- 'tags': [],+ "type": "domain",+ "domain": "example.com",+ "tags": [],}result = clean_neo4j_node_data(node_data)- assert result == {'domain': 'example.com'}+ assert result == {"domain": "example.com"}def test_clean_removes_empty_dicts(self):"""Test that empty dicts are removed."""node_data = {- 'type': 'domain',- 'domain': 'example.com',- 'metadata': {},+ "type": "domain",+ "domain": "example.com",+ "metadata": {},}result = clean_neo4j_node_data(node_data)- assert result == {'domain': 'example.com'}+ assert result == {"domain": "example.com"}def test_clean_preserves_valid_data(self):"""Test that valid data is preserved (except Neo4j 'type' field)."""node_data = {- 'type': 'ip',- 'address': '8.8.8.8',- 'label': '8.8.8.8',- 'latitude': 37.386,- 'longitude': -122.0838,- 'country': 'US',+ "type": "ip",+ "address": "8.8.8.8",+ "label": "8.8.8.8",+ "latitude": 37.386,+ "longitude": -122.0838,+ "country": "US",}result = clean_neo4j_node_data(node_data)# 'type' is filtered, rest is preservedexpected = {- 'address': '8.8.8.8',- 'label': '8.8.8.8',- 'latitude': 37.386,- 'longitude': -122.0838,- 'country': 'US',+ "address": "8.8.8.8",+ "label": "8.8.8.8",+ "latitude": 37.386,+ "longitude": -122.0838,+ "country": "US",}assert result == expecteddef test_clean_preserves_zero_values(self):"""Test that zero values are preserved (not treated as empty)."""node_data = {- 'type': 'ip',- 'address': '127.0.0.1',- 'latitude': 0,- 'longitude': 0,+ "type": "ip",+ "address": "127.0.0.1",+ "latitude": 0,+ "longitude": 0,}result = clean_neo4j_node_data(node_data)- assert result['latitude'] == 0- assert result['longitude'] == 0+ assert result["latitude"] == 0+ assert result["longitude"] == 0def test_clean_empty_dict(self):"""Test cleaning an empty dict."""@@ -160,44 +151,44 @@ class TestParseNodeToPydantic:def test_parse_domain_valid(self):"""Test parsing a valid domain node from Neo4j."""node_data = {- 'type': 'domain',- 'domain': 'example.com',- 'label': 'example.com',- 'root': True,- 'sketch_id': 'abc-123',- 'created_at': '2024-01-01T00:00:00Z',- 'x': 100,- 'y': 200,+ "type": "domain",+ "domain": "example.com",+ "label": "example.com",+ "root": True,+ "sketch_id": "abc-123",+ "created_at": "2024-01-01T00:00:00Z",+ "x": 100,+ "y": 200,}result = parse_node_to_pydantic(node_data)assert result is not Noneassert isinstance(result, Domain)- assert result.domain == 'example.com'- assert result.label == 'example.com'+ assert result.domain == "example.com"+ assert result.label == "example.com"assert result.root == Truedef test_parse_ip_with_empty_optional_fields(self):"""Test parsing IP node with empty strings for optional fields."""node_data = {- 'type': 'ip',- 'address': '192.168.1.1',- 'label': '192.168.1.1',- 'latitude': '', # Empty string should be filtered- 'longitude': '', # Empty string should be filtered- 'country': '',- 'city': '',- 'isp': '',- 'sketch_id': 'abc-123',- 'created_at': '2024-01-01T00:00:00Z',+ "type": "ip",+ "address": "192.168.1.1",+ "label": "192.168.1.1",+ "latitude": "", # Empty string should be filtered+ "longitude": "", # Empty string should be filtered+ "country": "",+ "city": "",+ "isp": "",+ "sketch_id": "abc-123",+ "created_at": "2024-01-01T00:00:00Z",}result = parse_node_to_pydantic(node_data)assert result is not Noneassert isinstance(result, Ip)- assert result.address == '192.168.1.1'+ assert result.address == "192.168.1.1"assert result.latitude is Noneassert result.longitude is Noneassert result.country is None@@ -207,83 +198,83 @@ class TestParseNodeToPydantic:def test_parse_ip_with_valid_optional_fields(self):"""Test parsing IP node with valid optional fields."""node_data = {- 'type': 'ip',- 'address': '8.8.8.8',- 'label': '8.8.8.8',- 'latitude': 37.386,- 'longitude': -122.0838,- 'country': 'US',- 'city': 'Mountain View',- 'isp': 'Google LLC',+ "type": "ip",+ "address": "8.8.8.8",+ "label": "8.8.8.8",+ "latitude": 37.386,+ "longitude": -122.0838,+ "country": "US",+ "city": "Mountain View",+ "isp": "Google LLC",}result = parse_node_to_pydantic(node_data)assert result is not Noneassert isinstance(result, Ip)- assert result.address == '8.8.8.8'+ assert result.address == "8.8.8.8"assert result.latitude == 37.386assert result.longitude == -122.0838- assert result.country == 'US'- assert result.city == 'Mountain View'- assert result.isp == 'Google LLC'+ assert result.country == "US"+ assert result.city == "Mountain View"+ assert result.isp == "Google LLC"def test_parse_email_valid(self):"""Test parsing a valid email node."""node_data = {- 'type': 'email',- 'email': '[email protected]',- 'label': '[email protected]',+ "type": "email",+ "email": "[email protected]",+ "label": "[email protected]",}result = parse_node_to_pydantic(node_data)assert result is not Noneassert isinstance(result, Email)- assert result.email == '[email protected]'+ assert result.email == "[email protected]"def test_parse_phone_valid(self):"""Test parsing a valid phone node."""node_data = {- 'type': 'phone',- 'number': '+33612345678',- 'label': '+33612345678',- 'country': 'FR',+ "type": "phone",+ "number": "+33612345678",+ "label": "+33612345678",+ "country": "FR",}result = parse_node_to_pydantic(node_data)assert result is not Noneassert isinstance(result, Phone)- assert result.number == '+33612345678'- assert result.country == 'FR'+ assert result.number == "+33612345678"+ assert result.country == "FR"def test_parse_organization_valid(self):"""Test parsing a valid organization node."""node_data = {- 'type': 'organization',- 'name': 'ACME Corp',- 'label': 'ACME Corp',+ "type": "organization",+ "name": "ACME Corp",+ "label": "ACME Corp",}result = parse_node_to_pydantic(node_data)assert result is not Noneassert isinstance(result, Organization)- assert result.name == 'ACME Corp'+ assert result.name == "ACME Corp"def test_parse_node_filters_neo4j_fields(self):"""Test that Neo4j-specific fields are filtered out."""node_data = {- 'type': 'domain',- 'domain': 'test.com',- 'label': 'test.com',- 'sketch_id': 'should-be-filtered',- 'created_at': 'should-be-filtered',- 'x': 100,- 'y': 200,- 'caption': 'should-be-filtered',- 'color': 'should-be-filtered',+ "type": "domain",+ "domain": "test.com",+ "label": "test.com",+ "sketch_id": "should-be-filtered",+ "created_at": "should-be-filtered",+ "x": 100,+ "y": 200,+ "caption": "should-be-filtered",+ "color": "should-be-filtered",}result = parse_node_to_pydantic(node_data)@@ -291,13 +282,13 @@ class TestParseNodeToPydantic:assert result is not Noneassert isinstance(result, Domain)# These fields should not cause errors even though they're not in the Pydantic model- assert result.domain == 'test.com'+ assert result.domain == "test.com"def test_parse_node_missing_type(self):"""Test that nodes without 'type' field return None."""node_data = {- 'domain': 'example.com',- 'label': 'example.com',+ "domain": "example.com",+ "label": "example.com",}result = parse_node_to_pydantic(node_data)@@ -307,8 +298,8 @@ class TestParseNodeToPydantic:def test_parse_node_unknown_type(self):"""Test that nodes with unknown type return None."""node_data = {- 'type': 'unknown_type_xyz',- 'some_field': 'some_value',+ "type": "unknown_type_xyz",+ "some_field": "some_value",}result = parse_node_to_pydantic(node_data)@@ -318,9 +309,9 @@ class TestParseNodeToPydantic:def test_parse_node_invalid_email(self):"""Test that invalid data fails validation and returns None."""node_data = {- 'type': 'email',- 'email': 'not-an-email', # Invalid email- 'label': 'not-an-email',+ "type": "email",+ "email": "not-an-email", # Invalid email+ "label": "not-an-email",}result = parse_node_to_pydantic(node_data)@@ -330,9 +321,9 @@ class TestParseNodeToPydantic:def test_parse_node_invalid_ip(self):"""Test that invalid IP address fails validation and returns None."""node_data = {- 'type': 'ip',- 'address': '999.999.999.999', # Invalid IP- 'label': '999.999.999.999',+ "type": "ip",+ "address": "999.999.999.999", # Invalid IP+ "label": "999.999.999.999",}result = parse_node_to_pydantic(node_data)@@ -342,9 +333,9 @@ class TestParseNodeToPydantic:def test_parse_node_missing_required_field(self):"""Test that missing required fields cause validation to fail."""node_data = {- 'type': 'email',+ "type": "email",# Missing 'email' field which is required- 'label': 'test',+ "label": "test",}result = parse_node_to_pydantic(node_data)@@ -354,9 +345,9 @@ class TestParseNodeToPydantic:def test_parse_node_empty_string_for_required_field(self):"""Test that empty string for required field is filtered and causes validation to fail."""node_data = {- 'type': 'domain',- 'domain': '', # Empty string should be filtered, causing validation to fail- 'label': 'test',+ "type": "domain",+ "domain": "", # Empty string should be filtered, causing validation to fail+ "label": "test",}result = parse_node_to_pydantic(node_data)@@ -366,11 +357,11 @@ class TestParseNodeToPydantic:def test_parse_node_filters_none_values(self):"""Test that None values are filtered out."""node_data = {- 'type': 'ip',- 'address': '1.2.3.4',- 'label': '1.2.3.4',- 'latitude': None,- 'longitude': None,+ "type": "ip",+ "address": "1.2.3.4",+ "label": "1.2.3.4",+ "latitude": None,+ "longitude": None,}result = parse_node_to_pydantic(node_data)@@ -383,10 +374,10 @@ class TestParseNodeToPydantic:def test_parse_node_filters_empty_lists(self):"""Test that empty lists are filtered out."""node_data = {- 'type': 'domain',- 'domain': 'example.com',- 'label': 'example.com',- 'some_list_field': [], # Should be filtered+ "type": "domain",+ "domain": "example.com",+ "label": "example.com",+ "some_list_field": [], # Should be filtered}result = parse_node_to_pydantic(node_data)@@ -397,10 +388,10 @@ class TestParseNodeToPydantic:def test_parse_node_filters_empty_dicts(self):"""Test that empty dicts are filtered out."""node_data = {- 'type': 'domain',- 'domain': 'example.com',- 'label': 'example.com',- 'some_dict_field': {}, # Should be filtered+ "type": "domain",+ "domain": "example.com",+ "label": "example.com",+ "some_dict_field": {}, # Should be filtered}result = parse_node_to_pydantic(node_data)@@ -425,11 +416,11 @@ class TestEdgeCases:def test_ip_with_string_zero_coordinates(self):"""Test IP with '0' string for coordinates (edge case from real data)."""node_data = {- 'type': 'ip',- 'address': '127.0.0.1',- 'label': '127.0.0.1',- 'latitude': '0',- 'longitude': '0',+ "type": "ip",+ "address": "127.0.0.1",+ "label": "127.0.0.1",+ "latitude": "0",+ "longitude": "0",}result = parse_node_to_pydantic(node_data)@@ -443,34 +434,34 @@ class TestEdgeCases:def test_domain_with_subdomain(self):"""Test domain parsing with subdomain."""node_data = {- 'type': 'domain',- 'domain': 'sub.example.com',- 'label': 'sub.example.com',+ "type": "domain",+ "domain": "sub.example.com",+ "label": "sub.example.com",}result = parse_node_to_pydantic(node_data)assert result is not Noneassert isinstance(result, Domain)- assert result.domain == 'sub.example.com'+ assert result.domain == "sub.example.com"assert result.root == False # Should be computed as not rootdef test_multiple_neo4j_specific_fields(self):"""Test that all Neo4j-specific fields are properly filtered."""neo4j_fields = {- 'sketch_id': 'test',- 'created_at': '2024-01-01',- 'type': 'domain',- 'caption': 'test caption',- 'x': 100.5,- 'y': 200.5,- 'color': '#FF0000',+ "sketch_id": "test",+ "created_at": "2024-01-01",+ "type": "domain",+ "caption": "test caption",+ "x": 100.5,+ "y": 200.5,+ "color": "#FF0000",}node_data = {**neo4j_fields,- 'domain': 'example.com',- 'label': 'example.com',+ "domain": "example.com",+ "label": "example.com",}result = parse_node_to_pydantic(node_data)@@ -478,8 +469,8 @@ class TestEdgeCases:assert result is not Noneassert isinstance(result, Domain)# Verify only valid fields were used- assert result.domain == 'example.com'- assert result.label == 'example.com'+ assert result.domain == "example.com"+ assert result.label == "example.com"class TestAllTypes:@@ -488,78 +479,95 @@ class TestAllTypes:# Mapping of type names to valid minimal data for testing# This ensures every type in TYPE_TO_MODEL can be parsedVALID_TEST_DATA = {- 'domain': {'domain': 'example.com'},- 'email': {'email': '[email protected]'},- 'ip': {'address': '192.168.1.1'},- 'phone': {'number': '+33612345678'},- 'username': {'value': 'john_doe'},- 'organization': {'name': 'ACME Corp'},- 'individual': {'first_name': 'John', 'last_name': 'Doe'},- 'socialaccount': {'username': {'value': 'johndoe'}},- 'asn': {'asn_str': 'AS15169'},- 'cidr': {'network': '192.168.1.0/24'},- 'cryptowallet': {'address': '1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa'},- 'cryptowallettransaction': {'source': {'address': '1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa'}},- 'cryptonft': {'wallet': {'address': '1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa'}, 'contract_address': '0x' + 'a' * 40, 'token_id': '123'},- 'website': {'url': 'https://example.com'},- 'port': {'number': 443},- 'phrase': {'text': 'test phrase'},- 'breach': {'name': 'Test Breach'},- 'credential': {'username': 'user'},- 'device': {'device_id': 'device-123'},- 'document': {'title': 'Test Document'},- 'file': {'filename': 'test.txt'},- 'malware': {'name': 'TestMalware'},- 'sslcertificate': {'subject': 'CN=example.com'},- 'location': {'address': '123 Main St', 'city': 'Paris', 'country': 'France', 'zip': '75001'},- 'affiliation': {'organization': 'test org'},- 'alias': {'alias': 'test_alias'},- 'bankaccount': {'account_number': '123456789'},- 'creditcard': {'card_number': '4111111111111111'},- 'dnsrecord': {'name': 'example.com', 'record_type': 'A', 'value': '192.168.1.1'},- 'gravatar': {'src': 'https://gravatar.com/avatar/abc', 'hash': 'abc123'},- 'leak': {'name': 'test leak'},- 'message': {'message_id': 'msg-123', 'content': 'test message'},- 'reputationscore': {'entity_id': 'entity-123'},- 'riskprofile': {'entity_id': 'entity-123'},- 'script': {'script_id': 'script-123'},- 'session': {'session_id': 'session-123'},- 'webtracker': {'tracker_id': 'tracker-123'},- 'weapon': {'name': 'Test Weapon'},- 'whois': {'domain': {'domain': 'example.com'}},+ "domain": {"domain": "example.com"},+ "email": {"email": "[email protected]"},+ "ip": {"address": "192.168.1.1"},+ "phone": {"number": "+33612345678"},+ "username": {"value": "john_doe"},+ "organization": {"name": "ACME Corp"},+ "individual": {"first_name": "John", "last_name": "Doe"},+ "socialaccount": {"username": {"value": "johndoe"}},+ "asn": {"asn_str": "AS15169"},+ "cidr": {"network": "192.168.1.0/24"},+ "cryptowallet": {"address": "1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa"},+ "cryptowallettransaction": {+ "source": {"address": "1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa"}+ },+ "cryptonft": {+ "wallet": {"address": "1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa"},+ "contract_address": "0x" + "a" * 40,+ "token_id": "123",+ },+ "website": {"url": "https://example.com"},+ "port": {"number": 443},+ "phrase": {"text": "test phrase"},+ "breach": {"name": "Test Breach"},+ "credential": {"username": "user"},+ "device": {"device_id": "device-123"},+ "document": {"title": "Test Document"},+ "file": {"filename": "test.txt"},+ "malware": {"name": "TestMalware"},+ "sslcertificate": {"subject": "CN=example.com"},+ "location": {+ "address": "123 Main St",+ "city": "Paris",+ "country": "France",+ "zip": "75001",+ },+ "affiliation": {"organization": "test org"},+ "alias": {"alias": "test_alias"},+ "bankaccount": {"account_number": "123456789"},+ "creditcard": {"card_number": "4111111111111111"},+ "dnsrecord": {+ "name": "example.com",+ "record_type": "A",+ "value": "192.168.1.1",+ },+ "gravatar": {"src": "https://gravatar.com/avatar/abc", "hash": "abc123"},+ "leak": {"name": "test leak"},+ "message": {"message_id": "msg-123", "content": "test message"},+ "reputationscore": {"entity_id": "entity-123"},+ "riskprofile": {"entity_id": "entity-123"},+ "script": {"script_id": "script-123"},+ "session": {"session_id": "session-123"},+ "webtracker": {"tracker_id": "tracker-123"},+ "weapon": {"name": "Test Weapon"},+ "whois": {"domain": {"domain": "example.com"}},}- @pytest.mark.parametrize('type_name', list(TYPE_TO_MODEL.keys()))+ @pytest.mark.parametrize("type_name", list(TYPE_TO_MODEL.keys()))def test_type_in_registry_has_test_data(self, type_name):"""Verify that every type in registry has test data defined."""- assert type_name in self.VALID_TEST_DATA, \- f"Type '{type_name}' is in TYPE_TO_MODEL but has no test data in VALID_TEST_DATA. " \+ assert type_name in self.VALID_TEST_DATA, (+ f"Type '{type_name}' is in TYPE_TO_MODEL but has no test data in VALID_TEST_DATA. "f"Please add minimal valid data for this type."+ )- @pytest.mark.parametrize('type_name,model_class', TYPE_TO_MODEL.items())+ @pytest.mark.parametrize("type_name,model_class", TYPE_TO_MODEL.items())def test_parse_all_types_with_valid_data(self, type_name, model_class):"""Test parsing each type with valid minimal data."""if type_name not in self.VALID_TEST_DATA:pytest.skip(f"No test data for {type_name}")node_data = {- 'type': type_name,+ "type": type_name,**self.VALID_TEST_DATA[type_name],- 'label': f'test-{type_name}',+ "label": f"test-{type_name}",# Add Neo4j fields that should be filtered- 'sketch_id': 'test-sketch',- 'created_at': '2024-01-01T00:00:00Z',- 'x': 100,- 'y': 200,+ "sketch_id": "test-sketch",+ "created_at": "2024-01-01T00:00:00Z",+ "x": 100,+ "y": 200,}result = parse_node_to_pydantic(node_data)assert result is not None, f"Failed to parse valid {type_name} data"- assert isinstance(result, model_class), \- f"Expected {model_class.__name__} but got {type(result).__name__}"+ assert isinstance(+ result, model_class+ ), f"Expected {model_class.__name__} but got {type(result).__name__}"- @pytest.mark.parametrize('type_name,model_class', TYPE_TO_MODEL.items())+ @pytest.mark.parametrize("type_name,model_class", TYPE_TO_MODEL.items())def test_parse_all_types_with_empty_optional_fields(self, type_name, model_class):"""Test that empty strings in optional fields don't break parsing."""if type_name not in self.VALID_TEST_DATA:@@ -570,36 +578,39 @@ class TestAllTypes:# Add empty strings for some potential optional fieldsnode_data = {- 'type': type_name,+ "type": type_name,**required_data,- 'label': f'test-{type_name}',- 'description': '', # Common optional field- 'metadata': '',- 'tags': '',- 'notes': '',- 'custom_field': '',+ "label": f"test-{type_name}",+ "description": "", # Common optional field+ "metadata": "",+ "tags": "",+ "notes": "",+ "custom_field": "",}result = parse_node_to_pydantic(node_data)- assert result is not None, \- f"Failed to parse {type_name} with empty optional fields"+ assert (+ result is not None+ ), f"Failed to parse {type_name} with empty optional fields"assert isinstance(result, model_class)def test_type_registry_completeness(self):"""Verify TYPE_TO_MODEL contains all expected types."""# This is a sanity check to ensure the registry isn't empty- assert len(TYPE_TO_MODEL) > 30, \- f"TYPE_TO_MODEL should have 30+ types, found {len(TYPE_TO_MODEL)}"+ assert (+ len(TYPE_TO_MODEL) > 30+ ), f"TYPE_TO_MODEL should have 30+ types, found {len(TYPE_TO_MODEL)}"# Verify some key types are present- required_types = ['domain', 'email', 'ip', 'phone', 'username', 'organization']+ required_types = ["domain", "email", "ip", "phone", "username", "organization"]for type_name in required_types:- assert type_name in TYPE_TO_MODEL, \- f"Required type '{type_name}' missing from TYPE_TO_MODEL"+ assert (+ type_name in TYPE_TO_MODEL+ ), f"Required type '{type_name}' missing from TYPE_TO_MODEL"def test_get_model_for_type_case_insensitive(self):"""Test that get_model_for_type is case-insensitive."""- assert get_model_for_type('domain') == get_model_for_type('Domain')- assert get_model_for_type('EMAIL') == get_model_for_type('email')- assert get_model_for_type('Ip') == get_model_for_type('ip')+ assert get_model_for_type("domain") == get_model_for_type("Domain")+ assert get_model_for_type("EMAIL") == get_model_for_type("email")+ assert get_model_for_type("Ip") == get_model_for_type("ip")