A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat: minifiy transform logic
dextmorgnade74800aa34
diff --git a/Makefile b/Makefileindex 68e8fcaf..29c2bf3b 100644--- a/Makefile+++ b/Makefile@@ -1,6 +1,6 @@PROJECT_ROOT := $(shell pwd)-.PHONY: install run stop stop-dev stop-prod infra api frontend celery clean dev prod check-env build-dev build-prod open-browser+.PHONY: install run stop stop-dev stop-prod infra api frontend celery clean dev prod check-env build-dev build-prod open-browser testENV_DIRS := . flowsint-api flowsint-core flowsint-appopen-browser:@@ -43,6 +43,12 @@ check-env:fi; \done+test:+ @echo "🔎 Running tests..."+ cd $(PROJECT_ROOT)/flowsint-types && poetry run pytest+ cd $(PROJECT_ROOT)/flowsint-core && poetry run pytest+ cd $(PROJECT_ROOT)/flowsint-transforms && poetry run pytest+install:@echo "🚀 Installing Flowsint project modules..."@if ! command -v poetry >/dev/null 2>&1; then \diff --git a/flowsint-core/README.md b/flowsint-core/README.mdindex ea792210..f5ff96ef 100644--- a/flowsint-core/README.md+++ b/flowsint-core/README.md@@ -1,7 +1,7 @@# flowsint-coreThe repository containing the core components for flowsint.-It contains:+It contains:- Celery tasks- Authentication- NEO4J database connectors@@ -11,4 +11,10 @@ It contains:- Transform registry- Encrypted vault+## Testing++```bash+poetry run pytest+```+> ⚠️ 🚧 Work in progress !.diff --git a/flowsint-core/src/flowsint_core/core/transform_base.py b/flowsint-core/src/flowsint_core/core/transform_base.pyindex b09384cb..bcd32113 100644--- a/flowsint-core/src/flowsint_core/core/transform_base.py+++ b/flowsint-core/src/flowsint_core/core/transform_base.py@@ -62,9 +62,9 @@ class Transform(ABC):from flowsint_types import Ipclass MyTransform(Transform):- # Define types as class attributes- InputType = List[Domain]- OutputType = List[Ip]+ # Define types as class attributes (base types, not lists)+ InputType = Domain+ OutputType = Ip@classmethoddef name(cls):@@ -78,13 +78,14 @@ class Transform(ABC):def key(cls):return "domain"- def preprocess(self, data: InputType) -> InputType:- cleaned: InputType = []- # ... implementation- return cleaned+ # preprocess receives a list and returns a list of validated InputType instances+ def preprocess(self, data: List) -> List[InputType]:+ # Generic implementation handles validation automatically+ return super().preprocess(data)- async def scan(self, data: InputType) -> OutputType:- results: OutputType = []+ # scan receives a list of InputType and returns a list of OutputType+ async def scan(self, data: List[InputType]) -> List[OutputType]:+ results: List[OutputType] = []# ... implementationreturn results@@ -94,6 +95,7 @@ class Transform(ABC):```The base class automatically provides:+ - Generic preprocess() that validates inputs using InputType- input_schema() method using InputType- output_schema() method using OutputType- Error handling for missing type definitions@@ -263,17 +265,20 @@ class Transform(ABC):schema = adapter.json_schema()# Handle different schema structures- if "$defs" in schema and schema["$defs"]:- # Follow the $ref in items to get the correct type (not just the first one)- items_ref = schema.get("items", {}).get("$ref")- if items_ref:- # Extract type name from $ref like "#/$defs/Website" -> "Website"- type_name = items_ref.split("/")[-1]- details = schema["$defs"][type_name]- else:- # Fallback: get the first type definition (for backward compatibility)- type_name, details = list(schema["$defs"].items())[0]-+ # Check for direct properties first (even if $defs exists for nested types)+ if "properties" in schema and "title" in schema:+ # Direct type definition (e.g., Domain, Ip, Website)+ return {+ "type": schema.get("title", "Any"),+ "properties": [+ {"name": prop, "type": resolve_type(info, schema)}+ for prop, info in schema["properties"].items()+ ],+ }+ elif "$defs" in schema and schema["$defs"] and "$ref" in schema:+ # Reference to a type in $defs+ type_name = schema["$ref"].split("/")[-1]+ details = schema["$defs"][type_name]return {"type": type_name,"properties": [@@ -282,7 +287,7 @@ class Transform(ABC):],}else:- # Handle simpler schemas+ # Fallback for unknown schema structuresreturn {"type": schema.get("title", "Any"),"properties": [{"name": "value", "type": "object"}],@@ -303,17 +308,20 @@ class Transform(ABC):schema = adapter.json_schema()# Handle different schema structures- if "$defs" in schema and schema["$defs"]:- # Follow the $ref in items to get the correct type (not just the first one)- items_ref = schema.get("items", {}).get("$ref")- if items_ref:- # Extract type name from $ref like "#/$defs/Website" -> "Website"- type_name = items_ref.split("/")[-1]- details = schema["$defs"][type_name]- else:- # Fallback: get the first type definition (for backward compatibility)- type_name, details = list(schema["$defs"].items())[0]-+ # Check for direct properties first (even if $defs exists for nested types)+ if "properties" in schema and "title" in schema:+ # Direct type definition (e.g., Domain, Ip, Website)+ return {+ "type": schema.get("title", "Any"),+ "properties": [+ {"name": prop, "type": resolve_type(info, schema)}+ for prop, info in schema["properties"].items()+ ],+ }+ elif "$defs" in schema and schema["$defs"] and "$ref" in schema:+ # Reference to a type in $defs+ type_name = schema["$ref"].split("/")[-1]+ details = schema["$defs"][type_name]return {"type": type_name,"properties": [@@ -322,7 +330,7 @@ class Transform(ABC):],}else:- # Handle simpler schemas+ # Fallback for unknown schema structuresreturn {"type": schema.get("title", "Any"),"properties": [{"name": "value", "type": "object"}],@@ -350,24 +358,24 @@ class Transform(ABC):Returns:The secret value from the vault, or default if not found"""- return self.params.get(key_name, default)+ value = self.params.get(key_name, default)+ # If the value is None, return the default instead (allows fallback to env vars)+ return value if value is not None else defaultdef preprocess(self, values: List) -> List:"""Generic preprocess that validates and converts input using InputType.Automatically handles dicts, objects, and strings (using the model's primary field).Invalid items are skipped silently.++ Note: InputType should be defined as the base type (e.g., Ip, Domain),+ not as a List (e.g., List[Ip]). The preprocess method expects a list of values+ and returns a list of validated InputType instances."""if self.InputType is NotImplemented:return values- from typing import get_args-- type_args = get_args(self.InputType)- if not type_args:- return values-- base_type = type_args[0]+ base_type = self.InputTypeadapter = TypeAdapter(base_type)# Trouver le champ primaire marqué par Field(..., primary=True)@@ -485,8 +493,16 @@ class Transform(ABC):key_prop = primary_fieldkey_value = getattr(obj, primary_field)- # Merge object properties with any overrides- obj_properties = obj.__dict__.copy()+ # If key_value is itself a Pydantic model, extract its primary value+ if isinstance(key_value, BaseModel):+ key_value = self._extract_primary_value(key_value)++ # Merge object properties with any overrides, but skip nested Pydantic objects+ obj_properties = {}+ for k, v in obj.__dict__.items():+ # Skip nested Pydantic objects+ if not isinstance(v, BaseModel):+ obj_properties[k] = vobj_properties.update(properties)properties = obj_propertieselse:@@ -571,11 +587,19 @@ class Transform(ABC):from_primary_field = self._get_primary_field(from_obj)from_key_value = getattr(from_obj, from_primary_field)+ # If key_value is a Pydantic model, extract its primary value+ if isinstance(from_key_value, BaseModel):+ from_key_value = self._extract_primary_value(from_key_value)+# Extract to_node infoto_node_type = to_obj.__class__.__name__.lower()to_primary_field = self._get_primary_field(to_obj)to_key_value = getattr(to_obj, to_primary_field)+ # If key_value is a Pydantic model, extract its primary value+ if isinstance(to_key_value, BaseModel):+ to_key_value = self._extract_primary_value(to_key_value)+self._graph_service.create_relationship(from_type=from_node_type,from_key=from_primary_field,@@ -620,6 +644,20 @@ class Transform(ABC):return primary_field+ def _extract_primary_value(self, obj: BaseModel) -> Any:+ """+ Extract the primitive value from a Pydantic object recursively.+ If the primary field is itself a Pydantic object, recursively extract its primary value.+ """+ primary_field = self._get_primary_field(obj)+ value = getattr(obj, primary_field)++ # If the value is still a Pydantic model, recursively extract+ if isinstance(value, BaseModel):+ return self._extract_primary_value(value)++ return value+def log_graph_message(self, message: str) -> None:"""Log a graph operation message.diff --git a/flowsint-core/src/flowsint_core/core/vault.py b/flowsint-core/src/flowsint_core/core/vault.pyindex 34138eb1..d8427407 100644--- a/flowsint-core/src/flowsint_core/core/vault.py+++ b/flowsint-core/src/flowsint_core/core/vault.py@@ -4,7 +4,7 @@ import uuidfrom sqlalchemy.orm import Sessionfrom sqlalchemy import selectfrom .models import Key-from datetime import datetime+from datetime import datetime, UTCimport base64from cryptography.hazmat.primitives.kdf.hkdf import HKDFfrom cryptography.hazmat.primitives import hashes@@ -115,7 +115,7 @@ class Vault(VaultProtocol):key_version=self.version,owner_id=self.owner_id,ciphertext=encrypted_key_dict.get("ciphertext"),- created_at=datetime.utcnow(),+ created_at=datetime.now(UTC),)self.db.add(new_key)self.db.commit()diff --git a/flowsint-core/tests/test_transform_base_simplified_api.py b/flowsint-core/tests/test_transform_base_simplified_api.pyindex abcecbb9..b3aa19dc 100644--- a/flowsint-core/tests/test_transform_base_simplified_api.py+++ b/flowsint-core/tests/test_transform_base_simplified_api.py@@ -7,11 +7,11 @@ from flowsint_types.individual import Individualfrom typing import List-class TestTransform(Transform):+class MockTransform(Transform):"""Simple transform for testing."""- InputType = List[Domain]- OutputType = List[Domain]+ InputType = Domain+ OutputType = Domain@classmethoddef name(cls) -> str:@@ -25,13 +25,13 @@ class TestTransform(Transform):def key(cls) -> str:return "domain"- async def scan(self, data: InputType) -> OutputType:+ async def scan(self, data: List[InputType]) -> List[OutputType]:return datadef test_create_node_with_pydantic_object():"""Test that create_node works with Pydantic objects."""- transform = TestTransform(sketch_id="test", scan_id="test")+ transform = MockTransform(sketch_id="test", scan_id="test")# Create a domain objectdomain = Domain(domain="example.com")@@ -46,7 +46,7 @@ def test_create_node_with_pydantic_object():def test_create_relationship_with_pydantic_objects():"""Test that create_relationship works with Pydantic objects."""- transform = TestTransform(sketch_id="test", scan_id="test")+ transform = MockTransform(sketch_id="test", scan_id="test")# Create objectsindividual = Individual(@@ -62,7 +62,7 @@ def test_create_relationship_with_pydantic_objects():def test_create_node_legacy_signature():"""Test that legacy create_node signature still works."""- transform = TestTransform(sketch_id="test", scan_id="test")+ transform = MockTransform(sketch_id="test", scan_id="test")domain = Domain(domain="example.com")@@ -72,7 +72,7 @@ def test_create_node_legacy_signature():def test_create_relationship_legacy_signature():"""Test that legacy create_relationship signature still works."""- transform = TestTransform(sketch_id="test", scan_id="test")+ transform = MockTransform(sketch_id="test", scan_id="test")# Legacy signature should still worktransform.create_relationship(@@ -88,7 +88,7 @@ def test_create_relationship_legacy_signature():def test_get_primary_field():"""Test the _get_primary_field helper method."""- transform = TestTransform(sketch_id="test", scan_id="test")+ transform = MockTransform(sketch_id="test", scan_id="test")# Test with Domain (has primary field marked)domain = Domain(domain="example.com")@@ -109,7 +109,7 @@ def test_get_primary_field():def test_create_node_with_property_override():"""Test that property overrides work with Pydantic objects."""- transform = TestTransform(sketch_id="test", scan_id="test")+ transform = MockTransform(sketch_id="test", scan_id="test")domain = Domain(domain="example.com")diff --git a/flowsint-core/tests/vault/test_transform_vault_integration.py b/flowsint-core/tests/vault/test_transform_vault_integration.pyindex 40569ab8..96455765 100644--- a/flowsint-core/tests/vault/test_transform_vault_integration.py+++ b/flowsint-core/tests/vault/test_transform_vault_integration.py@@ -9,8 +9,8 @@ from flowsint_core.core.transform_base import Transformclass MockTransform(Transform):"""Mock transform for testing."""- InputType = List[str]- OutputType = List[str]+ InputType = str+ OutputType = str@classmethoddef name(cls) -> str:@@ -49,7 +49,7 @@ class MockTransform(Transform):},]- async def scan(self, data: InputType) -> OutputType:+ async def scan(self, data: List[InputType]) -> List[OutputType]:return data@@ -75,123 +75,169 @@ def transform(mock_vault, sketch_id):scan_id="scan_123",vault=mock_vault,params={},+ params_schema=MockTransform.get_params_schema(),)class TestResolveParams:"""Tests for resolve_params() method."""- def test_resolve_vault_secret_by_name(self, transform, mock_vault):+ @pytest.mark.asyncio+ async def test_resolve_vault_secret_by_name(self, transform, mock_vault):"""Test resolving a vault secret by parameter name."""secret_value = "my-api-key-12345"mock_vault.get_secret.return_value = secret_value- resolved = transform.resolve_params()+ await transform.async_init()# Should try to get by name "TEST_API_KEY"assert mock_vault.get_secret.called- assert "TEST_API_KEY" in resolved- assert resolved["TEST_API_KEY"] == secret_value+ assert "TEST_API_KEY" in transform.params+ assert transform.params["TEST_API_KEY"] == secret_value- def test_resolve_vault_secret_by_id(self, transform, mock_vault):+ @pytest.mark.asyncio+ async def test_resolve_vault_secret_by_id(self, sketch_id, mock_vault):"""Test resolving a vault secret by vault ID."""secret_value = "my-api-key-12345"vault_id = str(uuid.uuid4())- # Set params with a vault ID- transform.params = {"TEST_API_KEY": vault_id}+ # Mock vault to return secret for ID, and optional key default+ def get_secret_side_effect(key):+ if key == vault_id:+ return secret_value+ elif key == "OPTIONAL_KEY":+ return None # Will use default+ return None- # First call (by ID) returns the secret- mock_vault.get_secret.return_value = secret_value+ mock_vault.get_secret.side_effect = get_secret_side_effect- resolved = transform.resolve_params()+ # Create transform with vault ID in params+ transform = MockTransform(+ sketch_id=sketch_id,+ scan_id="scan_123",+ vault=mock_vault,+ params={"TEST_API_KEY": vault_id},+ params_schema=MockTransform.get_params_schema(),+ )- # Should try to get by ID first- mock_vault.get_secret.assert_called_with(vault_id)- assert "TEST_API_KEY" in resolved- assert resolved["TEST_API_KEY"] == secret_value+ await transform.async_init()++ # Should have called get_secret with the vault ID+ assert any(call[0][0] == vault_id for call in mock_vault.get_secret.call_args_list)+ assert "TEST_API_KEY" in transform.params+ assert transform.params["TEST_API_KEY"] == secret_value- def test_resolve_vault_secret_fallback_to_name(self, transform, mock_vault):+ @pytest.mark.asyncio+ async def test_resolve_vault_secret_fallback_to_name(self, sketch_id, mock_vault):"""Test that if vault ID fails, it falls back to name lookup."""secret_value = "my-api-key-12345"vault_id = str(uuid.uuid4())- # Set params with a vault ID- transform.params = {"TEST_API_KEY": vault_id}+ # Mock vault: ID returns None, name returns secret, optional returns None+ def get_secret_side_effect(key):+ if key == vault_id:+ return None # Vault ID not found+ elif key == "TEST_API_KEY":+ return secret_value # Fallback to name works+ elif key == "OPTIONAL_KEY":+ return None # Will use default+ return None- # First call (by ID) returns None, second call (by name) returns the secret- mock_vault.get_secret.side_effect = [None, secret_value]+ mock_vault.get_secret.side_effect = get_secret_side_effect- resolved = transform.resolve_params()+ # Create transform with vault ID in params+ transform = MockTransform(+ sketch_id=sketch_id,+ scan_id="scan_123",+ vault=mock_vault,+ params={"TEST_API_KEY": vault_id},+ params_schema=MockTransform.get_params_schema(),+ )- # Should try both ID and name- assert mock_vault.get_secret.call_count == 2- assert "TEST_API_KEY" in resolved- assert resolved["TEST_API_KEY"] == secret_value+ await transform.async_init()++ # Should have called get_secret with both vault ID and TEST_API_KEY name+ call_args = [call[0][0] for call in mock_vault.get_secret.call_args_list]+ assert vault_id in call_args+ assert "TEST_API_KEY" in call_args+ assert "TEST_API_KEY" in transform.params+ assert transform.params["TEST_API_KEY"] == secret_valuedef test_resolve_vault_secret_not_found_required(self, transform, mock_vault):- """Test that missing required vault secret logs error."""+ """Test that missing required vault secret raises exception."""mock_vault.get_secret.return_value = None- with patch("flowsint_core.core.transform_base.Logger") as mock_logger:- resolved = transform.resolve_params()+ # Should raise an exception for missing required secret+ with pytest.raises(Exception) as exc_info:+ transform.resolve_params()- # Should log an error for missing required secret- assert mock_logger.error.called- error_message = mock_logger.error.call_args[0][1]["message"]- assert "TEST_API_KEY" in error_message- assert "missing" in error_message.lower()- assert "Vault settings" in error_message+ assert "TEST_API_KEY" in str(exc_info.value)+ assert "missing" in str(exc_info.value).lower()+ assert "Vault settings" in str(exc_info.value)- # Should not be in resolved params- assert "TEST_API_KEY" not in resolved-- def test_resolve_vault_secret_optional_with_default(self, transform, mock_vault):+ @pytest.mark.asyncio+ async def test_resolve_vault_secret_optional_with_default(self, transform, mock_vault):"""Test that optional vault secret uses default if not found."""# Only return secret for TEST_API_KEY, not OPTIONAL_KEYmock_vault.get_secret.side_effect = lambda key: ("test-api-key" if key == "TEST_API_KEY" else None)- resolved = transform.resolve_params()+ await transform.async_init()# Optional key should use default value- assert "OPTIONAL_KEY" in resolved- assert resolved["OPTIONAL_KEY"] == "default_value"+ assert "OPTIONAL_KEY" in transform.params+ assert transform.params["OPTIONAL_KEY"] == "default_value"- def test_resolve_non_vault_param_from_params(self, transform):+ @pytest.mark.asyncio+ async def test_resolve_non_vault_param_from_params(self, mock_vault, sketch_id):"""Test that non-vault params are taken from params."""- transform.params = {"NON_VAULT_PARAM": "custom_value"}+ transform = MockTransform(+ sketch_id=sketch_id,+ scan_id="scan_123",+ vault=mock_vault,+ params={"NON_VAULT_PARAM": "custom_value"},+ params_schema=MockTransform.get_params_schema(),+ )+ mock_vault.get_secret.return_value = "test-api-key"- resolved = transform.resolve_params()+ await transform.async_init()- assert "NON_VAULT_PARAM" in resolved- assert resolved["NON_VAULT_PARAM"] == "custom_value"+ assert "NON_VAULT_PARAM" in transform.params+ assert transform.params["NON_VAULT_PARAM"] == "custom_value"- def test_resolve_non_vault_param_uses_default(self, transform):+ @pytest.mark.asyncio+ async def test_resolve_non_vault_param_uses_default(self, transform, mock_vault):"""Test that non-vault params use default if not provided."""- resolved = transform.resolve_params()+ mock_vault.get_secret.return_value = "test-api-key"- assert "NON_VAULT_PARAM" in resolved- assert resolved["NON_VAULT_PARAM"] == "test"+ await transform.async_init()++ assert "NON_VAULT_PARAM" in transform.params+ assert transform.params["NON_VAULT_PARAM"] == "test"def test_resolve_params_no_vault_instance(self, sketch_id):- """Test that resolve_params works without a vault instance."""+ """Test that resolve_params works without vault (uses defaults for optional params)."""transform = MockTransform(sketch_id=sketch_id,scan_id="scan_123",vault=None, # No vaultparams={},+ params_schema=MockTransform.get_params_schema(),)+ # Without vault, required secrets won't be resolved, optional ones will use defaultsresolved = transform.resolve_params()- # Should use defaults for optional params+ # Optional params should use defaultsassert "OPTIONAL_KEY" in resolvedassert resolved["OPTIONAL_KEY"] == "default_value"assert "NON_VAULT_PARAM" in resolvedassert resolved["NON_VAULT_PARAM"] == "test"+ # Required vault secrets won't be in resolved (no exception when vault is None)+ assert "TEST_API_KEY" not in resolved+class TestGetSecret:"""Tests for get_secret() method."""@@ -211,33 +257,54 @@ class TestGetSecret:assert result == secret_value@pytest.mark.asyncio- async def test_get_secret_not_found_uses_default(self, transform, mock_vault):+ async def test_get_secret_not_found_uses_default(self, mock_vault, sketch_id):"""Test get_secret returns default when secret not found."""- mock_vault.get_secret.return_value = None+ mock_vault.get_secret.return_value = "test-api-key"++ transform = MockTransform(+ sketch_id=sketch_id,+ scan_id="scan_123",+ vault=mock_vault,+ params={},+ params_schema=MockTransform.get_params_schema(),+ )# Run async_init to resolve paramsawait transform.async_init()- # Get the secret with default+ # Get the secret with default (NONEXISTENT_KEY doesn't exist in params)result = transform.get_secret("NONEXISTENT_KEY", "default")assert result == "default"@pytest.mark.asyncio- async def test_get_secret_returns_none_when_not_found_no_default(- self, transform, mock_vault+ async def test_get_secret_returns_default_when_value_is_none(+ self, mock_vault, sketch_id):- """Test get_secret returns None when no default provided."""- mock_vault.get_secret.return_value = None+ """Test get_secret returns default when value is None."""+ mock_vault.get_secret.return_value = "test-api-key"++ transform = MockTransform(+ sketch_id=sketch_id,+ scan_id="scan_123",+ vault=mock_vault,+ params={},+ params_schema=MockTransform.get_params_schema(),+ )# Run async_init to resolve paramsawait transform.async_init()- # Get the secret without default+ # Get a non-existent secret without defaultresult = transform.get_secret("NONEXISTENT_KEY")assert result is None+ # Get a non-existent secret with default+ result = transform.get_secret("NONEXISTENT_KEY", "fallback")++ assert result == "fallback"+class TestAsyncInit:"""Tests for async_init() method."""@@ -263,6 +330,7 @@ class TestAsyncInit:scan_id="scan_123",vault=mock_vault,params={}, # Empty params+ params_schema=MockTransform.get_params_schema(),)secret_value = "my-api-key-12345"@@ -307,7 +375,7 @@ class TestTransformExecuteWithVault:assert transform.params["TEST_API_KEY"] == secret_value@pytest.mark.asyncio- async def test_execute_uses_resolved_secrets_in_scan(self, transform, mock_vault):+ async def test_execute_uses_resolved_secrets_in_scan(self, mock_vault):"""Test that scan can access resolved secrets."""secret_value = "my-api-key-12345"mock_vault.get_secret.return_value = secret_value@@ -323,6 +391,7 @@ class TestTransformExecuteWithVault:scan_id="scan_123",vault=mock_vault,params={},+ params_schema=MockTransform.get_params_schema(),)# Executediff --git a/flowsint-core/tests/vault/test_vault.py b/flowsint-core/tests/vault/test_vault.pyindex 536e66bd..5d31f9b1 100644--- a/flowsint-core/tests/vault/test_vault.py+++ b/flowsint-core/tests/vault/test_vault.py@@ -220,10 +220,12 @@ class TestVaultEncryptionDecryption:def test_invalid_master_key_length(self, vault, monkeypatch):"""Test that invalid master key length raises error."""- # Set an invalid master key (wrong length)- monkeypatch.setenv("MASTER_VAULT_KEY_V1", "base64:invalid")+ import base64+ # Set a valid base64 but wrong length (16 bytes instead of 32)+ short_key = base64.b64encode(b"0" * 16).decode()+ monkeypatch.setenv("MASTER_VAULT_KEY_V1", f"base64:{short_key}")- with pytest.raises(ValueError, match="Master key must be 32 bytes"):+ with pytest.raises(ValueError, match="Master key must be 32 bytes \\(256 bits\\)"):vault._get_master_key()def test_missing_master_key(self, vault, monkeypatch):diff --git a/flowsint-transforms/README.md b/flowsint-transforms/README.mdindex 4b00e8d7..a1483c83 100644--- a/flowsint-transforms/README.md+++ b/flowsint-transforms/README.md@@ -1,4 +1,11 @@# flowsint-transforms+The repository containing open-source transforms for flowsint.+## Testing++```bash+poetry run pytest+```+> ⚠️ 🚧 Work in progress !.diff --git a/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.py b/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.pyindex 75f9b10a..84f85c74 100644--- a/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.py+++ b/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.py@@ -14,8 +14,8 @@ class AsnToCidrsTransform(Transform):"""[ASNMAP] Takes an ASN and returns its corresponding CIDRs."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[ASN]- OutputType = List[CIDR]+ InputType = ASN+ OutputType = CIDRdef __init__(self,@@ -62,9 +62,9 @@ class AsnToCidrsTransform(Transform):def key(cls) -> str:return "number"- async def scan(self, data: InputType) -> OutputType:+ async def scan(self, data: List[InputType]) -> List[OutputType]:"""Find CIDR from ASN using asnmap."""- cidrs: OutputType = []+ cidrs: List[OutputType] = []self._asn_to_cidrs_map = [] # Store mapping for postprocessasnmap = AsnmapTool()@@ -115,7 +115,7 @@ class AsnToCidrsTransform(Transform):return cidrs- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:# Create Neo4j relationships between ASNs and their corresponding CIDRs# Use the mapping from scan if available, else fallback to zipasn_to_cidrs = getattr(self, "_asn_to_cidrs_map", None)diff --git a/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.py b/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.pyindex bf1599e0..c7fde030 100644--- a/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.py+++ b/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.py@@ -12,8 +12,8 @@ class CidrToIpsTransform(Transform):"""[MAPCIDR] Takes a CIDR and returns its corresponding IP addresses."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[CIDR]- OutputType = List[Ip]+ InputType = CIDR+ OutputType = Ipdef __init__(self,@@ -60,9 +60,9 @@ class CidrToIpsTransform(Transform):def category(cls) -> str:return "Cidr"- async def scan(self, data: InputType) -> OutputType:+ async def scan(self, data: List[InputType]) -> List[OutputType]:"""Find IP addresses from CIDR using mapcidr."""- ips: OutputType = []+ ips: List[OutputType] = []self._cidr_to_ips_map = [] # Store mapping for postprocessmapcidr = MapcidrTool()@@ -111,7 +111,7 @@ class CidrToIpsTransform(Transform):return ips- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:# Create Neo4j relationships between CIDRs and their corresponding IPs# Use the mapping from scan if available, else fallback to zipcidr_to_ips = getattr(self, "_cidr_to_ips_map", None)diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.pyindex 4e7459cc..84e29b4d 100644--- a/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py@@ -14,8 +14,8 @@ class CryptoWalletAddressToNFTs(Transform):"""[ETHERSCAN] Resolve NFTs for a wallet address (ETH)."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[CryptoWallet]- OutputType = List[CryptoNFT]+ InputType = CryptoWallet+ OutputType = CryptoNFTdef __init__(self,@@ -73,8 +73,8 @@ class CryptoWalletAddressToNFTs(Transform):def key(cls) -> str:return "address"- async def scan(self, data: InputType) -> OutputType:- results: OutputType = []+ async def scan(self, data: List[InputType]) -> List[OutputType]:+ results: List[OutputType] = []api_key = self.get_secret("ETHERSCAN_API_KEY", os.getenv("ETHERSCAN_API_KEY"))api_url = self.get_params().get("ETHERSCAN_API_URL", "https://api.etherscan.io/v2/api"@@ -118,7 +118,7 @@ class CryptoWalletAddressToNFTs(Transform):)return nfts- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:if not self.neo4j_conn:return resultsdiff --git a/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.pyindex 50689812..00bfc5a0 100644--- a/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py@@ -20,8 +20,8 @@ class CryptoWalletAddressToTransactions(Transform):"""[ETHERSCAN] Resolve transactions for a wallet address (ETH)."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[CryptoWallet]- OutputType = List[CryptoWalletTransaction]+ InputType = CryptoWallet+ OutputType = CryptoWalletTransactiondef __init__(self,@@ -79,8 +79,8 @@ class CryptoWalletAddressToTransactions(Transform):def key(cls) -> str:return "address"- async def scan(self, data: InputType) -> OutputType:- results: OutputType = []+ async def scan(self, data: List[InputType]) -> List[OutputType]:+ results: List[OutputType] = []api_key = self.get_secret("ETHERSCAN_API_KEY", os.getenv("ETHERSCAN_API_KEY"))api_url = self.get_params().get("ETHERSCAN_API_URL", "https://api.etherscan.io/v2/api")for d in data:@@ -173,7 +173,7 @@ class CryptoWalletAddressToTransactions(Transform):)return transactions- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:if not self.neo4j_conn:return resultsdiff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_asn.py b/flowsint-transforms/src/flowsint_transforms/domain/to_asn.pyindex 3463077e..fd2b8abe 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_asn.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_asn.py@@ -15,8 +15,8 @@ class DomainToAsnTransform(Transform):"""[ASNMAP] Takes a domain and returns its corresponding ASN."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[Domain]- OutputType = List[ASN]+ InputType = Domain+ OutputType = ASNdef __init__(self,@@ -63,8 +63,8 @@ class DomainToAsnTransform(Transform):def key(cls) -> str:return "domain"- async def scan(self, data: InputType) -> OutputType:- results: OutputType = []+ async def scan(self, data: List[InputType]) -> List[OutputType]:+ results: List[OutputType] = []asnmap = AsnmapTool()# Retrieve API key from vault or environment@@ -113,8 +113,8 @@ class DomainToAsnTransform(Transform):return resultsdef postprocess(- self, results: OutputType, input_data: InputType = None- ) -> OutputType:+ self, results: List[OutputType], input_data: List[InputType] = None+ ) -> List[OutputType]:# Create Neo4j relationships between domains and their corresponding ASNsif input_data and self.neo4j_conn:for domain, asn in zip(input_data, results):diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_history.py b/flowsint-transforms/src/flowsint_transforms/domain/to_history.pyindex f6d07003..28d65f7d 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_history.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_history.py@@ -21,8 +21,8 @@ class DomainToHistoryTransform(Transform):"""[WHOXY] Takes a domain and returns history infos about it (history, organization, owners, emails, etc.)."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[Domain]- OutputType = List[Domain]+ InputType = Domain+ OutputType = Domaindef __init__(self,@@ -69,9 +69,9 @@ class DomainToHistoryTransform(Transform):def key(cls) -> str:return "domain"- async def scan(self, data: InputType) -> OutputType:+ async def scan(self, data: List[InputType]) -> List[OutputType]:"""Find infos related to domains using whoxy api."""- domains: OutputType = []+ domains: List[OutputType] = []self._extracted_data = [] # Store all extracted data for postprocessself._extracted_individuals = [] # Store extracted individuals for testingself._extracted_organizations = [] # Store extracted organizations for testing@@ -390,7 +390,7 @@ class DomainToHistoryTransform(Transform):},)- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:"""Create Neo4j nodes and relationships from extracted data."""if not self.neo4j_conn:Logger.info(diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py b/flowsint-transforms/src/flowsint_transforms/domain/to_ip.pyindex 55cc885b..e1682a53 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py@@ -9,8 +9,8 @@ from flowsint_types.ip import Ipclass ResolveTransform(Transform):"""Resolve domain names to IP addresses."""- InputType = List[Domain]- OutputType = List[Ip]+ InputType = Domain+ OutputType = Ip@classmethoddef name(cls) -> str:@@ -29,8 +29,8 @@ class ResolveTransform(Transform):"""Return formatted markdown documentation for the domain resolver transform."""return ""- async def scan(self, data: InputType) -> OutputType:- results: OutputType = []+ async def scan(self, data: List[InputType]) -> List[OutputType]:+ results: List[OutputType] = []for d in data:try:ip = socket.gethostbyname(d.domain)@@ -43,7 +43,7 @@ class ResolveTransform(Transform):continuereturn results- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:for domain_obj, ip_obj in zip(original_input, results):self.create_node(domain_obj)self.create_node(ip_obj)diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py b/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.pyindex 73fee128..5edcf516 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py@@ -8,8 +8,8 @@ from flowsint_core.core.logger import Loggerclass DomainToRootDomain(Transform):"""Subdomain to root domain."""- InputType = List[Domain]- OutputType = List[Domain]+ InputType = Domain+ OutputType = Domaindef __init__(self, *args, **kwargs):super().__init__(*args, **kwargs)@@ -28,8 +28,8 @@ class DomainToRootDomain(Transform):def key(cls) -> str:return "domain"- async def scan(self, data: InputType) -> OutputType:- results: OutputType = []+ async def scan(self, data: List[InputType]) -> List[OutputType]:+ results: List[OutputType] = []self.domain_root_mapping = [] # Reset mappingfor domain in data:@@ -51,7 +51,7 @@ class DomainToRootDomain(Transform):return results- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:# Use the mapping we created during scan to create relationshipsfor original_domain, root_domain in self.domain_root_mapping:if not self.neo4j_conn:diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py b/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.pyindex 6f0b6013..85c9ec91 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py@@ -12,8 +12,8 @@ class SubdomainTransform(Transform):"""Transform to find subdomains associated with a domain."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[Domain]- OutputType = List[Domain]+ InputType = Domain+ OutputType = Domain@classmethoddef name(cls) -> str:@@ -27,9 +27,9 @@ class SubdomainTransform(Transform):def key(cls) -> str:return "domain"- async def scan(self, data: InputType) -> OutputType:+ async def scan(self, data: List[InputType]) -> List[OutputType]:"""Find subdomains using subfinder (Docker) or fallback to crt.sh."""- domains: OutputType = []+ domains: List[OutputType] = []for md in data:d = Domain(domain=md.domain)@@ -88,8 +88,8 @@ class SubdomainTransform(Transform):)return subdomains- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:- output: OutputType = []+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:+ output: List[OutputType] = []for domain_obj in results:if not self.neo4j_conn:continuediff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_website.py b/flowsint-transforms/src/flowsint_transforms/domain/to_website.pyindex a90432bf..e26614ce 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_website.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_website.py@@ -11,8 +11,8 @@ class DomainToWebsiteTransform(Transform):"""From domain to website."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[Domain]- OutputType = List[Website]+ InputType = Domain+ OutputType = Website@classmethoddef name(cls) -> str:@@ -26,8 +26,8 @@ class DomainToWebsiteTransform(Transform):def key(cls) -> str:return "domain"- async def scan(self, data: InputType) -> OutputType:- results: OutputType = []+ async def scan(self, data: List[InputType]) -> List[OutputType]:+ results: List[OutputType] = []for domain in data:try:# Try HTTPS first@@ -67,7 +67,7 @@ class DomainToWebsiteTransform(Transform):return results- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:for website in results:# Log each redirect stepif website.redirects:diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py b/flowsint-transforms/src/flowsint_transforms/domain/to_whois.pyindex e00128cb..41e0dfd7 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py@@ -12,8 +12,8 @@ class WhoisTransform(Transform):"""Scan for WHOIS information of a domain."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[Domain]- OutputType = List[Whois]+ InputType = Domain+ OutputType = Whois@classmethoddef name(cls) -> str:@@ -27,8 +27,8 @@ class WhoisTransform(Transform):def key(cls) -> str:return "domain"- async def scan(self, data: InputType) -> OutputType:- results: OutputType = []+ async def scan(self, data: List[InputType]) -> List[OutputType]:+ results: List[OutputType] = []for domain in data:try:whois_info = whois.whois(domain.domain)@@ -107,66 +107,25 @@ class WhoisTransform(Transform):return results- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:for whois_obj in results:if not self.neo4j_conn:continue-# Create domain nodeself.create_node(whois_obj.domain)-- # Create whois node with custom key (only primitive fields)- whois_key = f"{whois_obj.domain.domain}_{self.sketch_id}"- whois_props = {- "registry_domain_id": whois_obj.registry_domain_id,- "registrar": whois_obj.registrar,- "city": whois_obj.city,- "country": whois_obj.country,- "creation_date": whois_obj.creation_date,- "expiration_date": whois_obj.expiration_date,- "label": whois_obj.label,- }- self.create_node("whois", "whois_id", whois_key, **whois_props)-+ # Create whois node (nested Pydantic objects are automatically skipped)+ self.create_node(whois_obj)# Create relationship between domain and whois- domain_obj = Domain(domain=whois_obj.domain.domain)- whois_node = Whois(- domain=domain_obj,- registry_domain_id=whois_obj.registry_domain_id,- registrar=whois_obj.registrar,- city=whois_obj.city,- country=whois_obj.country,- creation_date=whois_obj.creation_date,- expiration_date=whois_obj.expiration_date,- )- self.create_relationship(domain_obj, whois_node, "HAS_WHOIS")-- # Create organization node if organization information is available+ self.create_relationship(whois_obj.domain, whois_obj, "HAS_WHOIS")+ # Create organization node if availableif whois_obj.organization:self.create_node(whois_obj.organization)-- # Create relationship between organization and domain- domain_obj2 = Domain(domain=whois_obj.domain.domain)- self.create_relationship(whois_obj.organization, domain_obj2, "HAS_DOMAIN")-- self.log_graph_message(- f"{whois_obj.domain.domain} -> {whois_obj.organization.name} (organization)"- )-- # Create email node if email information is available+ self.create_relationship(whois_obj.organization, whois_obj.domain, "HAS_DOMAIN")+ # Create email node if availableif whois_obj.email:self.create_node(whois_obj.email)- whois_node2 = Whois(- domain=Domain(domain=whois_obj.domain.domain),- registry_domain_id=whois_obj.registry_domain_id,- registrar=whois_obj.registrar,- city=whois_obj.city,- country=whois_obj.country,- creation_date=whois_obj.creation_date,- expiration_date=whois_obj.expiration_date,- )- self.create_relationship(whois_node2, whois_obj.email, "REGISTERED_BY")-+ self.create_relationship(whois_obj, whois_obj.email, "REGISTERED_BY")+ # Log messageself.log_graph_message(f"WHOIS for {whois_obj.domain.domain} -> registry_id: {whois_obj.registry_domain_id} registrar: {whois_obj.registrar} org: {whois_obj.organization.name if whois_obj.organization else None} city: {whois_obj.city} country: {whois_obj.country} creation_date: {whois_obj.creation_date} expiration_date: {whois_obj.expiration_date}")diff --git a/flowsint-transforms/src/flowsint_transforms/email/to_domains.py b/flowsint-transforms/src/flowsint_transforms/email/to_domains.pyindex 5a66a2c3..252bcd3c 100644--- a/flowsint-transforms/src/flowsint_transforms/email/to_domains.py+++ b/flowsint-transforms/src/flowsint_transforms/email/to_domains.py@@ -21,8 +21,8 @@ class EmailToDomainsTransform(Transform):"""[WHOXY] Takes an email and returns the domains it registered."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[Email]- OutputType = List[Domain]+ InputType = Email+ OutputType = Domaindef __init__(self,@@ -69,9 +69,9 @@ class EmailToDomainsTransform(Transform):def key(cls) -> str:return "email"- async def scan(self, data: InputType) -> OutputType:+ async def scan(self, data: List[InputType]) -> List[OutputType]:"""Find domains related to emails using whoxy api."""- domains: OutputType = []+ domains: List[OutputType] = []self._extracted_data = [] # Store all extracted data for postprocessapi_key = self.get_secret("WHOXY_API_KEY", os.getenv("WHOXY_API_KEY"))@@ -215,7 +215,7 @@ class EmailToDomainsTransform(Transform):address=address, city=city, zip=zip_code, country=country)- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:"""Create Neo4j nodes and relationships from extracted data."""if not self.neo4j_conn:return resultsdiff --git a/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py b/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.pyindex 2b9299a3..7bec2d14 100644--- a/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py+++ b/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py@@ -10,8 +10,8 @@ from flowsint_types.gravatar import Gravatarclass EmailToGravatarTransform(Transform):"""From md5 hash of email to gravatar."""- InputType = List[Email]- OutputType = List[Gravatar]+ InputType = Email+ OutputType = Gravatar@classmethoddef name(cls) -> str:@@ -25,8 +25,8 @@ class EmailToGravatarTransform(Transform):def key(cls) -> str:return "email"- async def scan(self, data: InputType) -> OutputType:- results: OutputType = []+ async def scan(self, data: List[InputType]) -> List[OutputType]:+ results: List[OutputType] = []for email in data:try:@@ -82,7 +82,7 @@ class EmailToGravatarTransform(Transform):return results- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:for email_obj, gravatar_obj in zip(original_input, results):if not self.neo4j_conn:continuediff --git a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.pyindex acd3b7f0..04ed6516 100644--- a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py+++ b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py@@ -18,8 +18,8 @@ HIBP_API_KEY = os.getenv("HIBP_API_KEY")class EmailToBreachesTransform(Transform):"""[HIBPWNED] Get the breaches the email might be invovled in."""- InputType = List[Email]- OutputType = List[tuple] # List of (email, breach) tuples+ InputType = Email+ OutputType = tuple # (email, breach) tupledef __init__(self,@@ -73,8 +73,8 @@ class EmailToBreachesTransform(Transform):},]- async def scan(self, data: InputType) -> OutputType:- results: OutputType = []+ async def scan(self, data: List[InputType]) -> List[OutputType]:+ results: List[OutputType] = []api_key = self.get_secret("HIBP_API_KEY", os.getenv("HIBP_API_KEY"))api_url = self.get_params().get("HIBP_API_URL", "https://haveibeenpwned.com/api/v3/breachedaccount/")headers = {"hibp-api-key": api_key, "User-Agent": "FlowsInt-Transform"}@@ -131,7 +131,7 @@ class EmailToBreachesTransform(Transform):return results- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:# Create email nodes firstfor email_obj in original_input:if not self.neo4j_conn:diff --git a/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py b/flowsint-transforms/src/flowsint_transforms/individual/to_domains.pyindex 235a140b..4aacad62 100644--- a/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py+++ b/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py@@ -20,8 +20,8 @@ class IndividualToDomainsTransform(Transform):"""[WHOXY] Takes an individual and returns the domains it registered."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[Individual]- OutputType = List[Domain]+ InputType = Individual+ OutputType = Domaindef __init__(self,@@ -68,10 +68,10 @@ class IndividualToDomainsTransform(Transform):def key(cls) -> str:return "full_name"- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:+ def preprocess(self, data: Union[List[str], List[dict], List[InputType]]) -> List[InputType]:if not isinstance(data, list):raise ValueError(f"Expected list input, got {type(data).__name__}")- cleaned: InputType = []+ cleaned: List[InputType] = []for item in data:if isinstance(item, str):parts = item.strip().split()@@ -98,9 +98,9 @@ class IndividualToDomainsTransform(Transform):)return cleaned- async def scan(self, data: InputType) -> OutputType:+ async def scan(self, data: List[InputType]) -> List[OutputType]:"""Find domains related to individuals using whoxy api."""- domains: OutputType = []+ domains: List[OutputType] = []self._extracted_data = [] # Store all extracted data for postprocessapi_key = self.get_secret("WHOXY_API_KEY", os.getenv("WHOXY_API_KEY"))@@ -247,7 +247,7 @@ class IndividualToDomainsTransform(Transform):return Location(address=address, city=city, zip=zip_code, country=country)- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:"""Create Neo4j nodes and relationships from extracted data."""if not self.neo4j_conn:return resultsdiff --git a/flowsint-transforms/src/flowsint_transforms/individual/to_org.py b/flowsint-transforms/src/flowsint_transforms/individual/to_org.pyindex c25c88de..b7aee213 100644--- a/flowsint-transforms/src/flowsint_transforms/individual/to_org.py+++ b/flowsint-transforms/src/flowsint_transforms/individual/to_org.py@@ -11,8 +11,8 @@ class IndividualToOrgTransform(Transform):"""[SIRENE] Find organization from a person with data from SIRENE (France only)."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[Individual]- OutputType = List[Organization]+ InputType = Individual+ OutputType = Organization@classmethoddef name(cls) -> str:@@ -26,10 +26,10 @@ class IndividualToOrgTransform(Transform):def key(cls) -> str:return "fullname"- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:+ def preprocess(self, data: Union[List[str], List[dict], List[InputType]]) -> List[InputType]:if not isinstance(data, list):raise ValueError(f"Expected list input, got {type(data).__name__}")- cleaned: InputType = []+ cleaned: List[InputType] = []for item in data:if isinstance(item, str):parts = item.strip().split()@@ -56,9 +56,9 @@ class IndividualToOrgTransform(Transform):)return cleaned- async def scan(self, data: InputType) -> OutputType:+ async def scan(self, data: List[InputType]) -> List[OutputType]:- results: OutputType = []+ results: List[OutputType] = []for individual in data:try:sirene = SireneTool()@@ -278,7 +278,7 @@ class IndividualToOrgTransform(Transform):)return None- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:if not self.neo4j_conn:return resultsdiff --git a/flowsint-transforms/src/flowsint_transforms/ip/to_asn.py b/flowsint-transforms/src/flowsint_transforms/ip/to_asn.pyindex 4024b3e7..1cb9af61 100644--- a/flowsint-transforms/src/flowsint_transforms/ip/to_asn.py+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_asn.py@@ -14,8 +14,8 @@ class IpToAsnTransform(Transform):"""[ASNMAP] Takes an IP address and returns its corresponding ASN."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[Ip]- OutputType = List[ASN]+ InputType = Ip+ OutputType = ASNdef __init__(self,@@ -62,8 +62,8 @@ class IpToAsnTransform(Transform):def key(cls) -> str:return "address"- async def scan(self, data: InputType) -> OutputType:- results: OutputType = []+ async def scan(self, data: List[InputType]) -> List[OutputType]:+ results: List[OutputType] = []asnmap = AsnmapTool()# Retrieve API key from vault or environment@@ -108,8 +108,8 @@ class IpToAsnTransform(Transform):return resultsdef postprocess(- self, results: OutputType, input_data: InputType = None- ) -> OutputType:+ self, results: List[OutputType], input_data: List[InputType] = None+ ) -> List[OutputType]:# Create Neo4j relationships between IPs and their corresponding ASNsif input_data and self.neo4j_conn:for ip, asn in zip(input_data, results):diff --git a/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py b/flowsint-transforms/src/flowsint_transforms/ip/to_domain.pyindex 3de35ab4..e31ebf49 100644--- a/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py@@ -1,22 +1,17 @@-import reimport socketimport requests-from typing import List, Union+from typing import Listfrom flowsint_core.core.logger import Loggerfrom flowsint_core.core.transform_base import Transformfrom flowsint_types.domain import Domainfrom flowsint_types.ip import Ip-from flowsint_core.utils import is_valid_ip--PTR_BLACKLIST = re.compile(r"^ip\d+\.ip-\d+-\d+-\d+-\d+\.")class ReverseResolveTransform(Transform):"""Resolve IP addresses to domain names using PTR, Certificate Transparency and optional API calls."""- # Define types as class attributes - base class handles schema generation automatically- InputType = List[Ip]- OutputType = List[Domain]+ InputType = Ip+ OutputType = Domain@classmethoddef name(cls) -> str:@@ -30,28 +25,26 @@ class ReverseResolveTransform(Transform):def key(cls) -> str:return "address"- async def scan(self, data: InputType) -> OutputType:- results: OutputType = []+ async def scan(self, data: List[InputType]) -> List[OutputType]:+ results: List[OutputType] = []for ip in data:try:- # Try PTR lookuptry:hostname = socket.gethostbyaddr(ip.address)[0]- if hostname and not PTR_BLACKLIST.match(hostname):+ if hostname:domain = Domain(domain=hostname)results.append(domain)continueexcept socket.herror:pass- # Try Certificate Transparency logstry:ct_url = f"https://crt.sh/?q={ip.address}&output=json"response = requests.get(ct_url, timeout=10)if response.status_code == 200:ct_data = response.json()- for entry in ct_data[:5]: # Limit to first 5 results+ for entry in ct_data[:15]:name_value = entry.get("name_value", "")if name_value and name_value != ip.address:domain = Domain(domain=name_value)@@ -69,12 +62,9 @@ class ReverseResolveTransform(Transform):return results- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:- # Create nodes and relationships for each resolved domain+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:for ip_obj in original_input:- # Create IP nodeself.create_node(ip_obj)- # Create domain nodes and relationships for each resolved domainfor domain_obj in results:self.create_node(domain_obj)self.create_relationship(ip_obj, domain_obj, "REVERSE_RESOLVES_TO")diff --git a/flowsint-transforms/src/flowsint_transforms/ip/to_infos.py b/flowsint-transforms/src/flowsint_transforms/ip/to_infos.pyindex 6c5dd14a..5b1d3d40 100644--- a/flowsint-transforms/src/flowsint_transforms/ip/to_infos.py+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_infos.py@@ -8,8 +8,8 @@ class IpToInfosTransform(Transform):"""[ip-api.com] Get information data for IP addresses."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[Ip]- OutputType = List[Ip]+ InputType = Ip+ OutputType = Ip@classmethoddef name(cls) -> str:@@ -23,8 +23,8 @@ class IpToInfosTransform(Transform):def key(cls) -> str:return "address"- async def scan(self, data: InputType) -> OutputType:- results: OutputType = []+ async def scan(self, data: List[InputType]) -> List[OutputType]:+ results: List[OutputType] = []for ip in data:try:geo_data = self.get_location_data(ip.address)@@ -39,7 +39,7 @@ class IpToInfosTransform(Transform):print(f"Error geolocating {ip.address}: {e}")return results- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:"""Update IP nodes in Neo4j with geolocation information."""if self.neo4j_conn:for ip in results:diff --git a/flowsint-transforms/src/flowsint_transforms/ip/to_ports.py b/flowsint-transforms/src/flowsint_transforms/ip/to_ports.pyindex 2b9d2c33..6cb341d5 100644--- a/flowsint-transforms/src/flowsint_transforms/ip/to_ports.py+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_ports.py@@ -12,8 +12,8 @@ class IpToPortsTransform(Transform):"""[NAABU] Performs port scanning on IP addresses to discover open ports and services."""# Define types as class attributes- InputType = List[Ip]- OutputType = List[Port]+ InputType = Ip+ OutputType = Portdef __init__(self,@@ -107,8 +107,8 @@ class IpToPortsTransform(Transform):def key(cls) -> str:return "address"- async def scan(self, data: InputType) -> OutputType:- results: OutputType = []+ async def scan(self, data: List[InputType]) -> List[OutputType]:+ results: List[OutputType] = []naabu = NaabuTool()# Get parameters from transform config@@ -189,8 +189,8 @@ class IpToPortsTransform(Transform):return resultsdef postprocess(- self, results: OutputType, input_data: InputType = None- ) -> OutputType:+ self, results: List[OutputType], input_data: List[InputType] = None+ ) -> List[OutputType]:"""Create Neo4j nodes for ports and relationships with IP addresses"""if self.neo4j_conn and results:for port in results:diff --git a/flowsint-transforms/src/flowsint_transforms/n8n/connector.py b/flowsint-transforms/src/flowsint_transforms/n8n/connector.pyindex 22289c1f..7c0ba3b7 100644--- a/flowsint-transforms/src/flowsint_transforms/n8n/connector.py+++ b/flowsint-transforms/src/flowsint_transforms/n8n/connector.py@@ -12,8 +12,8 @@ class N8nConnector(Transform):"""# Define types as class attributes - base class handles schema generation automatically- InputType = List[Any]- OutputType = List[Any]+ InputType = Any+ OutputType = Anydef __init__(self,@@ -256,7 +256,7 @@ Check Flowsint logs for detailed debugging information.},]- async def scan(self, values: InputType) -> OutputType:+ async def scan(self, values: List[InputType]) -> List[OutputType]:params = self.get_params()url = params["webhook_url"]Logger.info(self.sketch_id, {"message": f"n8n connector url: {url}"})@@ -345,7 +345,7 @@ Check Flowsint logs for detailed debugging information.# Re-raise the exception so the caller knows something went wrongraise- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:Logger.success(self.sketch_id, {"message": f"n8n connector results: {json.dumps(results)}"})diff --git a/flowsint-transforms/src/flowsint_transforms/organization/to_asn.py b/flowsint-transforms/src/flowsint_transforms/organization/to_asn.pyindex 645f4ebb..b1eb8986 100644--- a/flowsint-transforms/src/flowsint_transforms/organization/to_asn.py+++ b/flowsint-transforms/src/flowsint_transforms/organization/to_asn.py@@ -12,8 +12,8 @@ class OrgToAsnTransform(Transform):"""Takes an organization and returns its corresponding ASN."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[Organization]- OutputType = List[ASN]+ InputType = Organization+ OutputType = ASNdef __init__(self,@@ -60,9 +60,9 @@ class OrgToAsnTransform(Transform):def key(cls) -> str:return "name"- async def scan(self, data: InputType) -> OutputType:+ async def scan(self, data: List[InputType]) -> List[OutputType]:"""Find ASN information for organizations using asnmap."""- results: OutputType = []+ results: List[OutputType] = []asnmap = AsnmapTool()# Retrieve API key from vault or environment@@ -106,7 +106,7 @@ class OrgToAsnTransform(Transform):return results- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:# Create Neo4j relationships between organizations and their corresponding ASNsfor input_org, result_asn in zip(original_input, results):# Skip if no valid ASN was founddiff --git a/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py b/flowsint-transforms/src/flowsint_transforms/organization/to_domains.pyindex d0c7a205..bd396bce 100644--- a/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py+++ b/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py@@ -20,8 +20,8 @@ class OrgToDomainsTransform(Transform):"""[WHOXY] Takes an organization and returns the domains it registered."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[Organization]- OutputType = List[Domain]+ InputType = Organization+ OutputType = Domaindef __init__(self,@@ -68,9 +68,9 @@ class OrgToDomainsTransform(Transform):def key(cls) -> str:return "name"- async def scan(self, data: InputType) -> OutputType:+ async def scan(self, data: List[InputType]) -> List[OutputType]:"""Find domains related to organizations using whoxy api."""- domains: OutputType = []+ domains: List[OutputType] = []self._extracted_data = [] # Store all extracted data for postprocessself._extracted_individuals = [] # Store extracted individuals for testingself._extracted_organizations = [] # Store extracted organizations for testing@@ -386,7 +386,7 @@ class OrgToDomainsTransform(Transform):},)- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:"""Create Neo4j nodes and relationships from extracted data."""if not self.neo4j_conn:Logger.info(diff --git a/flowsint-transforms/src/flowsint_transforms/organization/to_infos.py b/flowsint-transforms/src/flowsint_transforms/organization/to_infos.pyindex bdf72d46..a216aa05 100644--- a/flowsint-transforms/src/flowsint_transforms/organization/to_infos.py+++ b/flowsint-transforms/src/flowsint_transforms/organization/to_infos.py@@ -9,8 +9,8 @@ class OrgToInfosTransform(Transform):"""Enrich Organization with data from SIRENE (France only)."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[Organization]- OutputType = List[Organization]+ InputType = Organization+ OutputType = Organization@classmethoddef name(cls) -> str:@@ -24,9 +24,9 @@ class OrgToInfosTransform(Transform):def key(cls) -> str:return "name"- async def scan(self, data: InputType) -> OutputType:+ async def scan(self, data: List[InputType]) -> List[OutputType]:- results: OutputType = []+ results: List[OutputType] = []for org in data:try:sirene = SireneTool()@@ -246,7 +246,7 @@ class OrgToInfosTransform(Transform):)return None- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:if not self.neo4j_conn:return resultsdiff --git a/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py b/flowsint-transforms/src/flowsint_transforms/phone/to_infos.pyindex 202b4552..dc54931e 100644--- a/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py+++ b/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py@@ -10,8 +10,8 @@ import httpxclass IgnorantTransform(Transform):# Define types as class attributes - base class handles schema generation automatically- InputType = List[Phone] # Phone objects- OutputType = List[Dict[str, Any]] # Results as dictionaries+ InputType = Phone # Phone objects+ OutputType = Dict[str, Any] # Results as dictionaries@classmethoddef name(cls) -> str:@@ -25,11 +25,11 @@ class IgnorantTransform(Transform):def key(cls) -> str:return "number"- async def scan(self, data: InputType) -> OutputType:+ async def scan(self, data: List[InputType]) -> List[OutputType]:"""Performs the Ignorant search for each specified phone number."""- results: OutputType = []+ results: List[OutputType] = []for phone_obj in data:try:cleaned_phone = is_valid_number(phone_obj.number)@@ -77,7 +77,7 @@ class IgnorantTransform(Transform):except Exception as e:return {"number": phone, "error": f"Error in Ignorant research: {str(e)}"}- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:"""Create Neo4j relationships for found phone accounts."""diff --git a/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py b/flowsint-transforms/src/flowsint_transforms/social/to_maigret.pyindex 67af127a..c22d3b18 100644--- a/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py+++ b/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py@@ -14,8 +14,8 @@ class MaigretTransform(Transform):"""[MAIGRET] Scans usernames for associated social accounts using Maigret."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[Username]- OutputType = List[SocialAccount]+ InputType = Username+ OutputType = SocialAccount@classmethoddef name(cls) -> str:@@ -108,8 +108,8 @@ class MaigretTransform(Transform):return results- async def scan(self, data: InputType) -> OutputType:- results: OutputType = []+ async def scan(self, data: List[InputType]) -> List[OutputType]:+ results: List[OutputType] = []for profile in data:if not profile.value:continue@@ -118,7 +118,7 @@ class MaigretTransform(Transform):results.extend(parsed)return results- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:if not self.neo4j_conn:return resultsdiff --git a/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py b/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.pyindex a2b4aa8b..17d58a84 100644--- a/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py+++ b/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py@@ -11,8 +11,8 @@ class SherlockTransform(Transform):"""[SHERLOCK] Scans the usernames for associated social accounts using Sherlock."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[Username]- OutputType = List[SocialAccount]+ InputType = Username+ OutputType = SocialAccount@classmethoddef name(cls) -> str:@@ -26,9 +26,9 @@ class SherlockTransform(Transform):def key(cls) -> str:return "username"- async def scan(self, data: InputType) -> OutputType:+ async def scan(self, data: List[InputType]) -> List[OutputType]:"""Performs the scan using Sherlock on the list of usernames."""- results: OutputType = []+ results: List[OutputType] = []for username in data:output_file = Path(f"/tmp/sherlock_{username.value}.txt")@@ -88,7 +88,7 @@ class SherlockTransform(Transform):return results- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:"""Create Neo4j relationships for found social accounts."""if not self.neo4j_conn:return resultsdiff --git a/flowsint-transforms/src/flowsint_transforms/website/to_crawler.py b/flowsint-transforms/src/flowsint_transforms/website/to_crawler.pyindex 72ffab1f..4c7a9f4d 100644--- a/flowsint-transforms/src/flowsint_transforms/website/to_crawler.py+++ b/flowsint-transforms/src/flowsint_transforms/website/to_crawler.py@@ -19,8 +19,8 @@ class WebsiteToCrawler(Transform):"""From website to crawler."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[Website]- OutputType = List[ReturnType] # Simplified output type+ InputType = Website+ OutputType = ReturnType # Simplified output type@classmethoddef name(cls) -> str:@@ -43,7 +43,7 @@ class WebsiteToCrawler(Transform):except Exception:return False- async def scan(self, data: InputType) -> OutputType:+ async def scan(self, data: List[InputType]) -> List[OutputType]:"""Crawl websites to extract emails and phone numbers."""results = []@@ -134,7 +134,7 @@ class WebsiteToCrawler(Transform):return results- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:# Create Neo4j relationships between websites and their corresponding emails and phonesfor input_website, result in zip(original_input, results):website_url = str(input_website.url)diff --git a/flowsint-transforms/src/flowsint_transforms/website/to_domain.py b/flowsint-transforms/src/flowsint_transforms/website/to_domain.pyindex ff0eda28..37ef9c40 100644--- a/flowsint-transforms/src/flowsint_transforms/website/to_domain.py+++ b/flowsint-transforms/src/flowsint_transforms/website/to_domain.py@@ -10,8 +10,8 @@ class WebsiteToDomainTransform(Transform):"""From website to domain."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[Website]- OutputType = List[Domain]+ InputType = Website+ OutputType = Domain@classmethoddef name(cls) -> str:@@ -25,8 +25,8 @@ class WebsiteToDomainTransform(Transform):def key(cls) -> str:return "website"- async def scan(self, data: InputType) -> OutputType:- results: OutputType = []+ async def scan(self, data: List[InputType]) -> List[OutputType]:+ results: List[OutputType] = []for website in data:try:parsed_url = urlparse(str(website.url))@@ -56,8 +56,8 @@ class WebsiteToDomainTransform(Transform):return resultsdef postprocess(- self, results: OutputType, input_data: InputType = None- ) -> OutputType:+ self, results: List[OutputType], input_data: List[InputType] = None+ ) -> List[OutputType]:# Create Neo4j relationships between websites and their corresponding domainsif input_data and self.neo4j_conn:for input_website, result in zip(input_data, results):diff --git a/flowsint-transforms/src/flowsint_transforms/website/to_links.py b/flowsint-transforms/src/flowsint_transforms/website/to_links.pyindex 77164d7e..94cb29fb 100644--- a/flowsint-transforms/src/flowsint_transforms/website/to_links.py+++ b/flowsint-transforms/src/flowsint_transforms/website/to_links.py@@ -11,8 +11,8 @@ class WebsiteToLinks(Transform):"""From website to spread crawler that extracts domains and internal/external links."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[Website]- OutputType = List[Website]+ InputType = Website+ OutputType = Website@classmethoddef name(cls) -> str:@@ -34,7 +34,7 @@ class WebsiteToLinks(Transform):except Exception:return ""- async def scan(self, data: InputType) -> OutputType:+ async def scan(self, data: List[InputType]) -> List[OutputType]:"""Crawl websites using reconspread to extract internal and external links."""results = []@@ -202,7 +202,7 @@ class WebsiteToLinks(Transform):return results- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:# Neo4j nodes and relationships are created in real-time during the callback# No additional processing needed herereturn resultsdiff --git a/flowsint-transforms/src/flowsint_transforms/website/to_text.py b/flowsint-transforms/src/flowsint_transforms/website/to_text.pyindex ae4918ab..91292346 100644--- a/flowsint-transforms/src/flowsint_transforms/website/to_text.py+++ b/flowsint-transforms/src/flowsint_transforms/website/to_text.py@@ -9,8 +9,8 @@ from bs4 import BeautifulSoupclass WebsiteToText(Transform):"""Extracts the texts in a webpage."""- InputType = List[Website]- OutputType = List[Phrase]+ InputType = Website+ OutputType = Phrase@classmethoddef name(cls) -> str:@@ -24,8 +24,8 @@ class WebsiteToText(Transform):def key(cls) -> str:return "website"- async def scan(self, data: InputType) -> OutputType:- results: OutputType = []+ async def scan(self, data: List[InputType]) -> List[OutputType]:+ results: List[OutputType] = []for website in data:text_data = self._extract_text(website.url)if text_data:@@ -47,7 +47,7 @@ class WebsiteToText(Transform):print(f"An error occurred: {e}")return None- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:# Create Neo4j relationships between websites and their corresponding phrasesfor input_website, result in zip(original_input, results):website_url = str(input_website.url)diff --git a/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py b/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.pyindex 338b4072..92512ab5 100644--- a/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py+++ b/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py@@ -12,8 +12,8 @@ class WebsiteToWebtrackersTransform(Transform):"""From website to webtrackers."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[Website]- OutputType = List[WebTracker]+ InputType = Website+ OutputType = WebTrackerdef __init__(self,@@ -39,8 +39,8 @@ class WebsiteToWebtrackersTransform(Transform):def key(cls) -> str:return "website"- async def scan(self, data: InputType) -> OutputType:- results: OutputType = []+ async def scan(self, data: List[InputType]) -> List[OutputType]:+ results: List[OutputType] = []for website in data:try:@@ -70,7 +70,7 @@ class WebsiteToWebtrackersTransform(Transform):return results- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:# Create Neo4j relationships between websites and their corresponding trackersif self.neo4j_conn:# Group trackers by website using the mapping we created during scandiff --git a/flowsint-transforms/tests/test_vault_integration.py b/flowsint-transforms/tests/test_vault_integration.pyindex 05ff283d..c332e6c5 100644--- a/flowsint-transforms/tests/test_vault_integration.py+++ b/flowsint-transforms/tests/test_vault_integration.py@@ -227,44 +227,37 @@ class TestMissingRequiredVaultSecret:"""Tests for error handling when required vault secrets are missing."""@pytest.mark.asyncio- async def test_missing_required_secret_logs_error(self, mock_vault, sketch_id):- """Test that missing required secret logs an error message."""+ async def test_missing_required_secret_raises_exception(self, mock_vault, sketch_id):+ """Test that missing required secret raises an exception."""# Vault returns None (secret not found)mock_vault.get_secret.return_value = None- from unittest.mock import patch-- with patch("flowsint_core.core.transform_base.Logger") as mock_logger:- transform = DomainToHistoryTransform(- sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}- )+ transform = DomainToHistoryTransform(+ sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}+ )+ # async_init should raise an exception when required secret is missing+ with pytest.raises(Exception) as exc_info:await transform.async_init()- # Verify error was logged- assert mock_logger.error.called- error_message = mock_logger.error.call_args[0][1]["message"]- assert "WHOXY_API_KEY" in error_message- assert "missing" in error_message.lower()- assert "Vault settings" in error_message+ # Verify the exception message contains the secret name+ assert "WHOXY_API_KEY" in str(exc_info.value)+ assert "missing" in str(exc_info.value).lower()+ assert "Vault settings" in str(exc_info.value)@pytest.mark.asyncio- async def test_transform_can_still_run_with_fallback(self, mock_vault, sketch_id):- """Test that transform can use environment variable fallback if vault fails."""- import os- from unittest.mock import patch-- # Vault returns None- mock_vault.get_secret.return_value = None+ async def test_transform_with_provided_secret_succeeds(self, mock_vault, sketch_id):+ """Test that transform succeeds when required secret is provided."""+ # Vault returns the secret+ api_key = "whoxy-api-key-12345"+ mock_vault.get_secret.return_value = api_key- # Mock environment variable- with patch.dict(os.environ, {"WHOXY_API_KEY": "fallback-key"}):- transform = DomainToHistoryTransform(- sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}- )+ transform = DomainToHistoryTransform(+ sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}+ )- await transform.async_init()+ # async_init should succeed+ await transform.async_init()- # Should be able to get the fallback value- api_key = transform.get_secret("WHOXY_API_KEY", os.getenv("WHOXY_API_KEY"))- assert api_key == "fallback-key"+ # Secret should be accessible+ assert transform.get_secret("WHOXY_API_KEY") == api_keydiff --git a/flowsint-types/README.md b/flowsint-types/README.mdindex ae01189b..b75937f9 100644--- a/flowsint-types/README.md+++ b/flowsint-types/README.md@@ -2,4 +2,10 @@The types used in the flowsint transforms and core for flowsint.+## Testing++```bash+poetry run pytest+```+> ⚠️ 🚧 Work in progress !.diff --git a/flowsint-types/tests/__init__.py b/flowsint-types/tests/__init__.pynew file mode 100644index 00000000..846e4770--- /dev/null+++ b/flowsint-types/tests/__init__.py@@ -0,0 +1 @@+# Tests for flowsint-typesdiff --git a/flowsint-types/tests/individual.py b/flowsint-types/tests/individual.pydeleted file mode 100644index cfe81bdc..00000000--- a/flowsint-types/tests/individual.py+++ /dev/null@@ -1,15 +0,0 @@-from flowsint_types.individual import Individual-import pytest---def test_valid_individual():- indivudual = Individual(first_name="John", last_name="Doe")- assert indivudual.first_name == "John"- assert indivudual.last_name == "Doe"- assert indivudual.label == "John Doe"---def test_invalid_individual():- with pytest.raises(Exception) as e_info:- Individual(name="John Doe")- assert "validation errors for Individual" in str(e_info.value)diff --git a/flowsint-types/tests/domain.py b/flowsint-types/tests/test_domain.pysimilarity index 100%rename from flowsint-types/tests/domain.pyrename to flowsint-types/tests/test_domain.pydiff --git a/flowsint-types/tests/ip.py b/flowsint-types/tests/test_ip.pysimilarity index 100%rename from flowsint-types/tests/ip.pyrename to flowsint-types/tests/test_ip.pydiff --git a/flowsint-types/tests/test_labels.py b/flowsint-types/tests/test_labels.pynew file mode 100644index 00000000..10e92823--- /dev/null+++ b/flowsint-types/tests/test_labels.py@@ -0,0 +1,16 @@+from flowsint_types import Domain, Ip, Individual+++def test_ip_label():+ ip = Ip(address="12.23.34.56")+ assert ip.label == "12.23.34.56"+++def test_domain_label():+ domain = Domain(domain="blog.mydomain.com")+ assert domain.label == "blog.mydomain.com"+++def test_individual_label():+ indivudual = Individual(first_name="John", last_name="Doe")+ assert indivudual.label == "John Doe"