Skip to content

A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.

feat: minifiy transform logic

dextmorgnade74800aa34
diff --git a/Makefile b/Makefile
index 68e8fcaf..29c2bf3b 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
PROJECT_ROOT := $(shell pwd)
-.PHONY: install run stop stop-dev stop-prod infra api frontend celery clean dev prod check-env build-dev build-prod open-browser
+.PHONY: install run stop stop-dev stop-prod infra api frontend celery clean dev prod check-env build-dev build-prod open-browser test
ENV_DIRS := . flowsint-api flowsint-core flowsint-app
open-browser:
@@ -43,6 +43,12 @@ check-env:
fi; \
done
+test:
+ @echo "🔎 Running tests..."
+ cd $(PROJECT_ROOT)/flowsint-types && poetry run pytest
+ cd $(PROJECT_ROOT)/flowsint-core && poetry run pytest
+ cd $(PROJECT_ROOT)/flowsint-transforms && poetry run pytest
+
install:
@echo "🚀 Installing Flowsint project modules..."
@if ! command -v poetry >/dev/null 2>&1; then \
diff --git a/flowsint-core/README.md b/flowsint-core/README.md
index ea792210..f5ff96ef 100644
--- a/flowsint-core/README.md
+++ b/flowsint-core/README.md
@@ -1,7 +1,7 @@
# flowsint-core
The repository containing the core components for flowsint.
-It contains:
+It contains:
- Celery tasks
- Authentication
- NEO4J database connectors
@@ -11,4 +11,10 @@ It contains:
- Transform registry
- Encrypted vault
+## Testing
+
+```bash
+poetry run pytest
+```
+
> ⚠️ 🚧 Work in progress !.
diff --git a/flowsint-core/src/flowsint_core/core/transform_base.py b/flowsint-core/src/flowsint_core/core/transform_base.py
index b09384cb..bcd32113 100644
--- a/flowsint-core/src/flowsint_core/core/transform_base.py
+++ b/flowsint-core/src/flowsint_core/core/transform_base.py
@@ -62,9 +62,9 @@ class Transform(ABC):
from flowsint_types import Ip
class MyTransform(Transform):
- # Define types as class attributes
- InputType = List[Domain]
- OutputType = List[Ip]
+ # Define types as class attributes (base types, not lists)
+ InputType = Domain
+ OutputType = Ip
@classmethod
def name(cls):
@@ -78,13 +78,14 @@ class Transform(ABC):
def key(cls):
return "domain"
- def preprocess(self, data: InputType) -> InputType:
- cleaned: InputType = []
- # ... implementation
- return cleaned
+ # preprocess receives a list and returns a list of validated InputType instances
+ def preprocess(self, data: List) -> List[InputType]:
+ # Generic implementation handles validation automatically
+ return super().preprocess(data)
- async def scan(self, data: InputType) -> OutputType:
- results: OutputType = []
+ # scan receives a list of InputType and returns a list of OutputType
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
+ results: List[OutputType] = []
# ... implementation
return results
@@ -94,6 +95,7 @@ class Transform(ABC):
```
The base class automatically provides:
+ - Generic preprocess() that validates inputs using InputType
- input_schema() method using InputType
- output_schema() method using OutputType
- Error handling for missing type definitions
@@ -263,17 +265,20 @@ class Transform(ABC):
schema = adapter.json_schema()
# Handle different schema structures
- if "$defs" in schema and schema["$defs"]:
- # Follow the $ref in items to get the correct type (not just the first one)
- items_ref = schema.get("items", {}).get("$ref")
- if items_ref:
- # Extract type name from $ref like "#/$defs/Website" -> "Website"
- type_name = items_ref.split("/")[-1]
- details = schema["$defs"][type_name]
- else:
- # Fallback: get the first type definition (for backward compatibility)
- type_name, details = list(schema["$defs"].items())[0]
-
+ # Check for direct properties first (even if $defs exists for nested types)
+ if "properties" in schema and "title" in schema:
+ # Direct type definition (e.g., Domain, Ip, Website)
+ return {
+ "type": schema.get("title", "Any"),
+ "properties": [
+ {"name": prop, "type": resolve_type(info, schema)}
+ for prop, info in schema["properties"].items()
+ ],
+ }
+ elif "$defs" in schema and schema["$defs"] and "$ref" in schema:
+ # Reference to a type in $defs
+ type_name = schema["$ref"].split("/")[-1]
+ details = schema["$defs"][type_name]
return {
"type": type_name,
"properties": [
@@ -282,7 +287,7 @@ class Transform(ABC):
],
}
else:
- # Handle simpler schemas
+ # Fallback for unknown schema structures
return {
"type": schema.get("title", "Any"),
"properties": [{"name": "value", "type": "object"}],
@@ -303,17 +308,20 @@ class Transform(ABC):
schema = adapter.json_schema()
# Handle different schema structures
- if "$defs" in schema and schema["$defs"]:
- # Follow the $ref in items to get the correct type (not just the first one)
- items_ref = schema.get("items", {}).get("$ref")
- if items_ref:
- # Extract type name from $ref like "#/$defs/Website" -> "Website"
- type_name = items_ref.split("/")[-1]
- details = schema["$defs"][type_name]
- else:
- # Fallback: get the first type definition (for backward compatibility)
- type_name, details = list(schema["$defs"].items())[0]
-
+ # Check for direct properties first (even if $defs exists for nested types)
+ if "properties" in schema and "title" in schema:
+ # Direct type definition (e.g., Domain, Ip, Website)
+ return {
+ "type": schema.get("title", "Any"),
+ "properties": [
+ {"name": prop, "type": resolve_type(info, schema)}
+ for prop, info in schema["properties"].items()
+ ],
+ }
+ elif "$defs" in schema and schema["$defs"] and "$ref" in schema:
+ # Reference to a type in $defs
+ type_name = schema["$ref"].split("/")[-1]
+ details = schema["$defs"][type_name]
return {
"type": type_name,
"properties": [
@@ -322,7 +330,7 @@ class Transform(ABC):
],
}
else:
- # Handle simpler schemas
+ # Fallback for unknown schema structures
return {
"type": schema.get("title", "Any"),
"properties": [{"name": "value", "type": "object"}],
@@ -350,24 +358,24 @@ class Transform(ABC):
Returns:
The secret value from the vault, or default if not found
"""
- return self.params.get(key_name, default)
+ value = self.params.get(key_name, default)
+ # If the value is None, return the default instead (allows fallback to env vars)
+ return value if value is not None else default
def preprocess(self, values: List) -> List:
"""
Generic preprocess that validates and converts input using InputType.
Automatically handles dicts, objects, and strings (using the model's primary field).
Invalid items are skipped silently.
+
+ Note: InputType should be defined as the base type (e.g., Ip, Domain),
+ not as a List (e.g., List[Ip]). The preprocess method expects a list of values
+ and returns a list of validated InputType instances.
"""
if self.InputType is NotImplemented:
return values
- from typing import get_args
-
- type_args = get_args(self.InputType)
- if not type_args:
- return values
-
- base_type = type_args[0]
+ base_type = self.InputType
adapter = TypeAdapter(base_type)
# Trouver le champ primaire marqué par Field(..., primary=True)
@@ -485,8 +493,16 @@ class Transform(ABC):
key_prop = primary_field
key_value = getattr(obj, primary_field)
- # Merge object properties with any overrides
- obj_properties = obj.__dict__.copy()
+ # If key_value is itself a Pydantic model, extract its primary value
+ if isinstance(key_value, BaseModel):
+ key_value = self._extract_primary_value(key_value)
+
+ # Merge object properties with any overrides, but skip nested Pydantic objects
+ obj_properties = {}
+ for k, v in obj.__dict__.items():
+ # Skip nested Pydantic objects
+ if not isinstance(v, BaseModel):
+ obj_properties[k] = v
obj_properties.update(properties)
properties = obj_properties
else:
@@ -571,11 +587,19 @@ class Transform(ABC):
from_primary_field = self._get_primary_field(from_obj)
from_key_value = getattr(from_obj, from_primary_field)
+ # If key_value is a Pydantic model, extract its primary value
+ if isinstance(from_key_value, BaseModel):
+ from_key_value = self._extract_primary_value(from_key_value)
+
# Extract to_node info
to_node_type = to_obj.__class__.__name__.lower()
to_primary_field = self._get_primary_field(to_obj)
to_key_value = getattr(to_obj, to_primary_field)
+ # If key_value is a Pydantic model, extract its primary value
+ if isinstance(to_key_value, BaseModel):
+ to_key_value = self._extract_primary_value(to_key_value)
+
self._graph_service.create_relationship(
from_type=from_node_type,
from_key=from_primary_field,
@@ -620,6 +644,20 @@ class Transform(ABC):
return primary_field
+ def _extract_primary_value(self, obj: BaseModel) -> Any:
+ """
+ Extract the primitive value from a Pydantic object recursively.
+ If the primary field is itself a Pydantic object, recursively extract its primary value.
+ """
+ primary_field = self._get_primary_field(obj)
+ value = getattr(obj, primary_field)
+
+ # If the value is still a Pydantic model, recursively extract
+ if isinstance(value, BaseModel):
+ return self._extract_primary_value(value)
+
+ return value
+
def log_graph_message(self, message: str) -> None:
"""
Log a graph operation message.
diff --git a/flowsint-core/src/flowsint_core/core/vault.py b/flowsint-core/src/flowsint_core/core/vault.py
index 34138eb1..d8427407 100644
--- a/flowsint-core/src/flowsint_core/core/vault.py
+++ b/flowsint-core/src/flowsint_core/core/vault.py
@@ -4,7 +4,7 @@ import uuid
from sqlalchemy.orm import Session
from sqlalchemy import select
from .models import Key
-from datetime import datetime
+from datetime import datetime, UTC
import base64
from cryptography.hazmat.primitives.kdf.hkdf import HKDF
from cryptography.hazmat.primitives import hashes
@@ -115,7 +115,7 @@ class Vault(VaultProtocol):
key_version=self.version,
owner_id=self.owner_id,
ciphertext=encrypted_key_dict.get("ciphertext"),
- created_at=datetime.utcnow(),
+ created_at=datetime.now(UTC),
)
self.db.add(new_key)
self.db.commit()
diff --git a/flowsint-core/tests/test_transform_base_simplified_api.py b/flowsint-core/tests/test_transform_base_simplified_api.py
index abcecbb9..b3aa19dc 100644
--- a/flowsint-core/tests/test_transform_base_simplified_api.py
+++ b/flowsint-core/tests/test_transform_base_simplified_api.py
@@ -7,11 +7,11 @@ from flowsint_types.individual import Individual
from typing import List
-class TestTransform(Transform):
+class MockTransform(Transform):
"""Simple transform for testing."""
- InputType = List[Domain]
- OutputType = List[Domain]
+ InputType = Domain
+ OutputType = Domain
@classmethod
def name(cls) -> str:
@@ -25,13 +25,13 @@ class TestTransform(Transform):
def key(cls) -> str:
return "domain"
- async def scan(self, data: InputType) -> OutputType:
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
return data
def test_create_node_with_pydantic_object():
"""Test that create_node works with Pydantic objects."""
- transform = TestTransform(sketch_id="test", scan_id="test")
+ transform = MockTransform(sketch_id="test", scan_id="test")
# Create a domain object
domain = Domain(domain="example.com")
@@ -46,7 +46,7 @@ def test_create_node_with_pydantic_object():
def test_create_relationship_with_pydantic_objects():
"""Test that create_relationship works with Pydantic objects."""
- transform = TestTransform(sketch_id="test", scan_id="test")
+ transform = MockTransform(sketch_id="test", scan_id="test")
# Create objects
individual = Individual(
@@ -62,7 +62,7 @@ def test_create_relationship_with_pydantic_objects():
def test_create_node_legacy_signature():
"""Test that legacy create_node signature still works."""
- transform = TestTransform(sketch_id="test", scan_id="test")
+ transform = MockTransform(sketch_id="test", scan_id="test")
domain = Domain(domain="example.com")
@@ -72,7 +72,7 @@ def test_create_node_legacy_signature():
def test_create_relationship_legacy_signature():
"""Test that legacy create_relationship signature still works."""
- transform = TestTransform(sketch_id="test", scan_id="test")
+ transform = MockTransform(sketch_id="test", scan_id="test")
# Legacy signature should still work
transform.create_relationship(
@@ -88,7 +88,7 @@ def test_create_relationship_legacy_signature():
def test_get_primary_field():
"""Test the _get_primary_field helper method."""
- transform = TestTransform(sketch_id="test", scan_id="test")
+ transform = MockTransform(sketch_id="test", scan_id="test")
# Test with Domain (has primary field marked)
domain = Domain(domain="example.com")
@@ -109,7 +109,7 @@ def test_get_primary_field():
def test_create_node_with_property_override():
"""Test that property overrides work with Pydantic objects."""
- transform = TestTransform(sketch_id="test", scan_id="test")
+ transform = MockTransform(sketch_id="test", scan_id="test")
domain = Domain(domain="example.com")
diff --git a/flowsint-core/tests/vault/test_transform_vault_integration.py b/flowsint-core/tests/vault/test_transform_vault_integration.py
index 40569ab8..96455765 100644
--- a/flowsint-core/tests/vault/test_transform_vault_integration.py
+++ b/flowsint-core/tests/vault/test_transform_vault_integration.py
@@ -9,8 +9,8 @@ from flowsint_core.core.transform_base import Transform
class MockTransform(Transform):
"""Mock transform for testing."""
- InputType = List[str]
- OutputType = List[str]
+ InputType = str
+ OutputType = str
@classmethod
def name(cls) -> str:
@@ -49,7 +49,7 @@ class MockTransform(Transform):
},
]
- async def scan(self, data: InputType) -> OutputType:
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
return data
@@ -75,123 +75,169 @@ def transform(mock_vault, sketch_id):
scan_id="scan_123",
vault=mock_vault,
params={},
+ params_schema=MockTransform.get_params_schema(),
)
class TestResolveParams:
"""Tests for resolve_params() method."""
- def test_resolve_vault_secret_by_name(self, transform, mock_vault):
+ @pytest.mark.asyncio
+ async def test_resolve_vault_secret_by_name(self, transform, mock_vault):
"""Test resolving a vault secret by parameter name."""
secret_value = "my-api-key-12345"
mock_vault.get_secret.return_value = secret_value
- resolved = transform.resolve_params()
+ await transform.async_init()
# Should try to get by name "TEST_API_KEY"
assert mock_vault.get_secret.called
- assert "TEST_API_KEY" in resolved
- assert resolved["TEST_API_KEY"] == secret_value
+ assert "TEST_API_KEY" in transform.params
+ assert transform.params["TEST_API_KEY"] == secret_value
- def test_resolve_vault_secret_by_id(self, transform, mock_vault):
+ @pytest.mark.asyncio
+ async def test_resolve_vault_secret_by_id(self, sketch_id, mock_vault):
"""Test resolving a vault secret by vault ID."""
secret_value = "my-api-key-12345"
vault_id = str(uuid.uuid4())
- # Set params with a vault ID
- transform.params = {"TEST_API_KEY": vault_id}
+ # Mock vault to return secret for ID, and optional key default
+ def get_secret_side_effect(key):
+ if key == vault_id:
+ return secret_value
+ elif key == "OPTIONAL_KEY":
+ return None # Will use default
+ return None
- # First call (by ID) returns the secret
- mock_vault.get_secret.return_value = secret_value
+ mock_vault.get_secret.side_effect = get_secret_side_effect
- resolved = transform.resolve_params()
+ # Create transform with vault ID in params
+ transform = MockTransform(
+ sketch_id=sketch_id,
+ scan_id="scan_123",
+ vault=mock_vault,
+ params={"TEST_API_KEY": vault_id},
+ params_schema=MockTransform.get_params_schema(),
+ )
- # Should try to get by ID first
- mock_vault.get_secret.assert_called_with(vault_id)
- assert "TEST_API_KEY" in resolved
- assert resolved["TEST_API_KEY"] == secret_value
+ await transform.async_init()
+
+ # Should have called get_secret with the vault ID
+ assert any(call[0][0] == vault_id for call in mock_vault.get_secret.call_args_list)
+ assert "TEST_API_KEY" in transform.params
+ assert transform.params["TEST_API_KEY"] == secret_value
- def test_resolve_vault_secret_fallback_to_name(self, transform, mock_vault):
+ @pytest.mark.asyncio
+ async def test_resolve_vault_secret_fallback_to_name(self, sketch_id, mock_vault):
"""Test that if vault ID fails, it falls back to name lookup."""
secret_value = "my-api-key-12345"
vault_id = str(uuid.uuid4())
- # Set params with a vault ID
- transform.params = {"TEST_API_KEY": vault_id}
+ # Mock vault: ID returns None, name returns secret, optional returns None
+ def get_secret_side_effect(key):
+ if key == vault_id:
+ return None # Vault ID not found
+ elif key == "TEST_API_KEY":
+ return secret_value # Fallback to name works
+ elif key == "OPTIONAL_KEY":
+ return None # Will use default
+ return None
- # First call (by ID) returns None, second call (by name) returns the secret
- mock_vault.get_secret.side_effect = [None, secret_value]
+ mock_vault.get_secret.side_effect = get_secret_side_effect
- resolved = transform.resolve_params()
+ # Create transform with vault ID in params
+ transform = MockTransform(
+ sketch_id=sketch_id,
+ scan_id="scan_123",
+ vault=mock_vault,
+ params={"TEST_API_KEY": vault_id},
+ params_schema=MockTransform.get_params_schema(),
+ )
- # Should try both ID and name
- assert mock_vault.get_secret.call_count == 2
- assert "TEST_API_KEY" in resolved
- assert resolved["TEST_API_KEY"] == secret_value
+ await transform.async_init()
+
+ # Should have called get_secret with both vault ID and TEST_API_KEY name
+ call_args = [call[0][0] for call in mock_vault.get_secret.call_args_list]
+ assert vault_id in call_args
+ assert "TEST_API_KEY" in call_args
+ assert "TEST_API_KEY" in transform.params
+ assert transform.params["TEST_API_KEY"] == secret_value
def test_resolve_vault_secret_not_found_required(self, transform, mock_vault):
- """Test that missing required vault secret logs error."""
+ """Test that missing required vault secret raises exception."""
mock_vault.get_secret.return_value = None
- with patch("flowsint_core.core.transform_base.Logger") as mock_logger:
- resolved = transform.resolve_params()
+ # Should raise an exception for missing required secret
+ with pytest.raises(Exception) as exc_info:
+ transform.resolve_params()
- # Should log an error for missing required secret
- assert mock_logger.error.called
- error_message = mock_logger.error.call_args[0][1]["message"]
- assert "TEST_API_KEY" in error_message
- assert "missing" in error_message.lower()
- assert "Vault settings" in error_message
+ assert "TEST_API_KEY" in str(exc_info.value)
+ assert "missing" in str(exc_info.value).lower()
+ assert "Vault settings" in str(exc_info.value)
- # Should not be in resolved params
- assert "TEST_API_KEY" not in resolved
-
- def test_resolve_vault_secret_optional_with_default(self, transform, mock_vault):
+ @pytest.mark.asyncio
+ async def test_resolve_vault_secret_optional_with_default(self, transform, mock_vault):
"""Test that optional vault secret uses default if not found."""
# Only return secret for TEST_API_KEY, not OPTIONAL_KEY
mock_vault.get_secret.side_effect = lambda key: (
"test-api-key" if key == "TEST_API_KEY" else None
)
- resolved = transform.resolve_params()
+ await transform.async_init()
# Optional key should use default value
- assert "OPTIONAL_KEY" in resolved
- assert resolved["OPTIONAL_KEY"] == "default_value"
+ assert "OPTIONAL_KEY" in transform.params
+ assert transform.params["OPTIONAL_KEY"] == "default_value"
- def test_resolve_non_vault_param_from_params(self, transform):
+ @pytest.mark.asyncio
+ async def test_resolve_non_vault_param_from_params(self, mock_vault, sketch_id):
"""Test that non-vault params are taken from params."""
- transform.params = {"NON_VAULT_PARAM": "custom_value"}
+ transform = MockTransform(
+ sketch_id=sketch_id,
+ scan_id="scan_123",
+ vault=mock_vault,
+ params={"NON_VAULT_PARAM": "custom_value"},
+ params_schema=MockTransform.get_params_schema(),
+ )
+ mock_vault.get_secret.return_value = "test-api-key"
- resolved = transform.resolve_params()
+ await transform.async_init()
- assert "NON_VAULT_PARAM" in resolved
- assert resolved["NON_VAULT_PARAM"] == "custom_value"
+ assert "NON_VAULT_PARAM" in transform.params
+ assert transform.params["NON_VAULT_PARAM"] == "custom_value"
- def test_resolve_non_vault_param_uses_default(self, transform):
+ @pytest.mark.asyncio
+ async def test_resolve_non_vault_param_uses_default(self, transform, mock_vault):
"""Test that non-vault params use default if not provided."""
- resolved = transform.resolve_params()
+ mock_vault.get_secret.return_value = "test-api-key"
- assert "NON_VAULT_PARAM" in resolved
- assert resolved["NON_VAULT_PARAM"] == "test"
+ await transform.async_init()
+
+ assert "NON_VAULT_PARAM" in transform.params
+ assert transform.params["NON_VAULT_PARAM"] == "test"
def test_resolve_params_no_vault_instance(self, sketch_id):
- """Test that resolve_params works without a vault instance."""
+ """Test that resolve_params works without vault (uses defaults for optional params)."""
transform = MockTransform(
sketch_id=sketch_id,
scan_id="scan_123",
vault=None, # No vault
params={},
+ params_schema=MockTransform.get_params_schema(),
)
+ # Without vault, required secrets won't be resolved, optional ones will use defaults
resolved = transform.resolve_params()
- # Should use defaults for optional params
+ # Optional params should use defaults
assert "OPTIONAL_KEY" in resolved
assert resolved["OPTIONAL_KEY"] == "default_value"
assert "NON_VAULT_PARAM" in resolved
assert resolved["NON_VAULT_PARAM"] == "test"
+ # Required vault secrets won't be in resolved (no exception when vault is None)
+ assert "TEST_API_KEY" not in resolved
+
class TestGetSecret:
"""Tests for get_secret() method."""
@@ -211,33 +257,54 @@ class TestGetSecret:
assert result == secret_value
@pytest.mark.asyncio
- async def test_get_secret_not_found_uses_default(self, transform, mock_vault):
+ async def test_get_secret_not_found_uses_default(self, mock_vault, sketch_id):
"""Test get_secret returns default when secret not found."""
- mock_vault.get_secret.return_value = None
+ mock_vault.get_secret.return_value = "test-api-key"
+
+ transform = MockTransform(
+ sketch_id=sketch_id,
+ scan_id="scan_123",
+ vault=mock_vault,
+ params={},
+ params_schema=MockTransform.get_params_schema(),
+ )
# Run async_init to resolve params
await transform.async_init()
- # Get the secret with default
+ # Get the secret with default (NONEXISTENT_KEY doesn't exist in params)
result = transform.get_secret("NONEXISTENT_KEY", "default")
assert result == "default"
@pytest.mark.asyncio
- async def test_get_secret_returns_none_when_not_found_no_default(
- self, transform, mock_vault
+ async def test_get_secret_returns_default_when_value_is_none(
+ self, mock_vault, sketch_id
):
- """Test get_secret returns None when no default provided."""
- mock_vault.get_secret.return_value = None
+ """Test get_secret returns default when value is None."""
+ mock_vault.get_secret.return_value = "test-api-key"
+
+ transform = MockTransform(
+ sketch_id=sketch_id,
+ scan_id="scan_123",
+ vault=mock_vault,
+ params={},
+ params_schema=MockTransform.get_params_schema(),
+ )
# Run async_init to resolve params
await transform.async_init()
- # Get the secret without default
+ # Get a non-existent secret without default
result = transform.get_secret("NONEXISTENT_KEY")
assert result is None
+ # Get a non-existent secret with default
+ result = transform.get_secret("NONEXISTENT_KEY", "fallback")
+
+ assert result == "fallback"
+
class TestAsyncInit:
"""Tests for async_init() method."""
@@ -263,6 +330,7 @@ class TestAsyncInit:
scan_id="scan_123",
vault=mock_vault,
params={}, # Empty params
+ params_schema=MockTransform.get_params_schema(),
)
secret_value = "my-api-key-12345"
@@ -307,7 +375,7 @@ class TestTransformExecuteWithVault:
assert transform.params["TEST_API_KEY"] == secret_value
@pytest.mark.asyncio
- async def test_execute_uses_resolved_secrets_in_scan(self, transform, mock_vault):
+ async def test_execute_uses_resolved_secrets_in_scan(self, mock_vault):
"""Test that scan can access resolved secrets."""
secret_value = "my-api-key-12345"
mock_vault.get_secret.return_value = secret_value
@@ -323,6 +391,7 @@ class TestTransformExecuteWithVault:
scan_id="scan_123",
vault=mock_vault,
params={},
+ params_schema=MockTransform.get_params_schema(),
)
# Execute
diff --git a/flowsint-core/tests/vault/test_vault.py b/flowsint-core/tests/vault/test_vault.py
index 536e66bd..5d31f9b1 100644
--- a/flowsint-core/tests/vault/test_vault.py
+++ b/flowsint-core/tests/vault/test_vault.py
@@ -220,10 +220,12 @@ class TestVaultEncryptionDecryption:
def test_invalid_master_key_length(self, vault, monkeypatch):
"""Test that invalid master key length raises error."""
- # Set an invalid master key (wrong length)
- monkeypatch.setenv("MASTER_VAULT_KEY_V1", "base64:invalid")
+ import base64
+ # Set a valid base64 but wrong length (16 bytes instead of 32)
+ short_key = base64.b64encode(b"0" * 16).decode()
+ monkeypatch.setenv("MASTER_VAULT_KEY_V1", f"base64:{short_key}")
- with pytest.raises(ValueError, match="Master key must be 32 bytes"):
+ with pytest.raises(ValueError, match="Master key must be 32 bytes \\(256 bits\\)"):
vault._get_master_key()
def test_missing_master_key(self, vault, monkeypatch):
diff --git a/flowsint-transforms/README.md b/flowsint-transforms/README.md
index 4b00e8d7..a1483c83 100644
--- a/flowsint-transforms/README.md
+++ b/flowsint-transforms/README.md
@@ -1,4 +1,11 @@
# flowsint-transforms
+
The repository containing open-source transforms for flowsint.
+## Testing
+
+```bash
+poetry run pytest
+```
+
> ⚠️ 🚧 Work in progress !.
diff --git a/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.py b/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.py
index 75f9b10a..84f85c74 100644
--- a/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.py
+++ b/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.py
@@ -14,8 +14,8 @@ class AsnToCidrsTransform(Transform):
"""[ASNMAP] Takes an ASN and returns its corresponding CIDRs."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[ASN]
- OutputType = List[CIDR]
+ InputType = ASN
+ OutputType = CIDR
def __init__(
self,
@@ -62,9 +62,9 @@ class AsnToCidrsTransform(Transform):
def key(cls) -> str:
return "number"
- async def scan(self, data: InputType) -> OutputType:
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
"""Find CIDR from ASN using asnmap."""
- cidrs: OutputType = []
+ cidrs: List[OutputType] = []
self._asn_to_cidrs_map = [] # Store mapping for postprocess
asnmap = AsnmapTool()
@@ -115,7 +115,7 @@ class AsnToCidrsTransform(Transform):
return cidrs
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
# Create Neo4j relationships between ASNs and their corresponding CIDRs
# Use the mapping from scan if available, else fallback to zip
asn_to_cidrs = getattr(self, "_asn_to_cidrs_map", None)
diff --git a/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.py b/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.py
index bf1599e0..c7fde030 100644
--- a/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.py
+++ b/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.py
@@ -12,8 +12,8 @@ class CidrToIpsTransform(Transform):
"""[MAPCIDR] Takes a CIDR and returns its corresponding IP addresses."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[CIDR]
- OutputType = List[Ip]
+ InputType = CIDR
+ OutputType = Ip
def __init__(
self,
@@ -60,9 +60,9 @@ class CidrToIpsTransform(Transform):
def category(cls) -> str:
return "Cidr"
- async def scan(self, data: InputType) -> OutputType:
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
"""Find IP addresses from CIDR using mapcidr."""
- ips: OutputType = []
+ ips: List[OutputType] = []
self._cidr_to_ips_map = [] # Store mapping for postprocess
mapcidr = MapcidrTool()
@@ -111,7 +111,7 @@ class CidrToIpsTransform(Transform):
return ips
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
# Create Neo4j relationships between CIDRs and their corresponding IPs
# Use the mapping from scan if available, else fallback to zip
cidr_to_ips = getattr(self, "_cidr_to_ips_map", None)
diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py
index 4e7459cc..84e29b4d 100644
--- a/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py
+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py
@@ -14,8 +14,8 @@ class CryptoWalletAddressToNFTs(Transform):
"""[ETHERSCAN] Resolve NFTs for a wallet address (ETH)."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[CryptoWallet]
- OutputType = List[CryptoNFT]
+ InputType = CryptoWallet
+ OutputType = CryptoNFT
def __init__(
self,
@@ -73,8 +73,8 @@ class CryptoWalletAddressToNFTs(Transform):
def key(cls) -> str:
return "address"
- async def scan(self, data: InputType) -> OutputType:
- results: OutputType = []
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
+ results: List[OutputType] = []
api_key = self.get_secret("ETHERSCAN_API_KEY", os.getenv("ETHERSCAN_API_KEY"))
api_url = self.get_params().get(
"ETHERSCAN_API_URL", "https://api.etherscan.io/v2/api"
@@ -118,7 +118,7 @@ class CryptoWalletAddressToNFTs(Transform):
)
return nfts
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
if not self.neo4j_conn:
return results
diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py
index 50689812..00bfc5a0 100644
--- a/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py
+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py
@@ -20,8 +20,8 @@ class CryptoWalletAddressToTransactions(Transform):
"""[ETHERSCAN] Resolve transactions for a wallet address (ETH)."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[CryptoWallet]
- OutputType = List[CryptoWalletTransaction]
+ InputType = CryptoWallet
+ OutputType = CryptoWalletTransaction
def __init__(
self,
@@ -79,8 +79,8 @@ class CryptoWalletAddressToTransactions(Transform):
def key(cls) -> str:
return "address"
- async def scan(self, data: InputType) -> OutputType:
- results: OutputType = []
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
+ results: List[OutputType] = []
api_key = self.get_secret("ETHERSCAN_API_KEY", os.getenv("ETHERSCAN_API_KEY"))
api_url = self.get_params().get("ETHERSCAN_API_URL", "https://api.etherscan.io/v2/api")
for d in data:
@@ -173,7 +173,7 @@ class CryptoWalletAddressToTransactions(Transform):
)
return transactions
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
if not self.neo4j_conn:
return results
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_asn.py b/flowsint-transforms/src/flowsint_transforms/domain/to_asn.py
index 3463077e..fd2b8abe 100644
--- a/flowsint-transforms/src/flowsint_transforms/domain/to_asn.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_asn.py
@@ -15,8 +15,8 @@ class DomainToAsnTransform(Transform):
"""[ASNMAP] Takes a domain and returns its corresponding ASN."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[Domain]
- OutputType = List[ASN]
+ InputType = Domain
+ OutputType = ASN
def __init__(
self,
@@ -63,8 +63,8 @@ class DomainToAsnTransform(Transform):
def key(cls) -> str:
return "domain"
- async def scan(self, data: InputType) -> OutputType:
- results: OutputType = []
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
+ results: List[OutputType] = []
asnmap = AsnmapTool()
# Retrieve API key from vault or environment
@@ -113,8 +113,8 @@ class DomainToAsnTransform(Transform):
return results
def postprocess(
- self, results: OutputType, input_data: InputType = None
- ) -> OutputType:
+ self, results: List[OutputType], input_data: List[InputType] = None
+ ) -> List[OutputType]:
# Create Neo4j relationships between domains and their corresponding ASNs
if input_data and self.neo4j_conn:
for domain, asn in zip(input_data, results):
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_history.py b/flowsint-transforms/src/flowsint_transforms/domain/to_history.py
index f6d07003..28d65f7d 100644
--- a/flowsint-transforms/src/flowsint_transforms/domain/to_history.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_history.py
@@ -21,8 +21,8 @@ class DomainToHistoryTransform(Transform):
"""[WHOXY] Takes a domain and returns history infos about it (history, organization, owners, emails, etc.)."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[Domain]
- OutputType = List[Domain]
+ InputType = Domain
+ OutputType = Domain
def __init__(
self,
@@ -69,9 +69,9 @@ class DomainToHistoryTransform(Transform):
def key(cls) -> str:
return "domain"
- async def scan(self, data: InputType) -> OutputType:
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
"""Find infos related to domains using whoxy api."""
- domains: OutputType = []
+ domains: List[OutputType] = []
self._extracted_data = [] # Store all extracted data for postprocess
self._extracted_individuals = [] # Store extracted individuals for testing
self._extracted_organizations = [] # Store extracted organizations for testing
@@ -390,7 +390,7 @@ class DomainToHistoryTransform(Transform):
},
)
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
"""Create Neo4j nodes and relationships from extracted data."""
if not self.neo4j_conn:
Logger.info(
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py b/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py
index 55cc885b..e1682a53 100644
--- a/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py
@@ -9,8 +9,8 @@ from flowsint_types.ip import Ip
class ResolveTransform(Transform):
"""Resolve domain names to IP addresses."""
- InputType = List[Domain]
- OutputType = List[Ip]
+ InputType = Domain
+ OutputType = Ip
@classmethod
def name(cls) -> str:
@@ -29,8 +29,8 @@ class ResolveTransform(Transform):
"""Return formatted markdown documentation for the domain resolver transform."""
return ""
- async def scan(self, data: InputType) -> OutputType:
- results: OutputType = []
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
+ results: List[OutputType] = []
for d in data:
try:
ip = socket.gethostbyname(d.domain)
@@ -43,7 +43,7 @@ class ResolveTransform(Transform):
continue
return results
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
for domain_obj, ip_obj in zip(original_input, results):
self.create_node(domain_obj)
self.create_node(ip_obj)
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py b/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py
index 73fee128..5edcf516 100644
--- a/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py
@@ -8,8 +8,8 @@ from flowsint_core.core.logger import Logger
class DomainToRootDomain(Transform):
"""Subdomain to root domain."""
- InputType = List[Domain]
- OutputType = List[Domain]
+ InputType = Domain
+ OutputType = Domain
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -28,8 +28,8 @@ class DomainToRootDomain(Transform):
def key(cls) -> str:
return "domain"
- async def scan(self, data: InputType) -> OutputType:
- results: OutputType = []
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
+ results: List[OutputType] = []
self.domain_root_mapping = [] # Reset mapping
for domain in data:
@@ -51,7 +51,7 @@ class DomainToRootDomain(Transform):
return results
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
# Use the mapping we created during scan to create relationships
for original_domain, root_domain in self.domain_root_mapping:
if not self.neo4j_conn:
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py b/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py
index 6f0b6013..85c9ec91 100644
--- a/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py
@@ -12,8 +12,8 @@ class SubdomainTransform(Transform):
"""Transform to find subdomains associated with a domain."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[Domain]
- OutputType = List[Domain]
+ InputType = Domain
+ OutputType = Domain
@classmethod
def name(cls) -> str:
@@ -27,9 +27,9 @@ class SubdomainTransform(Transform):
def key(cls) -> str:
return "domain"
- async def scan(self, data: InputType) -> OutputType:
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
"""Find subdomains using subfinder (Docker) or fallback to crt.sh."""
- domains: OutputType = []
+ domains: List[OutputType] = []
for md in data:
d = Domain(domain=md.domain)
@@ -88,8 +88,8 @@ class SubdomainTransform(Transform):
)
return subdomains
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
- output: OutputType = []
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ output: List[OutputType] = []
for domain_obj in results:
if not self.neo4j_conn:
continue
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_website.py b/flowsint-transforms/src/flowsint_transforms/domain/to_website.py
index a90432bf..e26614ce 100644
--- a/flowsint-transforms/src/flowsint_transforms/domain/to_website.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_website.py
@@ -11,8 +11,8 @@ class DomainToWebsiteTransform(Transform):
"""From domain to website."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[Domain]
- OutputType = List[Website]
+ InputType = Domain
+ OutputType = Website
@classmethod
def name(cls) -> str:
@@ -26,8 +26,8 @@ class DomainToWebsiteTransform(Transform):
def key(cls) -> str:
return "domain"
- async def scan(self, data: InputType) -> OutputType:
- results: OutputType = []
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
+ results: List[OutputType] = []
for domain in data:
try:
# Try HTTPS first
@@ -67,7 +67,7 @@ class DomainToWebsiteTransform(Transform):
return results
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
for website in results:
# Log each redirect step
if website.redirects:
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py b/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py
index e00128cb..41e0dfd7 100644
--- a/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py
@@ -12,8 +12,8 @@ class WhoisTransform(Transform):
"""Scan for WHOIS information of a domain."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[Domain]
- OutputType = List[Whois]
+ InputType = Domain
+ OutputType = Whois
@classmethod
def name(cls) -> str:
@@ -27,8 +27,8 @@ class WhoisTransform(Transform):
def key(cls) -> str:
return "domain"
- async def scan(self, data: InputType) -> OutputType:
- results: OutputType = []
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
+ results: List[OutputType] = []
for domain in data:
try:
whois_info = whois.whois(domain.domain)
@@ -107,66 +107,25 @@ class WhoisTransform(Transform):
return results
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
for whois_obj in results:
if not self.neo4j_conn:
continue
-
# Create domain node
self.create_node(whois_obj.domain)
-
- # Create whois node with custom key (only primitive fields)
- whois_key = f"{whois_obj.domain.domain}_{self.sketch_id}"
- whois_props = {
- "registry_domain_id": whois_obj.registry_domain_id,
- "registrar": whois_obj.registrar,
- "city": whois_obj.city,
- "country": whois_obj.country,
- "creation_date": whois_obj.creation_date,
- "expiration_date": whois_obj.expiration_date,
- "label": whois_obj.label,
- }
- self.create_node("whois", "whois_id", whois_key, **whois_props)
-
+ # Create whois node (nested Pydantic objects are automatically skipped)
+ self.create_node(whois_obj)
# Create relationship between domain and whois
- domain_obj = Domain(domain=whois_obj.domain.domain)
- whois_node = Whois(
- domain=domain_obj,
- registry_domain_id=whois_obj.registry_domain_id,
- registrar=whois_obj.registrar,
- city=whois_obj.city,
- country=whois_obj.country,
- creation_date=whois_obj.creation_date,
- expiration_date=whois_obj.expiration_date,
- )
- self.create_relationship(domain_obj, whois_node, "HAS_WHOIS")
-
- # Create organization node if organization information is available
+ self.create_relationship(whois_obj.domain, whois_obj, "HAS_WHOIS")
+ # Create organization node if available
if whois_obj.organization:
self.create_node(whois_obj.organization)
-
- # Create relationship between organization and domain
- domain_obj2 = Domain(domain=whois_obj.domain.domain)
- self.create_relationship(whois_obj.organization, domain_obj2, "HAS_DOMAIN")
-
- self.log_graph_message(
- f"{whois_obj.domain.domain} -> {whois_obj.organization.name} (organization)"
- )
-
- # Create email node if email information is available
+ self.create_relationship(whois_obj.organization, whois_obj.domain, "HAS_DOMAIN")
+ # Create email node if available
if whois_obj.email:
self.create_node(whois_obj.email)
- whois_node2 = Whois(
- domain=Domain(domain=whois_obj.domain.domain),
- registry_domain_id=whois_obj.registry_domain_id,
- registrar=whois_obj.registrar,
- city=whois_obj.city,
- country=whois_obj.country,
- creation_date=whois_obj.creation_date,
- expiration_date=whois_obj.expiration_date,
- )
- self.create_relationship(whois_node2, whois_obj.email, "REGISTERED_BY")
-
+ self.create_relationship(whois_obj, whois_obj.email, "REGISTERED_BY")
+ # Log message
self.log_graph_message(
f"WHOIS for {whois_obj.domain.domain} -> registry_id: {whois_obj.registry_domain_id} registrar: {whois_obj.registrar} org: {whois_obj.organization.name if whois_obj.organization else None} city: {whois_obj.city} country: {whois_obj.country} creation_date: {whois_obj.creation_date} expiration_date: {whois_obj.expiration_date}"
)
diff --git a/flowsint-transforms/src/flowsint_transforms/email/to_domains.py b/flowsint-transforms/src/flowsint_transforms/email/to_domains.py
index 5a66a2c3..252bcd3c 100644
--- a/flowsint-transforms/src/flowsint_transforms/email/to_domains.py
+++ b/flowsint-transforms/src/flowsint_transforms/email/to_domains.py
@@ -21,8 +21,8 @@ class EmailToDomainsTransform(Transform):
"""[WHOXY] Takes an email and returns the domains it registered."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[Email]
- OutputType = List[Domain]
+ InputType = Email
+ OutputType = Domain
def __init__(
self,
@@ -69,9 +69,9 @@ class EmailToDomainsTransform(Transform):
def key(cls) -> str:
return "email"
- async def scan(self, data: InputType) -> OutputType:
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
"""Find domains related to emails using whoxy api."""
- domains: OutputType = []
+ domains: List[OutputType] = []
self._extracted_data = [] # Store all extracted data for postprocess
api_key = self.get_secret("WHOXY_API_KEY", os.getenv("WHOXY_API_KEY"))
@@ -215,7 +215,7 @@ class EmailToDomainsTransform(Transform):
address=address, city=city, zip=zip_code, country=country
)
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
"""Create Neo4j nodes and relationships from extracted data."""
if not self.neo4j_conn:
return results
diff --git a/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py b/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py
index 2b9299a3..7bec2d14 100644
--- a/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py
+++ b/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py
@@ -10,8 +10,8 @@ from flowsint_types.gravatar import Gravatar
class EmailToGravatarTransform(Transform):
"""From md5 hash of email to gravatar."""
- InputType = List[Email]
- OutputType = List[Gravatar]
+ InputType = Email
+ OutputType = Gravatar
@classmethod
def name(cls) -> str:
@@ -25,8 +25,8 @@ class EmailToGravatarTransform(Transform):
def key(cls) -> str:
return "email"
- async def scan(self, data: InputType) -> OutputType:
- results: OutputType = []
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
+ results: List[OutputType] = []
for email in data:
try:
@@ -82,7 +82,7 @@ class EmailToGravatarTransform(Transform):
return results
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
for email_obj, gravatar_obj in zip(original_input, results):
if not self.neo4j_conn:
continue
diff --git a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py
index acd3b7f0..04ed6516 100644
--- a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py
+++ b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py
@@ -18,8 +18,8 @@ HIBP_API_KEY = os.getenv("HIBP_API_KEY")
class EmailToBreachesTransform(Transform):
"""[HIBPWNED] Get the breaches the email might be invovled in."""
- InputType = List[Email]
- OutputType = List[tuple] # List of (email, breach) tuples
+ InputType = Email
+ OutputType = tuple # (email, breach) tuple
def __init__(
self,
@@ -73,8 +73,8 @@ class EmailToBreachesTransform(Transform):
},
]
- async def scan(self, data: InputType) -> OutputType:
- results: OutputType = []
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
+ results: List[OutputType] = []
api_key = self.get_secret("HIBP_API_KEY", os.getenv("HIBP_API_KEY"))
api_url = self.get_params().get("HIBP_API_URL", "https://haveibeenpwned.com/api/v3/breachedaccount/")
headers = {"hibp-api-key": api_key, "User-Agent": "FlowsInt-Transform"}
@@ -131,7 +131,7 @@ class EmailToBreachesTransform(Transform):
return results
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
# Create email nodes first
for email_obj in original_input:
if not self.neo4j_conn:
diff --git a/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py b/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py
index 235a140b..4aacad62 100644
--- a/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py
+++ b/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py
@@ -20,8 +20,8 @@ class IndividualToDomainsTransform(Transform):
"""[WHOXY] Takes an individual and returns the domains it registered."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[Individual]
- OutputType = List[Domain]
+ InputType = Individual
+ OutputType = Domain
def __init__(
self,
@@ -68,10 +68,10 @@ class IndividualToDomainsTransform(Transform):
def key(cls) -> str:
return "full_name"
- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:
+ def preprocess(self, data: Union[List[str], List[dict], List[InputType]]) -> List[InputType]:
if not isinstance(data, list):
raise ValueError(f"Expected list input, got {type(data).__name__}")
- cleaned: InputType = []
+ cleaned: List[InputType] = []
for item in data:
if isinstance(item, str):
parts = item.strip().split()
@@ -98,9 +98,9 @@ class IndividualToDomainsTransform(Transform):
)
return cleaned
- async def scan(self, data: InputType) -> OutputType:
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
"""Find domains related to individuals using whoxy api."""
- domains: OutputType = []
+ domains: List[OutputType] = []
self._extracted_data = [] # Store all extracted data for postprocess
api_key = self.get_secret("WHOXY_API_KEY", os.getenv("WHOXY_API_KEY"))
@@ -247,7 +247,7 @@ class IndividualToDomainsTransform(Transform):
return Location(address=address, city=city, zip=zip_code, country=country)
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
"""Create Neo4j nodes and relationships from extracted data."""
if not self.neo4j_conn:
return results
diff --git a/flowsint-transforms/src/flowsint_transforms/individual/to_org.py b/flowsint-transforms/src/flowsint_transforms/individual/to_org.py
index c25c88de..b7aee213 100644
--- a/flowsint-transforms/src/flowsint_transforms/individual/to_org.py
+++ b/flowsint-transforms/src/flowsint_transforms/individual/to_org.py
@@ -11,8 +11,8 @@ class IndividualToOrgTransform(Transform):
"""[SIRENE] Find organization from a person with data from SIRENE (France only)."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[Individual]
- OutputType = List[Organization]
+ InputType = Individual
+ OutputType = Organization
@classmethod
def name(cls) -> str:
@@ -26,10 +26,10 @@ class IndividualToOrgTransform(Transform):
def key(cls) -> str:
return "fullname"
- def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:
+ def preprocess(self, data: Union[List[str], List[dict], List[InputType]]) -> List[InputType]:
if not isinstance(data, list):
raise ValueError(f"Expected list input, got {type(data).__name__}")
- cleaned: InputType = []
+ cleaned: List[InputType] = []
for item in data:
if isinstance(item, str):
parts = item.strip().split()
@@ -56,9 +56,9 @@ class IndividualToOrgTransform(Transform):
)
return cleaned
- async def scan(self, data: InputType) -> OutputType:
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
- results: OutputType = []
+ results: List[OutputType] = []
for individual in data:
try:
sirene = SireneTool()
@@ -278,7 +278,7 @@ class IndividualToOrgTransform(Transform):
)
return None
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
if not self.neo4j_conn:
return results
diff --git a/flowsint-transforms/src/flowsint_transforms/ip/to_asn.py b/flowsint-transforms/src/flowsint_transforms/ip/to_asn.py
index 4024b3e7..1cb9af61 100644
--- a/flowsint-transforms/src/flowsint_transforms/ip/to_asn.py
+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_asn.py
@@ -14,8 +14,8 @@ class IpToAsnTransform(Transform):
"""[ASNMAP] Takes an IP address and returns its corresponding ASN."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[Ip]
- OutputType = List[ASN]
+ InputType = Ip
+ OutputType = ASN
def __init__(
self,
@@ -62,8 +62,8 @@ class IpToAsnTransform(Transform):
def key(cls) -> str:
return "address"
- async def scan(self, data: InputType) -> OutputType:
- results: OutputType = []
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
+ results: List[OutputType] = []
asnmap = AsnmapTool()
# Retrieve API key from vault or environment
@@ -108,8 +108,8 @@ class IpToAsnTransform(Transform):
return results
def postprocess(
- self, results: OutputType, input_data: InputType = None
- ) -> OutputType:
+ self, results: List[OutputType], input_data: List[InputType] = None
+ ) -> List[OutputType]:
# Create Neo4j relationships between IPs and their corresponding ASNs
if input_data and self.neo4j_conn:
for ip, asn in zip(input_data, results):
diff --git a/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py b/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py
index 3de35ab4..e31ebf49 100644
--- a/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py
+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py
@@ -1,22 +1,17 @@
-import re
import socket
import requests
-from typing import List, Union
+from typing import List
from flowsint_core.core.logger import Logger
from flowsint_core.core.transform_base import Transform
from flowsint_types.domain import Domain
from flowsint_types.ip import Ip
-from flowsint_core.utils import is_valid_ip
-
-PTR_BLACKLIST = re.compile(r"^ip\d+\.ip-\d+-\d+-\d+-\d+\.")
class ReverseResolveTransform(Transform):
"""Resolve IP addresses to domain names using PTR, Certificate Transparency and optional API calls."""
- # Define types as class attributes - base class handles schema generation automatically
- InputType = List[Ip]
- OutputType = List[Domain]
+ InputType = Ip
+ OutputType = Domain
@classmethod
def name(cls) -> str:
@@ -30,28 +25,26 @@ class ReverseResolveTransform(Transform):
def key(cls) -> str:
return "address"
- async def scan(self, data: InputType) -> OutputType:
- results: OutputType = []
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
+ results: List[OutputType] = []
for ip in data:
try:
- # Try PTR lookup
try:
hostname = socket.gethostbyaddr(ip.address)[0]
- if hostname and not PTR_BLACKLIST.match(hostname):
+ if hostname:
domain = Domain(domain=hostname)
results.append(domain)
continue
except socket.herror:
pass
- # Try Certificate Transparency logs
try:
ct_url = f"https://crt.sh/?q={ip.address}&output=json"
response = requests.get(ct_url, timeout=10)
if response.status_code == 200:
ct_data = response.json()
- for entry in ct_data[:5]: # Limit to first 5 results
+ for entry in ct_data[:15]:
name_value = entry.get("name_value", "")
if name_value and name_value != ip.address:
domain = Domain(domain=name_value)
@@ -69,12 +62,9 @@ class ReverseResolveTransform(Transform):
return results
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
- # Create nodes and relationships for each resolved domain
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
for ip_obj in original_input:
- # Create IP node
self.create_node(ip_obj)
- # Create domain nodes and relationships for each resolved domain
for domain_obj in results:
self.create_node(domain_obj)
self.create_relationship(ip_obj, domain_obj, "REVERSE_RESOLVES_TO")
diff --git a/flowsint-transforms/src/flowsint_transforms/ip/to_infos.py b/flowsint-transforms/src/flowsint_transforms/ip/to_infos.py
index 6c5dd14a..5b1d3d40 100644
--- a/flowsint-transforms/src/flowsint_transforms/ip/to_infos.py
+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_infos.py
@@ -8,8 +8,8 @@ class IpToInfosTransform(Transform):
"""[ip-api.com] Get information data for IP addresses."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[Ip]
- OutputType = List[Ip]
+ InputType = Ip
+ OutputType = Ip
@classmethod
def name(cls) -> str:
@@ -23,8 +23,8 @@ class IpToInfosTransform(Transform):
def key(cls) -> str:
return "address"
- async def scan(self, data: InputType) -> OutputType:
- results: OutputType = []
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
+ results: List[OutputType] = []
for ip in data:
try:
geo_data = self.get_location_data(ip.address)
@@ -39,7 +39,7 @@ class IpToInfosTransform(Transform):
print(f"Error geolocating {ip.address}: {e}")
return results
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
"""Update IP nodes in Neo4j with geolocation information."""
if self.neo4j_conn:
for ip in results:
diff --git a/flowsint-transforms/src/flowsint_transforms/ip/to_ports.py b/flowsint-transforms/src/flowsint_transforms/ip/to_ports.py
index 2b9d2c33..6cb341d5 100644
--- a/flowsint-transforms/src/flowsint_transforms/ip/to_ports.py
+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_ports.py
@@ -12,8 +12,8 @@ class IpToPortsTransform(Transform):
"""[NAABU] Performs port scanning on IP addresses to discover open ports and services."""
# Define types as class attributes
- InputType = List[Ip]
- OutputType = List[Port]
+ InputType = Ip
+ OutputType = Port
def __init__(
self,
@@ -107,8 +107,8 @@ class IpToPortsTransform(Transform):
def key(cls) -> str:
return "address"
- async def scan(self, data: InputType) -> OutputType:
- results: OutputType = []
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
+ results: List[OutputType] = []
naabu = NaabuTool()
# Get parameters from transform config
@@ -189,8 +189,8 @@ class IpToPortsTransform(Transform):
return results
def postprocess(
- self, results: OutputType, input_data: InputType = None
- ) -> OutputType:
+ self, results: List[OutputType], input_data: List[InputType] = None
+ ) -> List[OutputType]:
"""Create Neo4j nodes for ports and relationships with IP addresses"""
if self.neo4j_conn and results:
for port in results:
diff --git a/flowsint-transforms/src/flowsint_transforms/n8n/connector.py b/flowsint-transforms/src/flowsint_transforms/n8n/connector.py
index 22289c1f..7c0ba3b7 100644
--- a/flowsint-transforms/src/flowsint_transforms/n8n/connector.py
+++ b/flowsint-transforms/src/flowsint_transforms/n8n/connector.py
@@ -12,8 +12,8 @@ class N8nConnector(Transform):
"""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[Any]
- OutputType = List[Any]
+ InputType = Any
+ OutputType = Any
def __init__(
self,
@@ -256,7 +256,7 @@ Check Flowsint logs for detailed debugging information.
},
]
- async def scan(self, values: InputType) -> OutputType:
+ async def scan(self, values: List[InputType]) -> List[OutputType]:
params = self.get_params()
url = params["webhook_url"]
Logger.info(self.sketch_id, {"message": f"n8n connector url: {url}"})
@@ -345,7 +345,7 @@ Check Flowsint logs for detailed debugging information.
# Re-raise the exception so the caller knows something went wrong
raise
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
Logger.success(
self.sketch_id, {"message": f"n8n connector results: {json.dumps(results)}"}
)
diff --git a/flowsint-transforms/src/flowsint_transforms/organization/to_asn.py b/flowsint-transforms/src/flowsint_transforms/organization/to_asn.py
index 645f4ebb..b1eb8986 100644
--- a/flowsint-transforms/src/flowsint_transforms/organization/to_asn.py
+++ b/flowsint-transforms/src/flowsint_transforms/organization/to_asn.py
@@ -12,8 +12,8 @@ class OrgToAsnTransform(Transform):
"""Takes an organization and returns its corresponding ASN."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[Organization]
- OutputType = List[ASN]
+ InputType = Organization
+ OutputType = ASN
def __init__(
self,
@@ -60,9 +60,9 @@ class OrgToAsnTransform(Transform):
def key(cls) -> str:
return "name"
- async def scan(self, data: InputType) -> OutputType:
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
"""Find ASN information for organizations using asnmap."""
- results: OutputType = []
+ results: List[OutputType] = []
asnmap = AsnmapTool()
# Retrieve API key from vault or environment
@@ -106,7 +106,7 @@ class OrgToAsnTransform(Transform):
return results
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
# Create Neo4j relationships between organizations and their corresponding ASNs
for input_org, result_asn in zip(original_input, results):
# Skip if no valid ASN was found
diff --git a/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py b/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py
index d0c7a205..bd396bce 100644
--- a/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py
+++ b/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py
@@ -20,8 +20,8 @@ class OrgToDomainsTransform(Transform):
"""[WHOXY] Takes an organization and returns the domains it registered."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[Organization]
- OutputType = List[Domain]
+ InputType = Organization
+ OutputType = Domain
def __init__(
self,
@@ -68,9 +68,9 @@ class OrgToDomainsTransform(Transform):
def key(cls) -> str:
return "name"
- async def scan(self, data: InputType) -> OutputType:
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
"""Find domains related to organizations using whoxy api."""
- domains: OutputType = []
+ domains: List[OutputType] = []
self._extracted_data = [] # Store all extracted data for postprocess
self._extracted_individuals = [] # Store extracted individuals for testing
self._extracted_organizations = [] # Store extracted organizations for testing
@@ -386,7 +386,7 @@ class OrgToDomainsTransform(Transform):
},
)
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
"""Create Neo4j nodes and relationships from extracted data."""
if not self.neo4j_conn:
Logger.info(
diff --git a/flowsint-transforms/src/flowsint_transforms/organization/to_infos.py b/flowsint-transforms/src/flowsint_transforms/organization/to_infos.py
index bdf72d46..a216aa05 100644
--- a/flowsint-transforms/src/flowsint_transforms/organization/to_infos.py
+++ b/flowsint-transforms/src/flowsint_transforms/organization/to_infos.py
@@ -9,8 +9,8 @@ class OrgToInfosTransform(Transform):
"""Enrich Organization with data from SIRENE (France only)."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[Organization]
- OutputType = List[Organization]
+ InputType = Organization
+ OutputType = Organization
@classmethod
def name(cls) -> str:
@@ -24,9 +24,9 @@ class OrgToInfosTransform(Transform):
def key(cls) -> str:
return "name"
- async def scan(self, data: InputType) -> OutputType:
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
- results: OutputType = []
+ results: List[OutputType] = []
for org in data:
try:
sirene = SireneTool()
@@ -246,7 +246,7 @@ class OrgToInfosTransform(Transform):
)
return None
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
if not self.neo4j_conn:
return results
diff --git a/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py b/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py
index 202b4552..dc54931e 100644
--- a/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py
+++ b/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py
@@ -10,8 +10,8 @@ import httpx
class IgnorantTransform(Transform):
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[Phone] # Phone objects
- OutputType = List[Dict[str, Any]] # Results as dictionaries
+ InputType = Phone # Phone objects
+ OutputType = Dict[str, Any] # Results as dictionaries
@classmethod
def name(cls) -> str:
@@ -25,11 +25,11 @@ class IgnorantTransform(Transform):
def key(cls) -> str:
return "number"
- async def scan(self, data: InputType) -> OutputType:
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
"""
Performs the Ignorant search for each specified phone number.
"""
- results: OutputType = []
+ results: List[OutputType] = []
for phone_obj in data:
try:
cleaned_phone = is_valid_number(phone_obj.number)
@@ -77,7 +77,7 @@ class IgnorantTransform(Transform):
except Exception as e:
return {"number": phone, "error": f"Error in Ignorant research: {str(e)}"}
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
"""
Create Neo4j relationships for found phone accounts.
"""
diff --git a/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py b/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py
index 67af127a..c22d3b18 100644
--- a/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py
+++ b/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py
@@ -14,8 +14,8 @@ class MaigretTransform(Transform):
"""[MAIGRET] Scans usernames for associated social accounts using Maigret."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[Username]
- OutputType = List[SocialAccount]
+ InputType = Username
+ OutputType = SocialAccount
@classmethod
def name(cls) -> str:
@@ -108,8 +108,8 @@ class MaigretTransform(Transform):
return results
- async def scan(self, data: InputType) -> OutputType:
- results: OutputType = []
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
+ results: List[OutputType] = []
for profile in data:
if not profile.value:
continue
@@ -118,7 +118,7 @@ class MaigretTransform(Transform):
results.extend(parsed)
return results
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
if not self.neo4j_conn:
return results
diff --git a/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py b/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py
index a2b4aa8b..17d58a84 100644
--- a/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py
+++ b/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py
@@ -11,8 +11,8 @@ class SherlockTransform(Transform):
"""[SHERLOCK] Scans the usernames for associated social accounts using Sherlock."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[Username]
- OutputType = List[SocialAccount]
+ InputType = Username
+ OutputType = SocialAccount
@classmethod
def name(cls) -> str:
@@ -26,9 +26,9 @@ class SherlockTransform(Transform):
def key(cls) -> str:
return "username"
- async def scan(self, data: InputType) -> OutputType:
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
"""Performs the scan using Sherlock on the list of usernames."""
- results: OutputType = []
+ results: List[OutputType] = []
for username in data:
output_file = Path(f"/tmp/sherlock_{username.value}.txt")
@@ -88,7 +88,7 @@ class SherlockTransform(Transform):
return results
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
"""Create Neo4j relationships for found social accounts."""
if not self.neo4j_conn:
return results
diff --git a/flowsint-transforms/src/flowsint_transforms/website/to_crawler.py b/flowsint-transforms/src/flowsint_transforms/website/to_crawler.py
index 72ffab1f..4c7a9f4d 100644
--- a/flowsint-transforms/src/flowsint_transforms/website/to_crawler.py
+++ b/flowsint-transforms/src/flowsint_transforms/website/to_crawler.py
@@ -19,8 +19,8 @@ class WebsiteToCrawler(Transform):
"""From website to crawler."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[Website]
- OutputType = List[ReturnType] # Simplified output type
+ InputType = Website
+ OutputType = ReturnType # Simplified output type
@classmethod
def name(cls) -> str:
@@ -43,7 +43,7 @@ class WebsiteToCrawler(Transform):
except Exception:
return False
- async def scan(self, data: InputType) -> OutputType:
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
"""Crawl websites to extract emails and phone numbers."""
results = []
@@ -134,7 +134,7 @@ class WebsiteToCrawler(Transform):
return results
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
# Create Neo4j relationships between websites and their corresponding emails and phones
for input_website, result in zip(original_input, results):
website_url = str(input_website.url)
diff --git a/flowsint-transforms/src/flowsint_transforms/website/to_domain.py b/flowsint-transforms/src/flowsint_transforms/website/to_domain.py
index ff0eda28..37ef9c40 100644
--- a/flowsint-transforms/src/flowsint_transforms/website/to_domain.py
+++ b/flowsint-transforms/src/flowsint_transforms/website/to_domain.py
@@ -10,8 +10,8 @@ class WebsiteToDomainTransform(Transform):
"""From website to domain."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[Website]
- OutputType = List[Domain]
+ InputType = Website
+ OutputType = Domain
@classmethod
def name(cls) -> str:
@@ -25,8 +25,8 @@ class WebsiteToDomainTransform(Transform):
def key(cls) -> str:
return "website"
- async def scan(self, data: InputType) -> OutputType:
- results: OutputType = []
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
+ results: List[OutputType] = []
for website in data:
try:
parsed_url = urlparse(str(website.url))
@@ -56,8 +56,8 @@ class WebsiteToDomainTransform(Transform):
return results
def postprocess(
- self, results: OutputType, input_data: InputType = None
- ) -> OutputType:
+ self, results: List[OutputType], input_data: List[InputType] = None
+ ) -> List[OutputType]:
# Create Neo4j relationships between websites and their corresponding domains
if input_data and self.neo4j_conn:
for input_website, result in zip(input_data, results):
diff --git a/flowsint-transforms/src/flowsint_transforms/website/to_links.py b/flowsint-transforms/src/flowsint_transforms/website/to_links.py
index 77164d7e..94cb29fb 100644
--- a/flowsint-transforms/src/flowsint_transforms/website/to_links.py
+++ b/flowsint-transforms/src/flowsint_transforms/website/to_links.py
@@ -11,8 +11,8 @@ class WebsiteToLinks(Transform):
"""From website to spread crawler that extracts domains and internal/external links."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[Website]
- OutputType = List[Website]
+ InputType = Website
+ OutputType = Website
@classmethod
def name(cls) -> str:
@@ -34,7 +34,7 @@ class WebsiteToLinks(Transform):
except Exception:
return ""
- async def scan(self, data: InputType) -> OutputType:
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
"""Crawl websites using reconspread to extract internal and external links."""
results = []
@@ -202,7 +202,7 @@ class WebsiteToLinks(Transform):
return results
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
# Neo4j nodes and relationships are created in real-time during the callback
# No additional processing needed here
return results
diff --git a/flowsint-transforms/src/flowsint_transforms/website/to_text.py b/flowsint-transforms/src/flowsint_transforms/website/to_text.py
index ae4918ab..91292346 100644
--- a/flowsint-transforms/src/flowsint_transforms/website/to_text.py
+++ b/flowsint-transforms/src/flowsint_transforms/website/to_text.py
@@ -9,8 +9,8 @@ from bs4 import BeautifulSoup
class WebsiteToText(Transform):
"""Extracts the texts in a webpage."""
- InputType = List[Website]
- OutputType = List[Phrase]
+ InputType = Website
+ OutputType = Phrase
@classmethod
def name(cls) -> str:
@@ -24,8 +24,8 @@ class WebsiteToText(Transform):
def key(cls) -> str:
return "website"
- async def scan(self, data: InputType) -> OutputType:
- results: OutputType = []
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
+ results: List[OutputType] = []
for website in data:
text_data = self._extract_text(website.url)
if text_data:
@@ -47,7 +47,7 @@ class WebsiteToText(Transform):
print(f"An error occurred: {e}")
return None
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
# Create Neo4j relationships between websites and their corresponding phrases
for input_website, result in zip(original_input, results):
website_url = str(input_website.url)
diff --git a/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py b/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py
index 338b4072..92512ab5 100644
--- a/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py
+++ b/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py
@@ -12,8 +12,8 @@ class WebsiteToWebtrackersTransform(Transform):
"""From website to webtrackers."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[Website]
- OutputType = List[WebTracker]
+ InputType = Website
+ OutputType = WebTracker
def __init__(
self,
@@ -39,8 +39,8 @@ class WebsiteToWebtrackersTransform(Transform):
def key(cls) -> str:
return "website"
- async def scan(self, data: InputType) -> OutputType:
- results: OutputType = []
+ async def scan(self, data: List[InputType]) -> List[OutputType]:
+ results: List[OutputType] = []
for website in data:
try:
@@ -70,7 +70,7 @@ class WebsiteToWebtrackersTransform(Transform):
return results
- def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
# Create Neo4j relationships between websites and their corresponding trackers
if self.neo4j_conn:
# Group trackers by website using the mapping we created during scan
diff --git a/flowsint-transforms/tests/test_vault_integration.py b/flowsint-transforms/tests/test_vault_integration.py
index 05ff283d..c332e6c5 100644
--- a/flowsint-transforms/tests/test_vault_integration.py
+++ b/flowsint-transforms/tests/test_vault_integration.py
@@ -227,44 +227,37 @@ class TestMissingRequiredVaultSecret:
"""Tests for error handling when required vault secrets are missing."""
@pytest.mark.asyncio
- async def test_missing_required_secret_logs_error(self, mock_vault, sketch_id):
- """Test that missing required secret logs an error message."""
+ async def test_missing_required_secret_raises_exception(self, mock_vault, sketch_id):
+ """Test that missing required secret raises an exception."""
# Vault returns None (secret not found)
mock_vault.get_secret.return_value = None
- from unittest.mock import patch
-
- with patch("flowsint_core.core.transform_base.Logger") as mock_logger:
- transform = DomainToHistoryTransform(
- sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}
- )
+ transform = DomainToHistoryTransform(
+ sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}
+ )
+ # async_init should raise an exception when required secret is missing
+ with pytest.raises(Exception) as exc_info:
await transform.async_init()
- # Verify error was logged
- assert mock_logger.error.called
- error_message = mock_logger.error.call_args[0][1]["message"]
- assert "WHOXY_API_KEY" in error_message
- assert "missing" in error_message.lower()
- assert "Vault settings" in error_message
+ # Verify the exception message contains the secret name
+ assert "WHOXY_API_KEY" in str(exc_info.value)
+ assert "missing" in str(exc_info.value).lower()
+ assert "Vault settings" in str(exc_info.value)
@pytest.mark.asyncio
- async def test_transform_can_still_run_with_fallback(self, mock_vault, sketch_id):
- """Test that transform can use environment variable fallback if vault fails."""
- import os
- from unittest.mock import patch
-
- # Vault returns None
- mock_vault.get_secret.return_value = None
+ async def test_transform_with_provided_secret_succeeds(self, mock_vault, sketch_id):
+ """Test that transform succeeds when required secret is provided."""
+ # Vault returns the secret
+ api_key = "whoxy-api-key-12345"
+ mock_vault.get_secret.return_value = api_key
- # Mock environment variable
- with patch.dict(os.environ, {"WHOXY_API_KEY": "fallback-key"}):
- transform = DomainToHistoryTransform(
- sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}
- )
+ transform = DomainToHistoryTransform(
+ sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}
+ )
- await transform.async_init()
+ # async_init should succeed
+ await transform.async_init()
- # Should be able to get the fallback value
- api_key = transform.get_secret("WHOXY_API_KEY", os.getenv("WHOXY_API_KEY"))
- assert api_key == "fallback-key"
+ # Secret should be accessible
+ assert transform.get_secret("WHOXY_API_KEY") == api_key
diff --git a/flowsint-types/README.md b/flowsint-types/README.md
index ae01189b..b75937f9 100644
--- a/flowsint-types/README.md
+++ b/flowsint-types/README.md
@@ -2,4 +2,10 @@
The types used in the flowsint transforms and core for flowsint.
+## Testing
+
+```bash
+poetry run pytest
+```
+
> ⚠️ 🚧 Work in progress !.
diff --git a/flowsint-types/tests/__init__.py b/flowsint-types/tests/__init__.py
new file mode 100644
index 00000000..846e4770
--- /dev/null
+++ b/flowsint-types/tests/__init__.py
@@ -0,0 +1 @@
+# Tests for flowsint-types
diff --git a/flowsint-types/tests/individual.py b/flowsint-types/tests/individual.py
deleted file mode 100644
index cfe81bdc..00000000
--- a/flowsint-types/tests/individual.py
+++ /dev/null
@@ -1,15 +0,0 @@
-from flowsint_types.individual import Individual
-import pytest
-
-
-def test_valid_individual():
- indivudual = Individual(first_name="John", last_name="Doe")
- assert indivudual.first_name == "John"
- assert indivudual.last_name == "Doe"
- assert indivudual.label == "John Doe"
-
-
-def test_invalid_individual():
- with pytest.raises(Exception) as e_info:
- Individual(name="John Doe")
- assert "validation errors for Individual" in str(e_info.value)
diff --git a/flowsint-types/tests/domain.py b/flowsint-types/tests/test_domain.py
similarity index 100%
rename from flowsint-types/tests/domain.py
rename to flowsint-types/tests/test_domain.py
diff --git a/flowsint-types/tests/ip.py b/flowsint-types/tests/test_ip.py
similarity index 100%
rename from flowsint-types/tests/ip.py
rename to flowsint-types/tests/test_ip.py
diff --git a/flowsint-types/tests/test_labels.py b/flowsint-types/tests/test_labels.py
new file mode 100644
index 00000000..10e92823
--- /dev/null
+++ b/flowsint-types/tests/test_labels.py
@@ -0,0 +1,16 @@
+from flowsint_types import Domain, Ip, Individual
+
+
+def test_ip_label():
+ ip = Ip(address="12.23.34.56")
+ assert ip.label == "12.23.34.56"
+
+
+def test_domain_label():
+ domain = Domain(domain="blog.mydomain.com")
+ assert domain.label == "blog.mydomain.com"
+
+
+def test_individual_label():
+ indivudual = Individual(first_name="John", last_name="Doe")
+ assert indivudual.label == "John Doe"