Skip to content

A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.

fix(core): keys are now only fetched from vault

dextmorgn7ab921f0da79
diff --git a/.env.example b/.env.example
index 6d2647eb..cb62c8d9 100644
--- a/.env.example
+++ b/.env.example
@@ -8,6 +8,3 @@ NEO4J_PASSWORD=password
VITE_API_URL=http://127.0.0.1:5001
DATABASE_URL=postgresql://flowsint:flowsint@localhost:5433/flowsint
REDIS_URL=redis://redis:6379/0
-HIBP_API_KEY=
-ETHERSCAN_API_KEY=
-WHOXY_API_KEY=
diff --git a/README.md b/README.md
index c822a7f6..cbebe525 100644
--- a/README.md
+++ b/README.md
@@ -21,15 +21,6 @@ cd flowsint
make prod
```
-Before running this command, setup your different API keys in the `.env.example` file:
-
-```bash
-HIBP_API_KEY= # HaveIBeenPwned
-ETHERSCAN_API_KEY= #Etherscan
-WHOXY_API_KEY= # Whoxy
-...
-```
-
Then go to [http://localhost:5173/register](http://localhost:5173/register) and create an account. There are no credentials or account by default.
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index b3e04ba5..4d2f21a7 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -85,9 +85,7 @@ services:
- NEO4J_USERNAME=${NEO4J_USERNAME}
- NEO4J_PASSWORD=${NEO4J_PASSWORD}
- AUTH_SECRET=${AUTH_SECRET}
- - HIBP_API_KEY=${HIBP_API_KEY}
- - ETHERSCAN_API_KEY=${ETHERSCAN_API_KEY}
- - WHOXY_API_KEY=${WHOXY_API_KEY}
+ - MASTER_VAULT_KEY_V1=${MASTER_VAULT_KEY_V1}
- REDIS_URL=redis://redis:6379/0
depends_on:
postgres:
@@ -119,6 +117,7 @@ services:
- NEO4J_URI_BOLT=bolt://neo4j:7687
- NEO4J_USERNAME=${NEO4J_USERNAME}
- NEO4J_PASSWORD=${NEO4J_PASSWORD}
+ - MASTER_VAULT_KEY_V1=${MASTER_VAULT_KEY_V1}
- REDIS_URL=redis://redis:6379/0
depends_on:
postgres:
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index d96add9a..b64c9bb3 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -82,9 +82,7 @@ services:
- NEO4J_USERNAME=${NEO4J_USERNAME}
- NEO4J_PASSWORD=${NEO4J_PASSWORD}
- AUTH_SECRET=${AUTH_SECRET}
- - HIBP_API_KEY=${HIBP_API_KEY}
- - ETHERSCAN_API_KEY=${ETHERSCAN_API_KEY}
- - WHOXY_API_KEY=${WHOXY_API_KEY}
+ - MASTER_VAULT_KEY_V1=${MASTER_VAULT_KEY_V1}
- REDIS_URL=redis://redis:6379/0
depends_on:
postgres:
@@ -112,6 +110,7 @@ services:
- NEO4J_URI_BOLT=bolt://neo4j:7687
- NEO4J_USERNAME=${NEO4J_USERNAME}
- NEO4J_PASSWORD=${NEO4J_PASSWORD}
+ - MASTER_VAULT_KEY_V1=${MASTER_VAULT_KEY_V1}
- REDIS_URL=redis://redis:6379/0
depends_on:
postgres:
diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py
index de0d4c16..5aef08bb 100644
--- a/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py
+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py
@@ -91,13 +91,8 @@ class CryptoWalletAddressToNFTs(Transform):
async def scan(self, data: InputType) -> OutputType:
results: OutputType = []
- params = self.get_params()
- Logger.debug(self.sketch_id, {"message": f"{str(params)}"})
- api_key = params.get("ETHERSCAN_API_KEY", os.getenv("ETHERSCAN_API_KEY"))
- api_url = params.get("ETHERSCAN_API_URL", None)
- if not api_key:
- Logger.error(self.sketch_id, {"message": "ETHERSCAN_API_KEY is required"})
- raise ValueError("ETHERSCAN_API_KEY is required")
+ api_key = self.get_secret("ETHERSCAN_API_KEY", os.getenv("ETHERSCAN_API_KEY"))
+ api_url = self.get_params().get("ETHERSCAN_API_URL", None)
for d in data:
try:
nfts = self._get_nfts(d.address, api_key, api_url)
diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py
index 0b10dc6a..9003fb1e 100644
--- a/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py
+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py
@@ -96,12 +96,8 @@ class CryptoWalletAddressToTransactions(Transform):
async def scan(self, data: InputType) -> OutputType:
results: OutputType = []
- params = self.get_params()
- api_key = params.get("ETHERSCAN_API_KEY", os.getenv("ETHERSCAN_API_KEY"))
- api_url = params.get("ETHERSCAN_API_URL", ETHERSCAN_API_URL)
- if not api_key:
- Logger.error(self.sketch_id, {"message": "ETHERSCAN_API_KEY is required"})
- raise ValueError("ETHERSCAN_API_KEY is required")
+ api_key = self.get_secret("ETHERSCAN_API_KEY", os.getenv("ETHERSCAN_API_KEY"))
+ api_url = self.get_params().get("ETHERSCAN_API_URL", ETHERSCAN_API_URL)
for d in data:
try:
transactions = await self._get_transactions(d.address, api_key, api_url)
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_history.py b/flowsint-transforms/src/flowsint_transforms/domain/to_history.py
index 545421d3..06440b93 100644
--- a/flowsint-transforms/src/flowsint_transforms/domain/to_history.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_history.py
@@ -1,7 +1,7 @@
import os
import re
import json
-from typing import Any, List, Union, Dict, Set
+from typing import Any, List, Union, Dict, Set, Optional
from flowsint_core.core.transform_base import Transform
from flowsint_types.domain import Domain
from flowsint_types.individual import Individual
@@ -9,6 +9,7 @@ from flowsint_types.organization import Organization
from flowsint_core.utils import is_valid_domain, is_root_domain
from flowsint_types.address import Location
from flowsint_core.core.logger import Logger
+from flowsint_core.core.graph_db import Neo4jConnection
from tools.network.whoxy import WhoxyTool
from dotenv import load_dotenv
@@ -24,6 +25,39 @@ class DomainToHistoryTransform(Transform):
InputType = List[Domain]
OutputType = List[Domain]
+ def __init__(
+ self,
+ sketch_id: Optional[str] = None,
+ scan_id: Optional[str] = None,
+ neo4j_conn: Optional[Neo4jConnection] = None,
+ vault=None,
+ params: Optional[Dict[str, Any]] = None,
+ ):
+ super().__init__(
+ sketch_id=sketch_id,
+ scan_id=scan_id,
+ neo4j_conn=neo4j_conn,
+ params_schema=self.get_params_schema(),
+ vault=vault,
+ params=params,
+ )
+
+ @classmethod
+ def required_params(cls) -> bool:
+ return True
+
+ @classmethod
+ def get_params_schema(cls) -> List[Dict[str, Any]]:
+ """Declare required parameters for this transform"""
+ return [
+ {
+ "name": "WHOXY_API_KEY",
+ "type": "vaultSecret",
+ "description": "The Whoxy API key to use for domain history lookups.",
+ "required": True,
+ },
+ ]
+
@classmethod
def name(cls) -> str:
return "domain_to_history"
@@ -61,9 +95,10 @@ class DomainToHistoryTransform(Transform):
self._extracted_data = [] # Store all extracted data for postprocess
self._extracted_individuals = [] # Store extracted individuals for testing
self._extracted_organizations = [] # Store extracted organizations for testing
+ api_key = self.get_secret("WHOXY_API_KEY", os.getenv("WHOXY_API_KEY"))
for domain in data:
- infos_data = self.__get_infos_from_whoxy(domain.domain)
+ infos_data = self.__get_infos_from_whoxy(domain.domain, api_key)
if infos_data and "whois_records" in infos_data:
Logger.info(
self.sketch_id,
@@ -180,7 +215,7 @@ class DomainToHistoryTransform(Transform):
},
)
- def __get_infos_from_whoxy(self, domain: str) -> Dict[str, Any]:
+ def __get_infos_from_whoxy(self, domain: str, api_key: str) -> Dict[str, Any]:
"""Get WHOIS history information from Whoxy API or test data."""
Logger.info(
self.sketch_id,
@@ -193,7 +228,7 @@ class DomainToHistoryTransform(Transform):
whoxy = WhoxyTool()
try:
params = {
- "key": WHOXY_API_KEY,
+ "key": api_key,
"history": domain,
}
infos = whoxy.launch(params=params)
diff --git a/flowsint-transforms/src/flowsint_transforms/email/to_domains.py b/flowsint-transforms/src/flowsint_transforms/email/to_domains.py
index a741a6dc..417e4f1c 100644
--- a/flowsint-transforms/src/flowsint_transforms/email/to_domains.py
+++ b/flowsint-transforms/src/flowsint_transforms/email/to_domains.py
@@ -1,12 +1,13 @@
import os
import re
-from typing import Any, List, Union, Dict, Set
+from typing import Any, List, Union, Dict, Set, Optional
from flowsint_core.core.transform_base import Transform
from flowsint_types.domain import Domain
from flowsint_types.individual import Individual
from flowsint_types.email import Email
from flowsint_types.address import Location
from flowsint_core.core.logger import Logger
+from flowsint_core.core.graph_db import Neo4jConnection
from tools.network.whoxy import WhoxyTool
from dotenv import load_dotenv
@@ -22,6 +23,39 @@ class EmailToDomainsTransform(Transform):
InputType = List[Email]
OutputType = List[Domain]
+ def __init__(
+ self,
+ sketch_id: Optional[str] = None,
+ scan_id: Optional[str] = None,
+ neo4j_conn: Optional[Neo4jConnection] = None,
+ vault=None,
+ params: Optional[Dict[str, Any]] = None,
+ ):
+ super().__init__(
+ sketch_id=sketch_id,
+ scan_id=scan_id,
+ neo4j_conn=neo4j_conn,
+ params_schema=self.get_params_schema(),
+ vault=vault,
+ params=params,
+ )
+
+ @classmethod
+ def required_params(cls) -> bool:
+ return True
+
+ @classmethod
+ def get_params_schema(cls) -> List[Dict[str, Any]]:
+ """Declare required parameters for this transform"""
+ return [
+ {
+ "name": "WHOXY_API_KEY",
+ "type": "vaultSecret",
+ "description": "The Whoxy API key to use for domain lookups.",
+ "required": True,
+ },
+ ]
+
@classmethod
def name(cls) -> str:
return "email_to_domains"
@@ -52,9 +86,10 @@ class EmailToDomainsTransform(Transform):
"""Find domains related to emails using whoxy api."""
domains: OutputType = []
self._extracted_data = [] # Store all extracted data for postprocess
+ api_key = self.get_secret("WHOXY_API_KEY", os.getenv("WHOXY_API_KEY"))
for email in data:
- infos_data = self.__get_infos_from_whoxy(email.email)
+ infos_data = self.__get_infos_from_whoxy(email.email, api_key)
if infos_data and "search_result" in infos_data:
# Process each domain result
for result in infos_data["search_result"]:
@@ -86,12 +121,12 @@ class EmailToDomainsTransform(Transform):
)
return domains
- def __get_infos_from_whoxy(self, email: str) -> Dict[str, Any]:
+ def __get_infos_from_whoxy(self, email: str, api_key: str) -> Dict[str, Any]:
infos: Dict[str, Any] = {}
whoxy = WhoxyTool()
try:
params = {
- "key": WHOXY_API_KEY,
+ "key": api_key,
"reverse": "whois",
"email": email,
}
diff --git a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py
index 34b78faa..610ccffe 100644
--- a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py
+++ b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py
@@ -62,8 +62,7 @@ class EmailToBreachesTransform(Transform):
"name": "HIBP_API_KEY",
"type": "vaultSecret",
"description": "The HIBP API key to use for breaches lookup.",
- "required": False,
- "default": HIBP_API_KEY,
+ "required": True,
},
{
"name": "HIBP_API_URL",
@@ -90,37 +89,18 @@ class EmailToBreachesTransform(Transform):
async def scan(self, data: InputType) -> OutputType:
results: OutputType = []
- print(self.get_params())
- api_key = self.get_params().get("HIBP_API_KEY", None)
- api_url = self.get_params().get("HIBP_API_URL", None)
- if not api_key:
- Logger.error(
- self.sketch_id,
- {"message": "A valid HIBP_API_KEY is required to scan for breaches."},
- )
- if not api_url:
- Logger.error(
- self.sketch_id, {"message": "Could not find HIBP_API_URL in params."}
- )
+ api_key = self.get_secret("HIBP_API_KEY", os.getenv("HIBP_API_KEY"))
+ api_url = self.get_params().get("HIBP_API_URL", "https://haveibeenpwned.com/api/v3/breachedaccount/")
headers = {"hibp-api-key": api_key, "User-Agent": "FlowsInt-Transform"}
- Logger.info(self.sketch_id, {"message": f"HIBP API URL: {api_url}"})
+
for email in data:
try:
# Query Have I Been Pwned API
full_url = urljoin(api_url, f"{email.email}?truncateResponse=false")
- Logger.error(self.sketch_id, {"message": f"full url: {full_url}"})
response = requests.get(full_url, headers=headers, timeout=10)
- Logger.info(
- self.sketch_id, {"message": f"HIBP API response: {response.json()}"}
- )
+
if response.status_code == 200:
breaches_data = response.json()
- Logger.info(
- self.sketch_id,
- {
- "message": f"Found {len(breaches_data)} breaches for {email.email}"
- },
- )
for breach_data in breaches_data:
breach = Breach(
name=breach_data.get("Name", ""),
@@ -139,21 +119,10 @@ class EmailToBreachesTransform(Transform):
isspamlist=breach_data.get("IsSpamList", False),
logopath=breach_data.get("LogoPath", ""),
)
- # Store email and breach as a tuple
results.append((email.email, breach))
- Logger.info(
- self.sketch_id,
- {
- "message": f"Added breach: {breach.name} for email: {email.email}"
- },
- )
elif response.status_code == 404:
# No breaches found for this email
- Logger.info(
- self.sketch_id,
- {"message": f"No breaches found for email {email.email}"},
- )
continue
else:
@@ -174,29 +143,14 @@ class EmailToBreachesTransform(Transform):
)
continue
- Logger.info(
- self.sketch_id,
- {"message": f"Scan completed. Total results: {len(results)}"},
- )
return results
def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
- Logger.info(
- self.sketch_id,
- {
- "message": f"Postprocess started. Results count: {len(results)}, Original input count: {len(original_input)}"
- },
- )
-
# Create email nodes first
for email_obj in original_input:
if not self.neo4j_conn:
continue
- # Create email node
self.create_node("email", "email", email_obj.email, **email_obj.__dict__)
- Logger.info(
- self.sketch_id, {"message": f"Created email node: {email_obj.email}"}
- )
# Process all breaches
for email_address, breach_obj in results:
@@ -213,9 +167,6 @@ class EmailToBreachesTransform(Transform):
label=breach_obj.name,
type="breach",
)
- Logger.info(
- self.sketch_id, {"message": f"Created breach node: {breach_key}"}
- )
# Create relationship between the specific email and this breach
self.create_relationship(
diff --git a/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py b/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py
index 6d61b3bf..d070ec27 100644
--- a/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py
+++ b/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py
@@ -1,12 +1,13 @@
import os
import re
-from typing import Any, List, Union, Dict, Set
+from typing import Any, List, Union, Dict, Set, Optional
from flowsint_core.core.transform_base import Transform
from flowsint_types.domain import Domain
from flowsint_types.organization import Organization
from flowsint_types.individual import Individual
from flowsint_types.address import Location
from flowsint_core.core.logger import Logger
+from flowsint_core.core.graph_db import Neo4jConnection
from tools.network.whoxy import WhoxyTool
from dotenv import load_dotenv
@@ -22,6 +23,39 @@ class IndividualToDomainsTransform(Transform):
InputType = List[Individual]
OutputType = List[Domain]
+ def __init__(
+ self,
+ sketch_id: Optional[str] = None,
+ scan_id: Optional[str] = None,
+ neo4j_conn: Optional[Neo4jConnection] = None,
+ vault=None,
+ params: Optional[Dict[str, Any]] = None,
+ ):
+ super().__init__(
+ sketch_id=sketch_id,
+ scan_id=scan_id,
+ neo4j_conn=neo4j_conn,
+ params_schema=self.get_params_schema(),
+ vault=vault,
+ params=params,
+ )
+
+ @classmethod
+ def required_params(cls) -> bool:
+ return True
+
+ @classmethod
+ def get_params_schema(cls) -> List[Dict[str, Any]]:
+ """Declare required parameters for this transform"""
+ return [
+ {
+ "name": "WHOXY_API_KEY",
+ "type": "vaultSecret",
+ "description": "The Whoxy API key to use for domain lookups.",
+ "required": True,
+ },
+ ]
+
@classmethod
def name(cls) -> str:
return "individual_to_domains"
@@ -68,9 +102,10 @@ class IndividualToDomainsTransform(Transform):
"""Find domains related to individuals using whoxy api."""
domains: OutputType = []
self._extracted_data = [] # Store all extracted data for postprocess
+ api_key = self.get_secret("WHOXY_API_KEY", os.getenv("WHOXY_API_KEY"))
for individual in data:
- infos_data = self.__get_infos_from_whoxy(individual.full_name)
+ infos_data = self.__get_infos_from_whoxy(individual.full_name, api_key)
if infos_data and "search_result" in infos_data:
# Process each domain result
for result in infos_data["search_result"]:
@@ -104,12 +139,12 @@ class IndividualToDomainsTransform(Transform):
)
return domains
- def __get_infos_from_whoxy(self, individual_name: str) -> Dict[str, Any]:
+ def __get_infos_from_whoxy(self, individual_name: str, api_key: str) -> Dict[str, Any]:
infos: Dict[str, Any] = {}
whoxy = WhoxyTool()
try:
params = {
- "key": WHOXY_API_KEY,
+ "key": api_key,
"reverse": "whois",
"name": individual_name,
}
diff --git a/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.py
index 55b0dba8..75556468 100644
--- a/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.py
+++ b/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.py
@@ -1,7 +1,8 @@
-from typing import Dict, Any, List, Union
+from typing import Dict, Any, List, Union, Optional
import hibpwned
from flowsint_core.core.transform_base import Transform
from flowsint_core.core.logger import Logger
+from flowsint_core.core.graph_db import Neo4jConnection
import os
from dotenv import load_dotenv
@@ -18,6 +19,39 @@ class HibpTransform(Transform):
InputType = List[str] # Email addresses as strings
OutputType = List[Dict[str, Any]] # Breach results as dictionaries
+ def __init__(
+ self,
+ sketch_id: Optional[str] = None,
+ scan_id: Optional[str] = None,
+ neo4j_conn: Optional[Neo4jConnection] = None,
+ vault=None,
+ params: Optional[Dict[str, Any]] = None,
+ ):
+ super().__init__(
+ sketch_id=sketch_id,
+ scan_id=scan_id,
+ neo4j_conn=neo4j_conn,
+ params_schema=self.get_params_schema(),
+ vault=vault,
+ params=params,
+ )
+
+ @classmethod
+ def required_params(cls) -> bool:
+ return True
+
+ @classmethod
+ def get_params_schema(cls) -> List[Dict[str, Any]]:
+ """Declare required parameters for this transform"""
+ return [
+ {
+ "name": "HIBP_API_KEY",
+ "type": "vaultSecret",
+ "description": "The HIBP API key to use for breach lookups.",
+ "required": True,
+ },
+ ]
+
@classmethod
def name(cls) -> str:
return "to_hibp_leaks"
@@ -42,9 +76,11 @@ class HibpTransform(Transform):
async def scan(self, data: InputType) -> OutputType:
"""Performs a search on HaveIBeenPwned for a list of emails."""
results: OutputType = []
+ api_key = self.get_secret("HIBP_API_KEY", os.getenv("HIBP_API_KEY"))
+
for email in data:
try:
- result = hibpwned.Pwned(email, "MyHIBPChecker", HIBP_API_KEY)
+ result = hibpwned.Pwned(email, "MyHIBPChecker", api_key)
# Clear data structure for results
breaches = result.search_all_breaches()
diff --git a/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.py
index e40cfe32..a75a8560 100644
--- a/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.py
+++ b/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.py
@@ -62,8 +62,7 @@ class PhoneToBreachesTransform(Transform):
"name": "HIBP_API_KEY",
"type": "vaultSecret",
"description": "The HIBP API key to use for breaches lookup.",
- "required": False,
- "default": HIBP_API_KEY,
+ "required": True,
},
{
"name": "HIBP_API_URL",
@@ -90,18 +89,8 @@ class PhoneToBreachesTransform(Transform):
async def scan(self, data: InputType) -> OutputType:
results: OutputType = []
- print(self.get_params())
- api_key = self.get_params().get("HIBP_API_KEY", None)
- api_url = self.get_params().get("HIBP_API_URL", None)
- if not api_key:
- Logger.error(
- self.sketch_id,
- {"message": "A valid HIBP_API_KEY is required to scan for breaches."},
- )
- if not api_url:
- Logger.error(
- self.sketch_id, {"message": "Could not find HIBP_API_URL in params."}
- )
+ api_key = self.get_secret("HIBP_API_KEY", os.getenv("HIBP_API_KEY"))
+ api_url = self.get_params().get("HIBP_API_URL", "https://haveibeenpwned.com/api/v3/breachedaccount/")
headers = {"hibp-api-key": api_key, "User-Agent": "FlowsInt-Transform"}
Logger.info(self.sketch_id, {"message": f"HIBP API URL: {api_url}"})
for phone in data:
diff --git a/flowsint-transforms/tests/test_vault_integration.py b/flowsint-transforms/tests/test_vault_integration.py
new file mode 100644
index 00000000..05ff283d
--- /dev/null
+++ b/flowsint-transforms/tests/test_vault_integration.py
@@ -0,0 +1,270 @@
+"""Integration tests for transforms using vault secrets."""
+import pytest
+import uuid
+from unittest.mock import Mock, MagicMock
+from flowsint_transforms.domain.to_history import DomainToHistoryTransform
+from flowsint_transforms.individual.to_domains import IndividualToDomainsTransform
+from flowsint_transforms.email.to_domains import EmailToDomainsTransform
+from flowsint_transforms.crypto.to_nfts import CryptoWalletAddressToNFTs
+from flowsint_transforms.crypto.to_transactions import CryptoWalletAddressToTransactions
+from flowsint_transforms.email.to_leaks import EmailToBreachesTransform
+
+
+def mock_vault():
+ """Create a mock vault instance."""
+ vault = Mock()
+ vault.get_secret = Mock()
+ return vault
+
+
+def sketch_id():
+ """Create a test sketch ID."""
+ return str(uuid.uuid4())
+
+
+class TestWhoxyTransformsVaultIntegration:
+ """Tests for WHOXY transforms vault integration."""
+
+ @pytest.mark.asyncio
+ async def test_domain_to_history_gets_api_key_from_vault(
+ self, mock_vault, sketch_id
+ ):
+ """Test that domain_to_history retrieves WHOXY_API_KEY from vault."""
+ api_key = "whoxy-api-key-12345"
+ mock_vault.get_secret.return_value = api_key
+
+ transform = DomainToHistoryTransform(
+ sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}
+ )
+
+ await transform.async_init()
+
+ # Verify vault was queried for the API key
+ mock_vault.get_secret.assert_called()
+ calls = [call[0][0] for call in mock_vault.get_secret.call_args_list]
+ assert "WHOXY_API_KEY" in calls
+
+ # Verify the key is accessible via get_secret
+ assert transform.get_secret("WHOXY_API_KEY") == api_key
+
+ @pytest.mark.asyncio
+ async def test_individual_to_domains_gets_api_key_from_vault(
+ self, mock_vault, sketch_id
+ ):
+ """Test that individual_to_domains retrieves WHOXY_API_KEY from vault."""
+ api_key = "whoxy-api-key-12345"
+ mock_vault.get_secret.return_value = api_key
+
+ transform = IndividualToDomainsTransform(
+ sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}
+ )
+
+ await transform.async_init()
+
+ # Verify vault was queried for the API key
+ mock_vault.get_secret.assert_called()
+ calls = [call[0][0] for call in mock_vault.get_secret.call_args_list]
+ assert "WHOXY_API_KEY" in calls
+
+ # Verify the key is accessible via get_secret
+ assert transform.get_secret("WHOXY_API_KEY") == api_key
+
+ @pytest.mark.asyncio
+ async def test_email_to_domains_gets_api_key_from_vault(
+ self, mock_vault, sketch_id
+ ):
+ """Test that email_to_domains retrieves WHOXY_API_KEY from vault."""
+ api_key = "whoxy-api-key-12345"
+ mock_vault.get_secret.return_value = api_key
+
+ transform = EmailToDomainsTransform(
+ sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}
+ )
+
+ await transform.async_init()
+
+ # Verify vault was queried for the API key
+ mock_vault.get_secret.assert_called()
+ calls = [call[0][0] for call in mock_vault.get_secret.call_args_list]
+ assert "WHOXY_API_KEY" in calls
+
+ # Verify the key is accessible via get_secret
+ assert transform.get_secret("WHOXY_API_KEY") == api_key
+
+
+class TestEtherscanTransformsVaultIntegration:
+ """Tests for Etherscan transforms vault integration."""
+
+ @pytest.mark.asyncio
+ async def test_wallet_to_nfts_gets_api_key_from_vault(self, mock_vault, sketch_id):
+ """Test that wallet_to_nfts retrieves ETHERSCAN_API_KEY from vault."""
+ api_key = "etherscan-api-key-12345"
+ mock_vault.get_secret.return_value = api_key
+
+ transform = CryptoWalletAddressToNFTs(
+ sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}
+ )
+
+ await transform.async_init()
+
+ # Verify vault was queried for the API key
+ mock_vault.get_secret.assert_called()
+ calls = [call[0][0] for call in mock_vault.get_secret.call_args_list]
+ assert "ETHERSCAN_API_KEY" in calls
+
+ # Verify the key is accessible via get_secret
+ assert transform.get_secret("ETHERSCAN_API_KEY") == api_key
+
+ @pytest.mark.asyncio
+ async def test_wallet_to_transactions_gets_api_key_from_vault(
+ self, mock_vault, sketch_id
+ ):
+ """Test that wallet_to_transactions retrieves ETHERSCAN_API_KEY from vault."""
+ api_key = "etherscan-api-key-12345"
+ mock_vault.get_secret.return_value = api_key
+
+ transform = CryptoWalletAddressToTransactions(
+ sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}
+ )
+
+ await transform.async_init()
+
+ # Verify vault was queried for the API key
+ mock_vault.get_secret.assert_called()
+ calls = [call[0][0] for call in mock_vault.get_secret.call_args_list]
+ assert "ETHERSCAN_API_KEY" in calls
+
+ # Verify the key is accessible via get_secret
+ assert transform.get_secret("ETHERSCAN_API_KEY") == api_key
+
+
+class TestHIBPTransformsVaultIntegration:
+ """Tests for HIBP transforms vault integration."""
+
+ @pytest.mark.asyncio
+ async def test_email_to_breaches_gets_api_key_from_vault(
+ self, mock_vault, sketch_id
+ ):
+ """Test that email_to_breaches retrieves HIBP_API_KEY from vault."""
+ api_key = "hibp-api-key-12345"
+ mock_vault.get_secret.return_value = api_key
+
+ transform = EmailToBreachesTransform(
+ sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}
+ )
+
+ await transform.async_init()
+
+ # Verify vault was queried for the API key
+ mock_vault.get_secret.assert_called()
+ calls = [call[0][0] for call in mock_vault.get_secret.call_args_list]
+ assert "HIBP_API_KEY" in calls
+
+ # Verify the key is accessible via get_secret
+ assert transform.get_secret("HIBP_API_KEY") == api_key
+
+
+class TestVaultSecretWithUserProvidedID:
+ """Tests for vault secrets when user provides a specific vault ID."""
+
+ @pytest.mark.asyncio
+ async def test_transform_uses_user_provided_vault_id(self, mock_vault, sketch_id):
+ """Test that transform uses user-provided vault ID if specified."""
+ vault_id = str(uuid.uuid4())
+ api_key = "whoxy-api-key-12345"
+
+ # First call (by ID) returns the secret
+ mock_vault.get_secret.return_value = api_key
+
+ transform = DomainToHistoryTransform(
+ sketch_id=sketch_id,
+ scan_id="scan_123",
+ vault=mock_vault,
+ params={"WHOXY_API_KEY": vault_id}, # User selected a specific vault entry
+ )
+
+ await transform.async_init()
+
+ # Verify vault was queried with the provided ID first
+ first_call = mock_vault.get_secret.call_args_list[0][0][0]
+ assert first_call == vault_id
+
+ # Verify the key is accessible
+ assert transform.get_secret("WHOXY_API_KEY") == api_key
+
+ @pytest.mark.asyncio
+ async def test_transform_fallback_to_name_if_id_not_found(
+ self, mock_vault, sketch_id
+ ):
+ """Test that transform falls back to name lookup if vault ID not found."""
+ vault_id = str(uuid.uuid4())
+ api_key = "whoxy-api-key-12345"
+
+ # First call (by ID) returns None, second call (by name) returns the secret
+ mock_vault.get_secret.side_effect = [None, api_key]
+
+ transform = DomainToHistoryTransform(
+ sketch_id=sketch_id,
+ scan_id="scan_123",
+ vault=mock_vault,
+ params={"WHOXY_API_KEY": vault_id},
+ )
+
+ await transform.async_init()
+
+ # Verify vault was queried twice: first by ID, then by name
+ assert mock_vault.get_secret.call_count == 2
+ assert mock_vault.get_secret.call_args_list[0][0][0] == vault_id
+ assert mock_vault.get_secret.call_args_list[1][0][0] == "WHOXY_API_KEY"
+
+ # Verify the key is accessible
+ assert transform.get_secret("WHOXY_API_KEY") == api_key
+
+
+class TestMissingRequiredVaultSecret:
+ """Tests for error handling when required vault secrets are missing."""
+
+ @pytest.mark.asyncio
+ async def test_missing_required_secret_logs_error(self, mock_vault, sketch_id):
+ """Test that missing required secret logs an error message."""
+ # Vault returns None (secret not found)
+ mock_vault.get_secret.return_value = None
+
+ from unittest.mock import patch
+
+ with patch("flowsint_core.core.transform_base.Logger") as mock_logger:
+ transform = DomainToHistoryTransform(
+ sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}
+ )
+
+ await transform.async_init()
+
+ # Verify error was logged
+ assert mock_logger.error.called
+ error_message = mock_logger.error.call_args[0][1]["message"]
+ assert "WHOXY_API_KEY" in error_message
+ assert "missing" in error_message.lower()
+ assert "Vault settings" in error_message
+
+ @pytest.mark.asyncio
+ async def test_transform_can_still_run_with_fallback(self, mock_vault, sketch_id):
+ """Test that transform can use environment variable fallback if vault fails."""
+ import os
+ from unittest.mock import patch
+
+ # Vault returns None
+ mock_vault.get_secret.return_value = None
+
+ # Mock environment variable
+ with patch.dict(os.environ, {"WHOXY_API_KEY": "fallback-key"}):
+ transform = DomainToHistoryTransform(
+ sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}
+ )
+
+ await transform.async_init()
+
+ # Should be able to get the fallback value
+ api_key = transform.get_secret("WHOXY_API_KEY", os.getenv("WHOXY_API_KEY"))
+ assert api_key == "fallback-key"