A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
fix(core): keys are now only fetched from vault
dextmorgn7ab921f0da79
diff --git a/.env.example b/.env.exampleindex 6d2647eb..cb62c8d9 100644--- a/.env.example+++ b/.env.example@@ -8,6 +8,3 @@ NEO4J_PASSWORD=passwordVITE_API_URL=http://127.0.0.1:5001DATABASE_URL=postgresql://flowsint:flowsint@localhost:5433/flowsintREDIS_URL=redis://redis:6379/0-HIBP_API_KEY=-ETHERSCAN_API_KEY=-WHOXY_API_KEY=diff --git a/README.md b/README.mdindex c822a7f6..cbebe525 100644--- a/README.md+++ b/README.md@@ -21,15 +21,6 @@ cd flowsintmake prod```-Before running this command, setup your different API keys in the `.env.example` file:--```bash-HIBP_API_KEY= # HaveIBeenPwned-ETHERSCAN_API_KEY= #Etherscan-WHOXY_API_KEY= # Whoxy-...-```-Then go to [http://localhost:5173/register](http://localhost:5173/register) and create an account. There are no credentials or account by default.diff --git a/docker-compose.dev.yml b/docker-compose.dev.ymlindex b3e04ba5..4d2f21a7 100644--- a/docker-compose.dev.yml+++ b/docker-compose.dev.yml@@ -85,9 +85,7 @@ services:- NEO4J_USERNAME=${NEO4J_USERNAME}- NEO4J_PASSWORD=${NEO4J_PASSWORD}- AUTH_SECRET=${AUTH_SECRET}- - HIBP_API_KEY=${HIBP_API_KEY}- - ETHERSCAN_API_KEY=${ETHERSCAN_API_KEY}- - WHOXY_API_KEY=${WHOXY_API_KEY}+ - MASTER_VAULT_KEY_V1=${MASTER_VAULT_KEY_V1}- REDIS_URL=redis://redis:6379/0depends_on:postgres:@@ -119,6 +117,7 @@ services:- NEO4J_URI_BOLT=bolt://neo4j:7687- NEO4J_USERNAME=${NEO4J_USERNAME}- NEO4J_PASSWORD=${NEO4J_PASSWORD}+ - MASTER_VAULT_KEY_V1=${MASTER_VAULT_KEY_V1}- REDIS_URL=redis://redis:6379/0depends_on:postgres:diff --git a/docker-compose.prod.yml b/docker-compose.prod.ymlindex d96add9a..b64c9bb3 100644--- a/docker-compose.prod.yml+++ b/docker-compose.prod.yml@@ -82,9 +82,7 @@ services:- NEO4J_USERNAME=${NEO4J_USERNAME}- NEO4J_PASSWORD=${NEO4J_PASSWORD}- AUTH_SECRET=${AUTH_SECRET}- - HIBP_API_KEY=${HIBP_API_KEY}- - ETHERSCAN_API_KEY=${ETHERSCAN_API_KEY}- - WHOXY_API_KEY=${WHOXY_API_KEY}+ - MASTER_VAULT_KEY_V1=${MASTER_VAULT_KEY_V1}- REDIS_URL=redis://redis:6379/0depends_on:postgres:@@ -112,6 +110,7 @@ services:- NEO4J_URI_BOLT=bolt://neo4j:7687- NEO4J_USERNAME=${NEO4J_USERNAME}- NEO4J_PASSWORD=${NEO4J_PASSWORD}+ - MASTER_VAULT_KEY_V1=${MASTER_VAULT_KEY_V1}- REDIS_URL=redis://redis:6379/0depends_on:postgres:diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.pyindex de0d4c16..5aef08bb 100644--- a/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py@@ -91,13 +91,8 @@ class CryptoWalletAddressToNFTs(Transform):async def scan(self, data: InputType) -> OutputType:results: OutputType = []- params = self.get_params()- Logger.debug(self.sketch_id, {"message": f"{str(params)}"})- api_key = params.get("ETHERSCAN_API_KEY", os.getenv("ETHERSCAN_API_KEY"))- api_url = params.get("ETHERSCAN_API_URL", None)- if not api_key:- Logger.error(self.sketch_id, {"message": "ETHERSCAN_API_KEY is required"})- raise ValueError("ETHERSCAN_API_KEY is required")+ api_key = self.get_secret("ETHERSCAN_API_KEY", os.getenv("ETHERSCAN_API_KEY"))+ api_url = self.get_params().get("ETHERSCAN_API_URL", None)for d in data:try:nfts = self._get_nfts(d.address, api_key, api_url)diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.pyindex 0b10dc6a..9003fb1e 100644--- a/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py@@ -96,12 +96,8 @@ class CryptoWalletAddressToTransactions(Transform):async def scan(self, data: InputType) -> OutputType:results: OutputType = []- params = self.get_params()- api_key = params.get("ETHERSCAN_API_KEY", os.getenv("ETHERSCAN_API_KEY"))- api_url = params.get("ETHERSCAN_API_URL", ETHERSCAN_API_URL)- if not api_key:- Logger.error(self.sketch_id, {"message": "ETHERSCAN_API_KEY is required"})- raise ValueError("ETHERSCAN_API_KEY is required")+ api_key = self.get_secret("ETHERSCAN_API_KEY", os.getenv("ETHERSCAN_API_KEY"))+ api_url = self.get_params().get("ETHERSCAN_API_URL", ETHERSCAN_API_URL)for d in data:try:transactions = await self._get_transactions(d.address, api_key, api_url)diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_history.py b/flowsint-transforms/src/flowsint_transforms/domain/to_history.pyindex 545421d3..06440b93 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_history.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_history.py@@ -1,7 +1,7 @@import osimport reimport json-from typing import Any, List, Union, Dict, Set+from typing import Any, List, Union, Dict, Set, Optionalfrom flowsint_core.core.transform_base import Transformfrom flowsint_types.domain import Domainfrom flowsint_types.individual import Individual@@ -9,6 +9,7 @@ from flowsint_types.organization import Organizationfrom flowsint_core.utils import is_valid_domain, is_root_domainfrom flowsint_types.address import Locationfrom flowsint_core.core.logger import Logger+from flowsint_core.core.graph_db import Neo4jConnectionfrom tools.network.whoxy import WhoxyToolfrom dotenv import load_dotenv@@ -24,6 +25,39 @@ class DomainToHistoryTransform(Transform):InputType = List[Domain]OutputType = List[Domain]+ def __init__(+ self,+ sketch_id: Optional[str] = None,+ scan_id: Optional[str] = None,+ neo4j_conn: Optional[Neo4jConnection] = None,+ vault=None,+ params: Optional[Dict[str, Any]] = None,+ ):+ super().__init__(+ sketch_id=sketch_id,+ scan_id=scan_id,+ neo4j_conn=neo4j_conn,+ params_schema=self.get_params_schema(),+ vault=vault,+ params=params,+ )++ @classmethod+ def required_params(cls) -> bool:+ return True++ @classmethod+ def get_params_schema(cls) -> List[Dict[str, Any]]:+ """Declare required parameters for this transform"""+ return [+ {+ "name": "WHOXY_API_KEY",+ "type": "vaultSecret",+ "description": "The Whoxy API key to use for domain history lookups.",+ "required": True,+ },+ ]+@classmethoddef name(cls) -> str:return "domain_to_history"@@ -61,9 +95,10 @@ class DomainToHistoryTransform(Transform):self._extracted_data = [] # Store all extracted data for postprocessself._extracted_individuals = [] # Store extracted individuals for testingself._extracted_organizations = [] # Store extracted organizations for testing+ api_key = self.get_secret("WHOXY_API_KEY", os.getenv("WHOXY_API_KEY"))for domain in data:- infos_data = self.__get_infos_from_whoxy(domain.domain)+ infos_data = self.__get_infos_from_whoxy(domain.domain, api_key)if infos_data and "whois_records" in infos_data:Logger.info(self.sketch_id,@@ -180,7 +215,7 @@ class DomainToHistoryTransform(Transform):},)- def __get_infos_from_whoxy(self, domain: str) -> Dict[str, Any]:+ def __get_infos_from_whoxy(self, domain: str, api_key: str) -> Dict[str, Any]:"""Get WHOIS history information from Whoxy API or test data."""Logger.info(self.sketch_id,@@ -193,7 +228,7 @@ class DomainToHistoryTransform(Transform):whoxy = WhoxyTool()try:params = {- "key": WHOXY_API_KEY,+ "key": api_key,"history": domain,}infos = whoxy.launch(params=params)diff --git a/flowsint-transforms/src/flowsint_transforms/email/to_domains.py b/flowsint-transforms/src/flowsint_transforms/email/to_domains.pyindex a741a6dc..417e4f1c 100644--- a/flowsint-transforms/src/flowsint_transforms/email/to_domains.py+++ b/flowsint-transforms/src/flowsint_transforms/email/to_domains.py@@ -1,12 +1,13 @@import osimport re-from typing import Any, List, Union, Dict, Set+from typing import Any, List, Union, Dict, Set, Optionalfrom flowsint_core.core.transform_base import Transformfrom flowsint_types.domain import Domainfrom flowsint_types.individual import Individualfrom flowsint_types.email import Emailfrom flowsint_types.address import Locationfrom flowsint_core.core.logger import Logger+from flowsint_core.core.graph_db import Neo4jConnectionfrom tools.network.whoxy import WhoxyToolfrom dotenv import load_dotenv@@ -22,6 +23,39 @@ class EmailToDomainsTransform(Transform):InputType = List[Email]OutputType = List[Domain]+ def __init__(+ self,+ sketch_id: Optional[str] = None,+ scan_id: Optional[str] = None,+ neo4j_conn: Optional[Neo4jConnection] = None,+ vault=None,+ params: Optional[Dict[str, Any]] = None,+ ):+ super().__init__(+ sketch_id=sketch_id,+ scan_id=scan_id,+ neo4j_conn=neo4j_conn,+ params_schema=self.get_params_schema(),+ vault=vault,+ params=params,+ )++ @classmethod+ def required_params(cls) -> bool:+ return True++ @classmethod+ def get_params_schema(cls) -> List[Dict[str, Any]]:+ """Declare required parameters for this transform"""+ return [+ {+ "name": "WHOXY_API_KEY",+ "type": "vaultSecret",+ "description": "The Whoxy API key to use for domain lookups.",+ "required": True,+ },+ ]+@classmethoddef name(cls) -> str:return "email_to_domains"@@ -52,9 +86,10 @@ class EmailToDomainsTransform(Transform):"""Find domains related to emails using whoxy api."""domains: OutputType = []self._extracted_data = [] # Store all extracted data for postprocess+ api_key = self.get_secret("WHOXY_API_KEY", os.getenv("WHOXY_API_KEY"))for email in data:- infos_data = self.__get_infos_from_whoxy(email.email)+ infos_data = self.__get_infos_from_whoxy(email.email, api_key)if infos_data and "search_result" in infos_data:# Process each domain resultfor result in infos_data["search_result"]:@@ -86,12 +121,12 @@ class EmailToDomainsTransform(Transform):)return domains- def __get_infos_from_whoxy(self, email: str) -> Dict[str, Any]:+ def __get_infos_from_whoxy(self, email: str, api_key: str) -> Dict[str, Any]:infos: Dict[str, Any] = {}whoxy = WhoxyTool()try:params = {- "key": WHOXY_API_KEY,+ "key": api_key,"reverse": "whois","email": email,}diff --git a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.pyindex 34b78faa..610ccffe 100644--- a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py+++ b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py@@ -62,8 +62,7 @@ class EmailToBreachesTransform(Transform):"name": "HIBP_API_KEY","type": "vaultSecret","description": "The HIBP API key to use for breaches lookup.",- "required": False,- "default": HIBP_API_KEY,+ "required": True,},{"name": "HIBP_API_URL",@@ -90,37 +89,18 @@ class EmailToBreachesTransform(Transform):async def scan(self, data: InputType) -> OutputType:results: OutputType = []- print(self.get_params())- api_key = self.get_params().get("HIBP_API_KEY", None)- api_url = self.get_params().get("HIBP_API_URL", None)- if not api_key:- Logger.error(- self.sketch_id,- {"message": "A valid HIBP_API_KEY is required to scan for breaches."},- )- if not api_url:- Logger.error(- self.sketch_id, {"message": "Could not find HIBP_API_URL in params."}- )+ api_key = self.get_secret("HIBP_API_KEY", os.getenv("HIBP_API_KEY"))+ api_url = self.get_params().get("HIBP_API_URL", "https://haveibeenpwned.com/api/v3/breachedaccount/")headers = {"hibp-api-key": api_key, "User-Agent": "FlowsInt-Transform"}- Logger.info(self.sketch_id, {"message": f"HIBP API URL: {api_url}"})+for email in data:try:# Query Have I Been Pwned APIfull_url = urljoin(api_url, f"{email.email}?truncateResponse=false")- Logger.error(self.sketch_id, {"message": f"full url: {full_url}"})response = requests.get(full_url, headers=headers, timeout=10)- Logger.info(- self.sketch_id, {"message": f"HIBP API response: {response.json()}"}- )+if response.status_code == 200:breaches_data = response.json()- Logger.info(- self.sketch_id,- {- "message": f"Found {len(breaches_data)} breaches for {email.email}"- },- )for breach_data in breaches_data:breach = Breach(name=breach_data.get("Name", ""),@@ -139,21 +119,10 @@ class EmailToBreachesTransform(Transform):isspamlist=breach_data.get("IsSpamList", False),logopath=breach_data.get("LogoPath", ""),)- # Store email and breach as a tupleresults.append((email.email, breach))- Logger.info(- self.sketch_id,- {- "message": f"Added breach: {breach.name} for email: {email.email}"- },- )elif response.status_code == 404:# No breaches found for this email- Logger.info(- self.sketch_id,- {"message": f"No breaches found for email {email.email}"},- )continueelse:@@ -174,29 +143,14 @@ class EmailToBreachesTransform(Transform):)continue- Logger.info(- self.sketch_id,- {"message": f"Scan completed. Total results: {len(results)}"},- )return resultsdef postprocess(self, results: OutputType, original_input: InputType) -> OutputType:- Logger.info(- self.sketch_id,- {- "message": f"Postprocess started. Results count: {len(results)}, Original input count: {len(original_input)}"- },- )-# Create email nodes firstfor email_obj in original_input:if not self.neo4j_conn:continue- # Create email nodeself.create_node("email", "email", email_obj.email, **email_obj.__dict__)- Logger.info(- self.sketch_id, {"message": f"Created email node: {email_obj.email}"}- )# Process all breachesfor email_address, breach_obj in results:@@ -213,9 +167,6 @@ class EmailToBreachesTransform(Transform):label=breach_obj.name,type="breach",)- Logger.info(- self.sketch_id, {"message": f"Created breach node: {breach_key}"}- )# Create relationship between the specific email and this breachself.create_relationship(diff --git a/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py b/flowsint-transforms/src/flowsint_transforms/individual/to_domains.pyindex 6d61b3bf..d070ec27 100644--- a/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py+++ b/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py@@ -1,12 +1,13 @@import osimport re-from typing import Any, List, Union, Dict, Set+from typing import Any, List, Union, Dict, Set, Optionalfrom flowsint_core.core.transform_base import Transformfrom flowsint_types.domain import Domainfrom flowsint_types.organization import Organizationfrom flowsint_types.individual import Individualfrom flowsint_types.address import Locationfrom flowsint_core.core.logger import Logger+from flowsint_core.core.graph_db import Neo4jConnectionfrom tools.network.whoxy import WhoxyToolfrom dotenv import load_dotenv@@ -22,6 +23,39 @@ class IndividualToDomainsTransform(Transform):InputType = List[Individual]OutputType = List[Domain]+ def __init__(+ self,+ sketch_id: Optional[str] = None,+ scan_id: Optional[str] = None,+ neo4j_conn: Optional[Neo4jConnection] = None,+ vault=None,+ params: Optional[Dict[str, Any]] = None,+ ):+ super().__init__(+ sketch_id=sketch_id,+ scan_id=scan_id,+ neo4j_conn=neo4j_conn,+ params_schema=self.get_params_schema(),+ vault=vault,+ params=params,+ )++ @classmethod+ def required_params(cls) -> bool:+ return True++ @classmethod+ def get_params_schema(cls) -> List[Dict[str, Any]]:+ """Declare required parameters for this transform"""+ return [+ {+ "name": "WHOXY_API_KEY",+ "type": "vaultSecret",+ "description": "The Whoxy API key to use for domain lookups.",+ "required": True,+ },+ ]+@classmethoddef name(cls) -> str:return "individual_to_domains"@@ -68,9 +102,10 @@ class IndividualToDomainsTransform(Transform):"""Find domains related to individuals using whoxy api."""domains: OutputType = []self._extracted_data = [] # Store all extracted data for postprocess+ api_key = self.get_secret("WHOXY_API_KEY", os.getenv("WHOXY_API_KEY"))for individual in data:- infos_data = self.__get_infos_from_whoxy(individual.full_name)+ infos_data = self.__get_infos_from_whoxy(individual.full_name, api_key)if infos_data and "search_result" in infos_data:# Process each domain resultfor result in infos_data["search_result"]:@@ -104,12 +139,12 @@ class IndividualToDomainsTransform(Transform):)return domains- def __get_infos_from_whoxy(self, individual_name: str) -> Dict[str, Any]:+ def __get_infos_from_whoxy(self, individual_name: str, api_key: str) -> Dict[str, Any]:infos: Dict[str, Any] = {}whoxy = WhoxyTool()try:params = {- "key": WHOXY_API_KEY,+ "key": api_key,"reverse": "whois","name": individual_name,}diff --git a/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.pyindex 55b0dba8..75556468 100644--- a/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.py+++ b/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.py@@ -1,7 +1,8 @@-from typing import Dict, Any, List, Union+from typing import Dict, Any, List, Union, Optionalimport hibpwnedfrom flowsint_core.core.transform_base import Transformfrom flowsint_core.core.logger import Logger+from flowsint_core.core.graph_db import Neo4jConnectionimport osfrom dotenv import load_dotenv@@ -18,6 +19,39 @@ class HibpTransform(Transform):InputType = List[str] # Email addresses as stringsOutputType = List[Dict[str, Any]] # Breach results as dictionaries+ def __init__(+ self,+ sketch_id: Optional[str] = None,+ scan_id: Optional[str] = None,+ neo4j_conn: Optional[Neo4jConnection] = None,+ vault=None,+ params: Optional[Dict[str, Any]] = None,+ ):+ super().__init__(+ sketch_id=sketch_id,+ scan_id=scan_id,+ neo4j_conn=neo4j_conn,+ params_schema=self.get_params_schema(),+ vault=vault,+ params=params,+ )++ @classmethod+ def required_params(cls) -> bool:+ return True++ @classmethod+ def get_params_schema(cls) -> List[Dict[str, Any]]:+ """Declare required parameters for this transform"""+ return [+ {+ "name": "HIBP_API_KEY",+ "type": "vaultSecret",+ "description": "The HIBP API key to use for breach lookups.",+ "required": True,+ },+ ]+@classmethoddef name(cls) -> str:return "to_hibp_leaks"@@ -42,9 +76,11 @@ class HibpTransform(Transform):async def scan(self, data: InputType) -> OutputType:"""Performs a search on HaveIBeenPwned for a list of emails."""results: OutputType = []+ api_key = self.get_secret("HIBP_API_KEY", os.getenv("HIBP_API_KEY"))+for email in data:try:- result = hibpwned.Pwned(email, "MyHIBPChecker", HIBP_API_KEY)+ result = hibpwned.Pwned(email, "MyHIBPChecker", api_key)# Clear data structure for resultsbreaches = result.search_all_breaches()diff --git a/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.pyindex e40cfe32..a75a8560 100644--- a/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.py+++ b/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.py@@ -62,8 +62,7 @@ class PhoneToBreachesTransform(Transform):"name": "HIBP_API_KEY","type": "vaultSecret","description": "The HIBP API key to use for breaches lookup.",- "required": False,- "default": HIBP_API_KEY,+ "required": True,},{"name": "HIBP_API_URL",@@ -90,18 +89,8 @@ class PhoneToBreachesTransform(Transform):async def scan(self, data: InputType) -> OutputType:results: OutputType = []- print(self.get_params())- api_key = self.get_params().get("HIBP_API_KEY", None)- api_url = self.get_params().get("HIBP_API_URL", None)- if not api_key:- Logger.error(- self.sketch_id,- {"message": "A valid HIBP_API_KEY is required to scan for breaches."},- )- if not api_url:- Logger.error(- self.sketch_id, {"message": "Could not find HIBP_API_URL in params."}- )+ api_key = self.get_secret("HIBP_API_KEY", os.getenv("HIBP_API_KEY"))+ api_url = self.get_params().get("HIBP_API_URL", "https://haveibeenpwned.com/api/v3/breachedaccount/")headers = {"hibp-api-key": api_key, "User-Agent": "FlowsInt-Transform"}Logger.info(self.sketch_id, {"message": f"HIBP API URL: {api_url}"})for phone in data:diff --git a/flowsint-transforms/tests/test_vault_integration.py b/flowsint-transforms/tests/test_vault_integration.pynew file mode 100644index 00000000..05ff283d--- /dev/null+++ b/flowsint-transforms/tests/test_vault_integration.py@@ -0,0 +1,270 @@+"""Integration tests for transforms using vault secrets."""+import pytest+import uuid+from unittest.mock import Mock, MagicMock+from flowsint_transforms.domain.to_history import DomainToHistoryTransform+from flowsint_transforms.individual.to_domains import IndividualToDomainsTransform+from flowsint_transforms.email.to_domains import EmailToDomainsTransform+from flowsint_transforms.crypto.to_nfts import CryptoWalletAddressToNFTs+from flowsint_transforms.crypto.to_transactions import CryptoWalletAddressToTransactions+from flowsint_transforms.email.to_leaks import EmailToBreachesTransform+++def mock_vault():+ """Create a mock vault instance."""+ vault = Mock()+ vault.get_secret = Mock()+ return vault+++def sketch_id():+ """Create a test sketch ID."""+ return str(uuid.uuid4())+++class TestWhoxyTransformsVaultIntegration:+ """Tests for WHOXY transforms vault integration."""++ @pytest.mark.asyncio+ async def test_domain_to_history_gets_api_key_from_vault(+ self, mock_vault, sketch_id+ ):+ """Test that domain_to_history retrieves WHOXY_API_KEY from vault."""+ api_key = "whoxy-api-key-12345"+ mock_vault.get_secret.return_value = api_key++ transform = DomainToHistoryTransform(+ sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}+ )++ await transform.async_init()++ # Verify vault was queried for the API key+ mock_vault.get_secret.assert_called()+ calls = [call[0][0] for call in mock_vault.get_secret.call_args_list]+ assert "WHOXY_API_KEY" in calls++ # Verify the key is accessible via get_secret+ assert transform.get_secret("WHOXY_API_KEY") == api_key++ @pytest.mark.asyncio+ async def test_individual_to_domains_gets_api_key_from_vault(+ self, mock_vault, sketch_id+ ):+ """Test that individual_to_domains retrieves WHOXY_API_KEY from vault."""+ api_key = "whoxy-api-key-12345"+ mock_vault.get_secret.return_value = api_key++ transform = IndividualToDomainsTransform(+ sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}+ )++ await transform.async_init()++ # Verify vault was queried for the API key+ mock_vault.get_secret.assert_called()+ calls = [call[0][0] for call in mock_vault.get_secret.call_args_list]+ assert "WHOXY_API_KEY" in calls++ # Verify the key is accessible via get_secret+ assert transform.get_secret("WHOXY_API_KEY") == api_key++ @pytest.mark.asyncio+ async def test_email_to_domains_gets_api_key_from_vault(+ self, mock_vault, sketch_id+ ):+ """Test that email_to_domains retrieves WHOXY_API_KEY from vault."""+ api_key = "whoxy-api-key-12345"+ mock_vault.get_secret.return_value = api_key++ transform = EmailToDomainsTransform(+ sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}+ )++ await transform.async_init()++ # Verify vault was queried for the API key+ mock_vault.get_secret.assert_called()+ calls = [call[0][0] for call in mock_vault.get_secret.call_args_list]+ assert "WHOXY_API_KEY" in calls++ # Verify the key is accessible via get_secret+ assert transform.get_secret("WHOXY_API_KEY") == api_key+++class TestEtherscanTransformsVaultIntegration:+ """Tests for Etherscan transforms vault integration."""++ @pytest.mark.asyncio+ async def test_wallet_to_nfts_gets_api_key_from_vault(self, mock_vault, sketch_id):+ """Test that wallet_to_nfts retrieves ETHERSCAN_API_KEY from vault."""+ api_key = "etherscan-api-key-12345"+ mock_vault.get_secret.return_value = api_key++ transform = CryptoWalletAddressToNFTs(+ sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}+ )++ await transform.async_init()++ # Verify vault was queried for the API key+ mock_vault.get_secret.assert_called()+ calls = [call[0][0] for call in mock_vault.get_secret.call_args_list]+ assert "ETHERSCAN_API_KEY" in calls++ # Verify the key is accessible via get_secret+ assert transform.get_secret("ETHERSCAN_API_KEY") == api_key++ @pytest.mark.asyncio+ async def test_wallet_to_transactions_gets_api_key_from_vault(+ self, mock_vault, sketch_id+ ):+ """Test that wallet_to_transactions retrieves ETHERSCAN_API_KEY from vault."""+ api_key = "etherscan-api-key-12345"+ mock_vault.get_secret.return_value = api_key++ transform = CryptoWalletAddressToTransactions(+ sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}+ )++ await transform.async_init()++ # Verify vault was queried for the API key+ mock_vault.get_secret.assert_called()+ calls = [call[0][0] for call in mock_vault.get_secret.call_args_list]+ assert "ETHERSCAN_API_KEY" in calls++ # Verify the key is accessible via get_secret+ assert transform.get_secret("ETHERSCAN_API_KEY") == api_key+++class TestHIBPTransformsVaultIntegration:+ """Tests for HIBP transforms vault integration."""++ @pytest.mark.asyncio+ async def test_email_to_breaches_gets_api_key_from_vault(+ self, mock_vault, sketch_id+ ):+ """Test that email_to_breaches retrieves HIBP_API_KEY from vault."""+ api_key = "hibp-api-key-12345"+ mock_vault.get_secret.return_value = api_key++ transform = EmailToBreachesTransform(+ sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}+ )++ await transform.async_init()++ # Verify vault was queried for the API key+ mock_vault.get_secret.assert_called()+ calls = [call[0][0] for call in mock_vault.get_secret.call_args_list]+ assert "HIBP_API_KEY" in calls++ # Verify the key is accessible via get_secret+ assert transform.get_secret("HIBP_API_KEY") == api_key+++class TestVaultSecretWithUserProvidedID:+ """Tests for vault secrets when user provides a specific vault ID."""++ @pytest.mark.asyncio+ async def test_transform_uses_user_provided_vault_id(self, mock_vault, sketch_id):+ """Test that transform uses user-provided vault ID if specified."""+ vault_id = str(uuid.uuid4())+ api_key = "whoxy-api-key-12345"++ # First call (by ID) returns the secret+ mock_vault.get_secret.return_value = api_key++ transform = DomainToHistoryTransform(+ sketch_id=sketch_id,+ scan_id="scan_123",+ vault=mock_vault,+ params={"WHOXY_API_KEY": vault_id}, # User selected a specific vault entry+ )++ await transform.async_init()++ # Verify vault was queried with the provided ID first+ first_call = mock_vault.get_secret.call_args_list[0][0][0]+ assert first_call == vault_id++ # Verify the key is accessible+ assert transform.get_secret("WHOXY_API_KEY") == api_key++ @pytest.mark.asyncio+ async def test_transform_fallback_to_name_if_id_not_found(+ self, mock_vault, sketch_id+ ):+ """Test that transform falls back to name lookup if vault ID not found."""+ vault_id = str(uuid.uuid4())+ api_key = "whoxy-api-key-12345"++ # First call (by ID) returns None, second call (by name) returns the secret+ mock_vault.get_secret.side_effect = [None, api_key]++ transform = DomainToHistoryTransform(+ sketch_id=sketch_id,+ scan_id="scan_123",+ vault=mock_vault,+ params={"WHOXY_API_KEY": vault_id},+ )++ await transform.async_init()++ # Verify vault was queried twice: first by ID, then by name+ assert mock_vault.get_secret.call_count == 2+ assert mock_vault.get_secret.call_args_list[0][0][0] == vault_id+ assert mock_vault.get_secret.call_args_list[1][0][0] == "WHOXY_API_KEY"++ # Verify the key is accessible+ assert transform.get_secret("WHOXY_API_KEY") == api_key+++class TestMissingRequiredVaultSecret:+ """Tests for error handling when required vault secrets are missing."""++ @pytest.mark.asyncio+ async def test_missing_required_secret_logs_error(self, mock_vault, sketch_id):+ """Test that missing required secret logs an error message."""+ # Vault returns None (secret not found)+ mock_vault.get_secret.return_value = None++ from unittest.mock import patch++ with patch("flowsint_core.core.transform_base.Logger") as mock_logger:+ transform = DomainToHistoryTransform(+ sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}+ )++ await transform.async_init()++ # Verify error was logged+ assert mock_logger.error.called+ error_message = mock_logger.error.call_args[0][1]["message"]+ assert "WHOXY_API_KEY" in error_message+ assert "missing" in error_message.lower()+ assert "Vault settings" in error_message++ @pytest.mark.asyncio+ async def test_transform_can_still_run_with_fallback(self, mock_vault, sketch_id):+ """Test that transform can use environment variable fallback if vault fails."""+ import os+ from unittest.mock import patch++ # Vault returns None+ mock_vault.get_secret.return_value = None++ # Mock environment variable+ with patch.dict(os.environ, {"WHOXY_API_KEY": "fallback-key"}):+ transform = DomainToHistoryTransform(+ sketch_id=sketch_id, scan_id="scan_123", vault=mock_vault, params={}+ )++ await transform.async_init()++ # Should be able to get the fallback value+ api_key = transform.get_secret("WHOXY_API_KEY", os.getenv("WHOXY_API_KEY"))+ assert api_key == "fallback-key"