A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
Fix zip alignment mismatch, phone validation return value, and vault owner_id type matching
Melih Daskiran25eb3a04009d
diff --git a/flowsint-api/app/utils.py b/flowsint-api/app/utils.pyindex 7126bdc2..46b0a5e1 100644--- a/flowsint-api/app/utils.py+++ b/flowsint-api/app/utils.py@@ -101,7 +101,7 @@ def is_root_domain(domain: str) -> bool:return False-def is_valid_number(phone: str, region: str = "FR") -> None:+def is_valid_number(phone: str, region: str = "FR") -> bool:"""Validates a phone number. Raises InvalidPhoneNumberError if invalid.- `region` should be ISO 3166-1 alpha-2 country code (e.g., 'FR' for France)@@ -113,6 +113,8 @@ def is_valid_number(phone: str, region: str = "FR") -> None:except NumberParseException:return False+ return True+def parse_asn(asn: str) -> int:if not is_valid_asn(asn):diff --git a/flowsint-core/src/flowsint_core/core/vault.py b/flowsint-core/src/flowsint_core/core/vault.pyindex d8427407..c48e9596 100644--- a/flowsint-core/src/flowsint_core/core/vault.py+++ b/flowsint-core/src/flowsint_core/core/vault.py@@ -25,7 +25,7 @@ class Vault(VaultProtocol):if not owner_id:raise ValueError("owner_id is required to use the vault.")self.db = db- self.owner_id = str(owner_id)+ self.owner_id = owner_idself.version = "V1"def _get_master_key(self) -> bytes:@@ -52,7 +52,7 @@ class Vault(VaultProtocol):algorithm=hashes.SHA256(),length=32,salt=salt,- info=self.owner_id.encode("utf-8"),+ info=str(self.owner_id).encode("utf-8"),)master_key = self._get_master_key()return hkdf.derive(master_key)@@ -72,7 +72,7 @@ class Vault(VaultProtocol):ciphertext = aesgcm.encrypt(iv,plaintext.encode("utf-8"),- self.owner_id.encode("utf-8"), # AAD = links secret to user_id+ str(self.owner_id).encode("utf-8"), # AAD = links secret to user_id)return {@@ -92,7 +92,7 @@ class Vault(VaultProtocol):plaintext = aesgcm.decrypt(row.get("iv"),row.get("ciphertext"),- self.owner_id.encode("utf-8"),+ str(self.owner_id).encode("utf-8"),)return plaintext.decode("utf-8")diff --git a/flowsint-core/tests/vault/test_vault.py b/flowsint-core/tests/vault/test_vault.pyindex 5d31f9b1..ef12466c 100644--- a/flowsint-core/tests/vault/test_vault.py+++ b/flowsint-core/tests/vault/test_vault.py@@ -45,7 +45,7 @@ class TestVaultInitialization:"""Test successful Vault initialization."""vault = Vault(db=mock_db, owner_id=owner_id)assert vault.db == mock_db- assert vault.owner_id == str(owner_id)+ assert vault.owner_id == owner_idassert vault.version == "V1"diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_asn.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_asn.pyindex b58ff2c9..fe5dc992 100644--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_asn.py+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_asn.py@@ -33,6 +33,7 @@ class DomainToAsnEnricher(Enricher):vault=vault,params=params,)+ self.domain_asn_mapping: List[tuple[Domain, ASN]] = []@classmethoddef required_params(cls) -> bool:@@ -64,6 +65,7 @@ class DomainToAsnEnricher(Enricher):async def scan(self, data: List[InputType]) -> List[OutputType]:results: List[OutputType] = []+ self.domain_asn_mapping = []asnmap = AsnmapTool()# Retrieve API key from vault or environment@@ -87,6 +89,7 @@ class DomainToAsnEnricher(Enricher):description=asn_data.get("as_name", ""),)results.append(asn)+ self.domain_asn_mapping.append((domain, asn))Logger.info(self.sketch_id,@@ -115,8 +118,8 @@ class DomainToAsnEnricher(Enricher):self, results: List[OutputType], input_data: List[InputType] = None) -> List[OutputType]:# Create Neo4j relationships between domains and their corresponding ASNs- if input_data and self._graph_service:- for domain, asn in zip(input_data, results):+ if self._graph_service:+ for domain, asn in self.domain_asn_mapping:# Create domain nodeself.create_node(domain)# Create ASN nodediff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_ip.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_ip.pyindex d583333e..4157e2ab 100644--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_ip.py+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_ip.py@@ -14,6 +14,10 @@ class ResolveEnricher(Enricher):InputType = DomainOutputType = Ip+ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ self.domain_ip_mapping: List[tuple[Domain, Ip]] = []+@classmethoddef name(cls) -> str:return "domain_to_ip"@@ -33,10 +37,13 @@ class ResolveEnricher(Enricher):async def scan(self, data: List[InputType]) -> List[OutputType]:results: List[OutputType] = []+ self.domain_ip_mapping = []for d in data:try:ip = socket.gethostbyname(d.domain)- results.append(Ip(address=ip))+ ip_obj = Ip(address=ip)+ results.append(ip_obj)+ self.domain_ip_mapping.append((d, ip_obj))except Exception as e:Logger.info(self.sketch_id,@@ -46,7 +53,9 @@ class ResolveEnricher(Enricher):return resultsdef postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:- for domain_obj, ip_obj in zip(original_input, results):+ for domain_obj, ip_obj in self.domain_ip_mapping:+ if not self._graph_service:+ continueself.create_node(domain_obj)self.create_node(ip_obj)self.create_relationship(diff --git a/flowsint-enrichers/src/flowsint_enrichers/email/to_gravatar.py b/flowsint-enrichers/src/flowsint_enrichers/email/to_gravatar.pyindex 81798922..bb0589cf 100644--- a/flowsint-enrichers/src/flowsint_enrichers/email/to_gravatar.py+++ b/flowsint-enrichers/src/flowsint_enrichers/email/to_gravatar.py@@ -15,6 +15,10 @@ class EmailToGravatarEnricher(Enricher):InputType = EmailOutputType = Gravatar+ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ self.email_gravatar_mapping: List[tuple[Email, Gravatar]] = []+@classmethoddef name(cls) -> str:return "email_to_gravatar"@@ -29,6 +33,7 @@ class EmailToGravatarEnricher(Enricher):async def scan(self, data: List[InputType]) -> List[OutputType]:results: List[OutputType] = []+ self.email_gravatar_mapping = []for email in data:try:@@ -72,6 +77,7 @@ class EmailToGravatarEnricher(Enricher):gravatar = Gravatar(**gravatar_data)results.append(gravatar)+ self.email_gravatar_mapping.append((email, gravatar))except Exception as e:Logger.error(@@ -85,13 +91,12 @@ class EmailToGravatarEnricher(Enricher):return resultsdef postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:- for email_obj, gravatar_obj in zip(original_input, results):+ for email_obj, gravatar_obj in self.email_gravatar_mapping:if not self._graph_service:continue# Create email nodeself.create_node(email_obj)# Create gravatar node- gravatar_key = f"{email_obj.email}_{self.sketch_id}"self.create_node(gravatar_obj)# Create relationship between email and gravatarself.create_relationship(email_obj, gravatar_obj, "HAS_GRAVATAR")diff --git a/flowsint-enrichers/src/flowsint_enrichers/ip/to_asn.py b/flowsint-enrichers/src/flowsint_enrichers/ip/to_asn.pyindex 60446268..0acb0c2e 100644--- a/flowsint-enrichers/src/flowsint_enrichers/ip/to_asn.py+++ b/flowsint-enrichers/src/flowsint_enrichers/ip/to_asn.py@@ -32,6 +32,7 @@ class IpToAsnEnricher(Enricher):vault=vault,params=params,)+ self.ip_asn_mapping: List[tuple[Ip, ASN]] = []@classmethoddef required_params(cls) -> bool:@@ -63,6 +64,7 @@ class IpToAsnEnricher(Enricher):async def scan(self, data: List[InputType]) -> List[OutputType]:results: List[OutputType] = []+ self.ip_asn_mapping = []asnmap = AsnmapTool()# Retrieve API key from vault or environment@@ -84,6 +86,7 @@ class IpToAsnEnricher(Enricher):description=asn_data.get("as_name", ""),)results.append(asn)+ self.ip_asn_mapping.append((ip, asn))Logger.info(self.sketch_id,{@@ -110,8 +113,8 @@ class IpToAsnEnricher(Enricher):self, results: List[OutputType], input_data: List[InputType] = None) -> List[OutputType]:# Create Neo4j relationships between IPs and their corresponding ASNs- if input_data and self._graph_service:- for ip, asn in zip(input_data, results):+ if self._graph_service:+ for ip, asn in self.ip_asn_mapping:# Create IP nodeself.create_node(ip)# Create ASN nodediff --git a/flowsint-enrichers/src/flowsint_enrichers/ip/to_fraudscore.py b/flowsint-enrichers/src/flowsint_enrichers/ip/to_fraudscore.pyindex e8ad32f7..25afa6e4 100644--- a/flowsint-enrichers/src/flowsint_enrichers/ip/to_fraudscore.py+++ b/flowsint-enrichers/src/flowsint_enrichers/ip/to_fraudscore.py@@ -33,6 +33,7 @@ class IpToFraudScore(Enricher):vault=vault,params=params,)+ self.ip_risk_mapping: List[tuple[Ip, RiskProfile]] = []# @classmethod# def required_params(cls) -> bool:@@ -70,6 +71,7 @@ class IpToFraudScore(Enricher):async def scan(self, data: List[InputType]) -> List[OutputType]:results: List[OutputType] = []+ self.ip_risk_mapping = []api_username = self.get_secret("SCAMLYTICS_USERNAME", os.getenv("SCAMLYTICS_USERNAME"))api_key = self.get_secret("SCAMLYTICS_API_KEY", os.getenv("SCAMLYTICS_API_KEY"))@@ -98,7 +100,7 @@ class IpToFraudScore(Enricher):Logger.error(self.sketch_id,{- "message": f"(IpToFraudScore) Request to Scamlytics failed (status): '{response_json["status"]}'."+ "message": f"(IpToFraudScore) Request to Scamlytics failed (status): '{response_json['status']}'."},)continue@@ -121,11 +123,11 @@ class IpToFraudScore(Enricher):if proxy_info.get(key):proxy_flags.append(label)- results.append(- RiskProfile(- entity_id=ip.address, entity_type="IP address", overall_risk_score=fraud_score, risk_level=fraud_risk, last_updated=datetime.datetime.utcnow().isoformat(), risk_factors=proxy_flags, source="Scamalytics"- )- )+ risk_profile = RiskProfile(+ entity_id=ip.address, entity_type="IP address", overall_risk_score=fraud_score, risk_level=fraud_risk, last_updated=datetime.datetime.utcnow().isoformat(), risk_factors=proxy_flags, source="Scamalytics"+ )+ results.append(risk_profile)+ self.ip_risk_mapping.append((ip, risk_profile))except Exception as e:Logger.error(self.sketch_id, {"message": f"(IpToFraudScore) Exception while querying {ip.address}: {e}"})@@ -138,16 +140,15 @@ class IpToFraudScore(Enricher):if not self._graph_service:return results- if input_data and self._graph_service:- for ip, risk_profile in zip(input_data, results):- self.create_node(ip)- self.create_node(risk_profile)+ for ip, risk_profile in self.ip_risk_mapping:+ self.create_node(ip)+ self.create_node(risk_profile)- # Create relationship- self.create_relationship(ip, risk_profile, "HAS_RISK_PROFILE")- self.log_graph_message(- f"(IpToFraudScore) IP {ip.address} has risk level of {risk_profile.risk_level}"- )+ # Create relationship+ self.create_relationship(ip, risk_profile, "HAS_RISK_PROFILE")+ self.log_graph_message(+ f"(IpToFraudScore) IP {ip.address} has risk level of {risk_profile.risk_level}"+ )return resultsdiff --git a/flowsint-enrichers/src/flowsint_enrichers/phone/to_infos.py b/flowsint-enrichers/src/flowsint_enrichers/phone/to_infos.pyindex e9ab8247..d904a290 100644--- a/flowsint-enrichers/src/flowsint_enrichers/phone/to_infos.py+++ b/flowsint-enrichers/src/flowsint_enrichers/phone/to_infos.py@@ -34,9 +34,8 @@ class IgnorantEnricher(Enricher):results: List[OutputType] = []for phone_obj in data:try:- cleaned_phone = is_valid_number(phone_obj.number)- if cleaned_phone:- result = await self._perform_ignorant_research(cleaned_phone)+ if is_valid_number(phone_obj.number):+ result = await self._perform_ignorant_research(phone_obj.number)results.append(result)else:results.append({"number": phone_obj.number, "error": "Invalid phone number"})diff --git a/flowsint-enrichers/src/flowsint_enrichers/utils.py b/flowsint-enrichers/src/flowsint_enrichers/utils.pyindex 103aa756..a7d01c82 100644--- a/flowsint-enrichers/src/flowsint_enrichers/utils.py+++ b/flowsint-enrichers/src/flowsint_enrichers/utils.py@@ -155,7 +155,7 @@ def get_root_domain(domain: str) -> str:return domain-def is_valid_number(phone: str, region: str = "FR") -> None:+def is_valid_number(phone: str, region: str = "FR") -> bool:"""Validates a phone number. Raises InvalidPhoneNumberError if invalid.- `region` should be ISO 3166-1 alpha-2 country code (e.g., 'FR' for France)@@ -167,6 +167,8 @@ def is_valid_number(phone: str, region: str = "FR") -> None:except NumberParseException:return False+ return True+def parse_asn(asn: str) -> int:if not is_valid_asn(asn):