A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
fix(python): correctness fixes surfaced by turning flake8 on
dextmorgn24e1e276c8d0
diff --git a/flowsint-api/app/security/permissions.py b/flowsint-api/app/security/permissions.pyindex 2650f8e4..16bc0334 100644--- a/flowsint-api/app/security/permissions.py+++ b/flowsint-api/app/security/permissions.py@@ -20,9 +20,6 @@ def can_user(roles: list[Role], actions: list[str]) -> bool:return False-from fastapi import HTTPException--def check_investigation_permission(user_id: UUID, investigation_id: str, actions: list[str], db):diff --git a/flowsint-api/tests/conftest.py b/flowsint-api/tests/conftest.pyindex 144fb230..b5980f7b 100644--- a/flowsint-api/tests/conftest.py+++ b/flowsint-api/tests/conftest.py@@ -9,14 +9,14 @@ os.environ.setdefault("MASTER_VAULT_KEY_V1", "base64:qnHTmwYb+uoygIw9MsRMY22vS5YPchY+QOi/E79GAvM=")-import pytest-from fastapi.testclient import TestClient-from sqlalchemy import create_engine-from sqlalchemy.orm import sessionmaker--from flowsint_core.core.models import Base-from flowsint_core.core.postgre_db import get_db-from app.main import app+import pytest # noqa: E402+from fastapi.testclient import TestClient # noqa: E402+from flowsint_core.core.models import Base # noqa: E402+from flowsint_core.core.postgre_db import get_db # noqa: E402+from sqlalchemy import create_engine # noqa: E402+from sqlalchemy.orm import sessionmaker # noqa: E402++from app.main import app # noqa: E402@pytest.fixturediff --git a/flowsint-core/src/flowsint_core/core/logger.py b/flowsint-core/src/flowsint_core/core/logger.pyindex 21a7a286..d8668dc9 100644--- a/flowsint-core/src/flowsint_core/core/logger.py+++ b/flowsint-core/src/flowsint_core/core/logger.py@@ -13,7 +13,7 @@ import atexitimport threadingimport timefrom datetime import datetime, timezone-from queue import Queue+from queue import Empty, Queuefrom typing import Dict, Optional, Unionfrom uuid import UUID@@ -132,7 +132,7 @@ class LoggerSingleton:):try:logs_to_insert.append(self._log_queue.get_nowait())- except:+ except Empty:breakif not logs_to_insert:diff --git a/flowsint-core/tests/core/base.py b/flowsint-core/tests/core/base.pyindex 2f9972ab..2e0e7416 100644--- a/flowsint-core/tests/core/base.py+++ b/flowsint-core/tests/core/base.py@@ -1,4 +1,5 @@from typing import List+from flowsint_core.core.enricher_base import Enricherfrom flowsint_types import Phonefrom flowsint_types.domain import Domain@@ -46,15 +47,15 @@ def test_correct_preprocess():assert len(preprocessed) == 3 # 3 valid domainsassert preprocessed[0].domain == "mydomain.com"assert preprocessed[0].label == "mydomain.com"- assert preprocessed[0].root == True+ assert preprocessed[0].root is Trueassert preprocessed[1].domain == "blog.mydomain2.com"assert preprocessed[1].label == "blog.mydomain2.com"- assert preprocessed[1].root == False+ assert preprocessed[1].root is Falseassert preprocessed[2].domain == "mydomain3.com"assert preprocessed[2].label == "mydomain3.com"- assert preprocessed[2].root == True+ assert preprocessed[2].root is Truedef test_incorrect_preprocess():@@ -67,5 +68,5 @@ def test_incorrect_preprocess():preprocessed = enricher.preprocess(inputs)assert len(preprocessed) == 1 # 1 valid domainassert preprocessed[0].domain == "mydomain.com"- assert preprocessed[0].root == True+ assert preprocessed[0].root is Trueassert preprocessed[0].label == "mydomain.com"diff --git a/flowsint-core/tests/import/test_entity_detection.py b/flowsint-core/tests/import/test_entity_detection.pyindex caa26887..fbb45d95 100644--- a/flowsint-core/tests/import/test_entity_detection.py+++ b/flowsint-core/tests/import/test_entity_detection.py@@ -1,6 +1,5 @@-from flowsint_types import Domain, File, Ip, Phone-from flowsint_core.imports import detect_type+from flowsint_types import Domain, File, Ip, Phonedef test_detection_domains():@@ -15,13 +14,13 @@ def test_detection_domains():results.append(py_obj)# domain 1assert results[0].domain == "mydomain.com"- assert results[0].root == True+ assert results[0].root is True# domain 2assert results[1].domain == "myseconddomain.com"- assert results[1].root == True+ assert results[1].root is True# domain 3assert results[2].domain == "blog.subdomain.uk"- assert results[2].root == False+ assert results[2].root is Falsedef test_detection_ips():@@ -69,7 +68,7 @@ def test_detection_mixed():# domainassert isinstance(results[1], Domain)assert results[1].domain == "my.domain.io"- assert results[1].root == False+ assert results[1].root is False# phoneassert isinstance(results[2], Phone)assert results[2].number == "+33632233223"diff --git a/flowsint-core/tests/repositories/test_investigation_repository.py b/flowsint-core/tests/repositories/test_investigation_repository.pyindex 31edf697..cf274e45 100644--- a/flowsint-core/tests/repositories/test_investigation_repository.py+++ b/flowsint-core/tests/repositories/test_investigation_repository.py@@ -1,14 +1,16 @@"""Tests for InvestigationRepository."""+from uuid import uuid4+from flowsint_core.core.models import InvestigationUserRole+from flowsint_core.core.repositories import InvestigationRepository+from flowsint_core.core.types import Role+from tests.factories import (InvestigationFactory,InvestigationUserRoleFactory,ProfileFactory,)-from flowsint_core.core.models import InvestigationUserRole-from flowsint_core.core.repositories import InvestigationRepository-from flowsint_core.core.types import Roleclass TestInvestigationRepository:@@ -31,9 +33,7 @@ class TestInvestigationRepository:self._setup(db_session)user = ProfileFactory()inv = InvestigationFactory(owner=user)- InvestigationUserRoleFactory(- user=user, investigation=inv, roles=[Role.OWNER]- )+ InvestigationUserRoleFactory(user=user, investigation=inv, roles=[Role.OWNER])# Another user's investigation (should not appear)other_user = ProfileFactory()@@ -52,9 +52,7 @@ class TestInvestigationRepository:self._setup(db_session)user = ProfileFactory()inv = InvestigationFactory()- InvestigationUserRoleFactory(- user=user, investigation=inv, roles=[Role.VIEWER]- )+ InvestigationUserRoleFactory(user=user, investigation=inv, roles=[Role.VIEWER])repo = InvestigationRepository(db_session)@@ -99,9 +97,7 @@ class TestInvestigationRepository:self._setup(db_session)user = ProfileFactory()inv = InvestigationFactory()- role_entry = InvestigationUserRoleFactory(- user=user, investigation=inv, roles=[Role.EDITOR]- )+ InvestigationUserRoleFactory(user=user, investigation=inv, roles=[Role.EDITOR])repo = InvestigationRepository(db_session)result = repo.get_user_role(user.id, inv.id)diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_dehashed.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_dehashed.pyindex 80fd8ba8..f4e52d21 100644--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_dehashed.py+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_dehashed.py@@ -1,13 +1,14 @@import jsonimport os-import requests-from typing import Any, Dict, List, Optional++import requestsfrom flowsint_core.core.enricher_base import Enricher+from flowsint_core.core.logger import Loggerfrom flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.domain import Domainfrom flowsint_types.individual import Individual-from flowsint_core.core.logger import Logger+@flowsint_enricherclass DomainToDehashed(Enricher):@@ -41,7 +42,7 @@ class DomainToDehashed(Enricher):"""Declare required parameters for this enricher"""return [{- "name": "DEHASHED_API_KEY", # Get your API key from dehashed.com/api+ "name": "DEHASHED_API_KEY", # Get your API key from dehashed.com/api"type": "vaultSecret","description": "Your Dehashed API key.","required": True,@@ -63,14 +64,22 @@ class DomainToDehashed(Enricher):async def scan(self, data: List[InputType]) -> List[OutputType]:results: List[OutputType] = []- api_key = self.get_secret("DEHASHED_API_KEY", os.getenv("DEHASHED_API_KEY"))+ api_key = self.get_secret("DEHASHED_API_KEY", os.getenv("DEHASHED_API_KEY"))for domain in data:try:- headers = {'Dehashed-Api-Key': api_key, 'Content-Type': 'application/json'}+ headers = {+ "Dehashed-Api-Key": api_key,+ "Content-Type": "application/json",+ }raw_data = json.dumps({"query": f"domain:{domain.domain}"})- api_request = requests.post(f'https://api.dehashed.com/v2/search', data=raw_data, headers=headers, timeout=30)+ api_request = requests.post(+ "https://api.dehashed.com/v2/search",+ data=raw_data,+ headers=headers,+ timeout=30,+ )if api_request.status_code != 200:if api_request.status_code == 401:@@ -92,23 +101,30 @@ class DomainToDehashed(Enricher):try:response_json = api_request.json()except Exception as e:- Logger.error(None, {"message": f"(DomainToDehashed) Failed to parse JSON for {domain.domain}: {e}"})+ Logger.error(+ None,+ {+ "message": f"(DomainToDehashed) Failed to parse JSON for {domain.domain}: {e}"+ },+ )continuedehashed_entries = response_json.get("entries", [])if not dehashed_entries:Logger.error(- self.sketch_id,- {- "message": f"(DomainToDehashed) Enricher failed for the domain: '{domain.domain}': {api_request.text}"- },- )+ self.sketch_id,+ {+ "message": f"(DomainToDehashed) Enricher failed for the domain: '{domain.domain}': {api_request.text}"+ },+ )continue-api_balance = response_json.get("balance")if api_balance:- Logger.info(self.sketch_id, f'(DomainToDehashed) Your remaining API balance is {api_balance}.')+ Logger.info(+ self.sketch_id,+ f"(DomainToDehashed) Your remaining API balance is {api_balance}.",+ )for entry in dehashed_entries:entry_email = entry.get("email")@@ -121,18 +137,25 @@ class DomainToDehashed(Enricher):results.append(Individual(- full_name=entry_name[0] if entry_name else None,- birth_date=entry_dob[0] if entry_dob else None,- email_addresses=entry_email if entry_email else None,- phone_numbers=entry_phone if entry_phone else None,- social_media_profiles=entry_socialmedia if entry_socialmedia else None,- ip_addresses=entry_ip if entry_ip else None,- usernames=entry_username if entry_username else None- )+ full_name=entry_name[0] if entry_name else None,+ birth_date=entry_dob[0] if entry_dob else None,+ email_addresses=entry_email if entry_email else None,+ phone_numbers=entry_phone if entry_phone else None,+ social_media_profiles=(+ entry_socialmedia if entry_socialmedia else None+ ),+ ip_addresses=entry_ip if entry_ip else None,+ usernames=entry_username if entry_username else None,)+ )except Exception as e:- Logger.error(self.sketch_id, {"message": f"(DomainToDehashed) Exception while querying {domain.domain}: {e}"})-+ Logger.error(+ self.sketch_id,+ {+ "message": f"(DomainToDehashed) Exception while querying {domain.domain}: {e}"+ },+ )+return resultsdef postprocess(@@ -158,4 +181,4 @@ class DomainToDehashed(Enricher):# Make types available at module level for easy accessInputType = DomainToDehashed.InputType-OutputType = DomainToDehashed.OutputType\ No newline at end of file+OutputType = DomainToDehashed.OutputTypediff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_dummy_individuals.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_dummy_individuals.pyindex 90e62256..9ed864f0 100644--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_dummy_individuals.py+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_dummy_individuals.py@@ -1,6 +1,5 @@-import socketfrom typing import Dict, List-from flowsint_core.core.logger import Logger+from flowsint_core.core.enricher_base import Enricherfrom flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.domain import Domain@@ -56,7 +55,7 @@ class ToDummyEnricher(Enricher):item,"IS_RELATED_T0",)- self.log_graph_message(f"Dummy enricher finished.")+ self.log_graph_message("Dummy enricher finished.")return resultsdiff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_whois.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_whois.pyindex df91076b..ece2a057 100644--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_whois.py+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_whois.py@@ -1,12 +1,13 @@from typing import List+import whoisfrom flowsint_core.core.enricher_base import Enricher+from flowsint_core.core.logger import Loggerfrom flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.domain import Domain-from flowsint_types.whois import Whoisfrom flowsint_types.email import Emailfrom flowsint_types.organization import Organization-from flowsint_core.core.logger import Logger+from flowsint_types.whois import Whois@flowsint_enricher@@ -29,8 +30,9 @@ class WhoisEnricher(Enricher):def key(cls) -> str:return "domain"- async def scan(self, data: List[InputType]) -> List[OutputType]:- results: List[OutputType] = []+ # noqa false positive below: OutputType is a class attr, resolves fine at def-time+ async def scan(self, data: List[InputType]) -> List[OutputType]: # noqa: F821+ results: List[OutputType] = [] # noqa: F821for domain in data:try:whois_info = whois.whois(domain.domain)@@ -109,7 +111,9 @@ class WhoisEnricher(Enricher):return results- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:+ def postprocess(+ self, results: List[OutputType], original_input: List[InputType]+ ) -> List[OutputType]:for whois_obj in results:if not self._graph_service:continue@@ -122,7 +126,9 @@ class WhoisEnricher(Enricher):# Create organization node if availableif whois_obj.organization:self.create_node(whois_obj.organization)- self.create_relationship(whois_obj.organization, whois_obj.domain, "HAS_DOMAIN")+ self.create_relationship(+ whois_obj.organization, whois_obj.domain, "HAS_DOMAIN"+ )# Create email node if availableif whois_obj.email:self.create_node(whois_obj.email)diff --git a/flowsint-enrichers/src/flowsint_enrichers/email/to_dehashed.py b/flowsint-enrichers/src/flowsint_enrichers/email/to_dehashed.pyindex 622e8d16..0a9ca678 100644--- a/flowsint-enrichers/src/flowsint_enrichers/email/to_dehashed.py+++ b/flowsint-enrichers/src/flowsint_enrichers/email/to_dehashed.py@@ -1,13 +1,14 @@import jsonimport os-import requests-from typing import Any, Dict, List, Optional++import requestsfrom flowsint_core.core.enricher_base import Enricher+from flowsint_core.core.logger import Loggerfrom flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.email import Emailfrom flowsint_types.individual import Individual-from flowsint_core.core.logger import Logger+@flowsint_enricherclass EmailToDehashed(Enricher):@@ -41,7 +42,7 @@ class EmailToDehashed(Enricher):"""Declare required parameters for this enricher"""return [{- "name": "DEHASHED_API_KEY", # Get your API key from dehashed.com/api+ "name": "DEHASHED_API_KEY", # Get your API key from dehashed.com/api"type": "vaultSecret","description": "Your Dehashed API key.","required": True,@@ -63,14 +64,22 @@ class EmailToDehashed(Enricher):async def scan(self, data: List[InputType]) -> List[OutputType]:results: List[OutputType] = []- api_key = self.get_secret("DEHASHED_API_KEY", os.getenv("DEHASHED_API_KEY"))+ api_key = self.get_secret("DEHASHED_API_KEY", os.getenv("DEHASHED_API_KEY"))for email in data:try:- headers = {'Dehashed-Api-Key': api_key, 'Content-Type': 'application/json'}+ headers = {+ "Dehashed-Api-Key": api_key,+ "Content-Type": "application/json",+ }raw_data = json.dumps({"query": f"email:{email.email}"})- api_request = requests.post(f'https://api.dehashed.com/v2/search', data=raw_data, headers=headers, timeout=30)+ api_request = requests.post(+ "https://api.dehashed.com/v2/search",+ data=raw_data,+ headers=headers,+ timeout=30,+ )if api_request.status_code != 200:if api_request.status_code == 401:@@ -92,23 +101,30 @@ class EmailToDehashed(Enricher):try:response_json = api_request.json()except Exception as e:- Logger.error(None, {"message": f"(EmailToDehashed) Failed to parse JSON for {email.email}: {e}"})+ Logger.error(+ None,+ {+ "message": f"(EmailToDehashed) Failed to parse JSON for {email.email}: {e}"+ },+ )continuedehashed_entries = response_json.get("entries", [])if not dehashed_entries:Logger.error(- self.sketch_id,- {- "message": f"(EmailToDehashed) Enricher failed for the email: '{email.email}': {api_request.text}"- },- )+ self.sketch_id,+ {+ "message": f"(EmailToDehashed) Enricher failed for the email: '{email.email}': {api_request.text}"+ },+ )continue-api_balance = response_json.get("balance")if api_balance:- Logger.info(self.sketch_id, f'(EmailToDehashed) Your remaining API balance is {api_balance}.')+ Logger.info(+ self.sketch_id,+ f"(EmailToDehashed) Your remaining API balance is {api_balance}.",+ )for entry in dehashed_entries:entry_email = entry.get("email")@@ -121,18 +137,25 @@ class EmailToDehashed(Enricher):results.append(Individual(- full_name=entry_name[0] if entry_name else None,- birth_date=entry_dob[0] if entry_dob else None,- email_addresses=entry_email if entry_email else None,- phone_numbers=entry_phone if entry_phone else None,- social_media_profiles=entry_socialmedia if entry_socialmedia else None,- ip_addresses=entry_ip if entry_ip else None,- usernames=entry_username if entry_username else None- )+ full_name=entry_name[0] if entry_name else None,+ birth_date=entry_dob[0] if entry_dob else None,+ email_addresses=entry_email if entry_email else None,+ phone_numbers=entry_phone if entry_phone else None,+ social_media_profiles=(+ entry_socialmedia if entry_socialmedia else None+ ),+ ip_addresses=entry_ip if entry_ip else None,+ usernames=entry_username if entry_username else None,)+ )except Exception as e:- Logger.error(self.sketch_id, {"message": f"(EmailToDehashed) Exception while querying email {email.email}: {e}"})-+ Logger.error(+ self.sketch_id,+ {+ "message": f"(EmailToDehashed) Exception while querying email {email.email}: {e}"+ },+ )+return resultsdef postprocess(@@ -158,4 +181,4 @@ class EmailToDehashed(Enricher):# Make types available at module level for easy accessInputType = EmailToDehashed.InputType-OutputType = EmailToDehashed.OutputType\ No newline at end of file+OutputType = EmailToDehashed.OutputTypediff --git a/flowsint-enrichers/src/flowsint_enrichers/ip/to_dehashed.py b/flowsint-enrichers/src/flowsint_enrichers/ip/to_dehashed.pyindex 6612f56e..69175c7e 100644--- a/flowsint-enrichers/src/flowsint_enrichers/ip/to_dehashed.py+++ b/flowsint-enrichers/src/flowsint_enrichers/ip/to_dehashed.py@@ -1,13 +1,14 @@import jsonimport os-import requests-from typing import Any, Dict, List, Optional++import requestsfrom flowsint_core.core.enricher_base import Enricher+from flowsint_core.core.logger import Loggerfrom flowsint_enrichers.registry import flowsint_enricher-from flowsint_types.ip import Ipfrom flowsint_types.individual import Individual-from flowsint_core.core.logger import Logger+from flowsint_types.ip import Ip+@flowsint_enricherclass IpToIntelligence(Enricher):@@ -41,7 +42,7 @@ class IpToIntelligence(Enricher):"""Declare required parameters for this enricher"""return [{- "name": "DEHASHED_API_KEY", # Get your API key from dehashed.com/api+ "name": "DEHASHED_API_KEY", # Get your API key from dehashed.com/api"type": "vaultSecret","description": "Your Dehashed API key.","required": True,@@ -63,14 +64,22 @@ class IpToIntelligence(Enricher):async def scan(self, data: List[InputType]) -> List[OutputType]:results: List[OutputType] = []- api_key = self.get_secret("DEHASHED_API_KEY", os.getenv("DEHASHED_API_KEY"))+ api_key = self.get_secret("DEHASHED_API_KEY", os.getenv("DEHASHED_API_KEY"))for ip in data:try:- headers = {'Dehashed-Api-Key': api_key, 'Content-Type': 'application/json'}+ headers = {+ "Dehashed-Api-Key": api_key,+ "Content-Type": "application/json",+ }raw_data = json.dumps({"query": f"ip_address:{ip.address}"})- api_request = requests.post(f'https://api.dehashed.com/v2/search', data=raw_data, headers=headers, timeout=30)+ api_request = requests.post(+ "https://api.dehashed.com/v2/search",+ data=raw_data,+ headers=headers,+ timeout=30,+ )if api_request.status_code != 200:if api_request.status_code == 401:@@ -92,25 +101,32 @@ class IpToIntelligence(Enricher):try:response_json = api_request.json()except Exception as e:- Logger.error(None, {"message": f"(IpToIntelligence) Failed to parse JSON for {ip.address}: {e}"})+ Logger.error(+ None,+ {+ "message": f"(IpToIntelligence) Failed to parse JSON for {ip.address}: {e}"+ },+ )continuedehashed_entries = response_json.get("entries", [])if not dehashed_entries:Logger.error(- self.sketch_id,- {- "message": f"(IpToIntelligence) Enricher failed for the IP address: '{ip.address}': {api_request.text}"- },- )+ self.sketch_id,+ {+ "message": f"(IpToIntelligence) Enricher failed for the IP address: '{ip.address}': {api_request.text}"+ },+ )continue-api_balance = response_json.get("balance")if api_balance:- Logger.info(self.sketch_id, f'(EmailToDehashed) Your remaining API balance is {api_balance}.')-- time_took = response_json.get("took")+ Logger.info(+ self.sketch_id,+ f"(EmailToDehashed) Your remaining API balance is {api_balance}.",+ )++ response_json.get("took")for entry in dehashed_entries:entry_email = entry.get("email")@@ -123,18 +139,25 @@ class IpToIntelligence(Enricher):results.append(Individual(- full_name=entry_name[0] if entry_name else None,- birth_date=entry_dob[0] if entry_dob else None,- email_addresses=entry_email if entry_email else None,- phone_numbers=entry_phone if entry_phone else None,- social_media_profiles=entry_socialmedia if entry_socialmedia else None,- ip_addresses=entry_ip if entry_ip else None,- usernames=entry_username if entry_username else None- )+ full_name=entry_name[0] if entry_name else None,+ birth_date=entry_dob[0] if entry_dob else None,+ email_addresses=entry_email if entry_email else None,+ phone_numbers=entry_phone if entry_phone else None,+ social_media_profiles=(+ entry_socialmedia if entry_socialmedia else None+ ),+ ip_addresses=entry_ip if entry_ip else None,+ usernames=entry_username if entry_username else None,)+ )except Exception as e:- Logger.error(self.sketch_id, {"message": f"(IpToIntelligence) Exception while querying {ip.address}: {e}"})-+ Logger.error(+ self.sketch_id,+ {+ "message": f"(IpToIntelligence) Exception while querying {ip.address}: {e}"+ },+ )+return resultsdef postprocess(@@ -160,4 +183,4 @@ class IpToIntelligence(Enricher):# Make types available at module level for easy accessInputType = IpToIntelligence.InputType-OutputType = IpToIntelligence.OutputType\ No newline at end of file+OutputType = IpToIntelligence.OutputTypediff --git a/flowsint-enrichers/src/flowsint_enrichers/ip/to_infos.py b/flowsint-enrichers/src/flowsint_enrichers/ip/to_infos.pyindex fe8240df..3f894cf8 100644--- a/flowsint-enrichers/src/flowsint_enrichers/ip/to_infos.py+++ b/flowsint-enrichers/src/flowsint_enrichers/ip/to_infos.py@@ -1,5 +1,6 @@+from typing import Any, Dict, List+import requests-from typing import List, Dict, Anyfrom flowsint_core.core.enricher_base import Enricherfrom flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.ip import Ip@@ -25,8 +26,9 @@ class IpToInfosEnricher(Enricher):def key(cls) -> str:return "address"- async def scan(self, data: List[InputType]) -> List[OutputType]:- results: List[OutputType] = []+ # noqa false positive below: OutputType is a class attr, resolves fine at def-time+ async def scan(self, data: List[InputType]) -> List[OutputType]: # noqa: F821+ results: List[OutputType] = [] # noqa: F821for ip in data:try:geo_data = self.get_location_data(ip.address)@@ -41,7 +43,9 @@ class IpToInfosEnricher(Enricher):print(f"Error geolocating {ip.address}: {e}")return results- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:+ def postprocess(+ self, results: List[OutputType], original_input: List[InputType]+ ) -> List[OutputType]:"""Update IP nodes in Neo4j with geolocation information."""if self._graph_service:for ip in results:diff --git a/flowsint-enrichers/src/flowsint_enrichers/ip/to_ports.py b/flowsint-enrichers/src/flowsint_enrichers/ip/to_ports.pyindex ad738160..2cc5364f 100644--- a/flowsint-enrichers/src/flowsint_enrichers/ip/to_ports.py+++ b/flowsint-enrichers/src/flowsint_enrichers/ip/to_ports.py@@ -1,10 +1,10 @@-from typing import Any, Dict, List, Optional, Union+from typing import Any, Dict, List, Optional+from flowsint_core.core.enricher_base import Enricher+from flowsint_core.core.logger import Loggerfrom flowsint_enrichers.registry import flowsint_enricherfrom flowsint_types.ip import Ipfrom flowsint_types.port import Port-from flowsint_core.utils import is_valid_ip-from flowsint_core.core.logger import Loggerfrom tools.network.naabu import NaabuTool@@ -133,9 +133,7 @@ class IpToPortsEnricher(Enricher):try:Logger.info(self.sketch_id,- {- "message": f"[NAABU] Scanning {ip.address} in {mode} mode..."- },+ {"message": f"[NAABU] Scanning {ip.address} in {mode} mode..."},)# Launch naabu scan@@ -197,8 +195,7 @@ class IpToPortsEnricher(Enricher):ip_address = getattr(port, "_ip_address", None)if not ip_address:continue-- port_id = f"{ip_address}:{port.number}"+self.create_node(port)# Create relationship from IP to Portdiff --git a/flowsint-enrichers/src/flowsint_enrichers/social/to_dehashed.py b/flowsint-enrichers/src/flowsint_enrichers/social/to_dehashed.pyindex 00638792..22860e48 100644--- a/flowsint-enrichers/src/flowsint_enrichers/social/to_dehashed.py+++ b/flowsint-enrichers/src/flowsint_enrichers/social/to_dehashed.py@@ -1,13 +1,14 @@import jsonimport os-import requests-from typing import Any, Dict, List, Optional++import requestsfrom flowsint_core.core.enricher_base import Enricher+from flowsint_core.core.logger import Loggerfrom flowsint_enrichers.registry import flowsint_enricher-from flowsint_types.username import Usernamefrom flowsint_types.individual import Individual-from flowsint_core.core.logger import Logger+from flowsint_types.username import Username+@flowsint_enricherclass UsernameToDehashed(Enricher):@@ -41,7 +42,7 @@ class UsernameToDehashed(Enricher):"""Declare required parameters for this enricher"""return [{- "name": "DEHASHED_API_KEY", # Get your API key from dehashed.com/api+ "name": "DEHASHED_API_KEY", # Get your API key from dehashed.com/api"type": "vaultSecret","description": "Your Dehashed API key.","required": True,@@ -63,14 +64,22 @@ class UsernameToDehashed(Enricher):async def scan(self, data: List[InputType]) -> List[OutputType]:results: List[OutputType] = []- api_key = self.get_secret("DEHASHED_API_KEY", os.getenv("DEHASHED_API_KEY"))+ api_key = self.get_secret("DEHASHED_API_KEY", os.getenv("DEHASHED_API_KEY"))for username in data:try:- headers = {'Dehashed-Api-Key': api_key, 'Content-Type': 'application/json'}+ headers = {+ "Dehashed-Api-Key": api_key,+ "Content-Type": "application/json",+ }raw_data = json.dumps({"query": f"username:{username.value}"})- api_request = requests.post(f'https://api.dehashed.com/v2/search', data=raw_data, headers=headers, timeout=30)+ api_request = requests.post(+ "https://api.dehashed.com/v2/search",+ data=raw_data,+ headers=headers,+ timeout=30,+ )if api_request.status_code != 200:if api_request.status_code == 401:@@ -92,23 +101,30 @@ class UsernameToDehashed(Enricher):try:response_json = api_request.json()except Exception as e:- Logger.error(None, {"message": f"(UsernameToDehashed) Failed to parse JSON for {username.value}: {e}"})+ Logger.error(+ None,+ {+ "message": f"(UsernameToDehashed) Failed to parse JSON for {username.value}: {e}"+ },+ )continuedehashed_entries = response_json.get("entries", [])if not dehashed_entries:Logger.error(- self.sketch_id,- {- "message": f"(UsernameToDehashed) failed for the username: '{username.value}': {api_request.text}"- },- )+ self.sketch_id,+ {+ "message": f"(UsernameToDehashed) failed for the username: '{username.value}': {api_request.text}"+ },+ )continue-api_balance = response_json.get("balance")if api_balance:- Logger.info(self.sketch_id, f'(UsernameToDehashed) Your remaining API balance is {api_balance}.')+ Logger.info(+ self.sketch_id,+ f"(UsernameToDehashed) Your remaining API balance is {api_balance}.",+ )for entry in dehashed_entries:entry_email = entry.get("email")@@ -121,18 +137,25 @@ class UsernameToDehashed(Enricher):results.append(Individual(- full_name=entry_name[0] if entry_name else None,- birth_date=entry_dob[0] if entry_dob else None,- email_addresses=entry_email if entry_email else None,- phone_numbers=entry_phone if entry_phone else None,- social_media_profiles=entry_socialmedia if entry_socialmedia else None,- ip_addresses=entry_ip if entry_ip else None,- usernames=entry_username if entry_username else None- )+ full_name=entry_name[0] if entry_name else None,+ birth_date=entry_dob[0] if entry_dob else None,+ email_addresses=entry_email if entry_email else None,+ phone_numbers=entry_phone if entry_phone else None,+ social_media_profiles=(+ entry_socialmedia if entry_socialmedia else None+ ),+ ip_addresses=entry_ip if entry_ip else None,+ usernames=entry_username if entry_username else None,)+ )except Exception as e:- Logger.error(self.sketch_id, {"message": f"(UsernameToDehashed) Exception while querying username {username.value}: {e}"})-+ Logger.error(+ self.sketch_id,+ {+ "message": f"(UsernameToDehashed) Exception while querying username {username.value}: {e}"+ },+ )+return resultsdef postprocess(@@ -158,4 +181,4 @@ class UsernameToDehashed(Enricher):# Make types available at module level for easy accessInputType = UsernameToDehashed.InputType-OutputType = UsernameToDehashed.OutputType\ No newline at end of file+OutputType = UsernameToDehashed.OutputTypediff --git a/flowsint-enrichers/src/tools/dockertool.py b/flowsint-enrichers/src/tools/dockertool.pyindex 94077da8..ebeee776 100644--- a/flowsint-enrichers/src/tools/dockertool.py+++ b/flowsint-enrichers/src/tools/dockertool.py@@ -1,5 +1,6 @@-from docker import from_env, DockerClient-from docker.errors import ImageNotFound, APIError, DockerException+from docker import from_env+from docker.errors import APIError, DockerException, ImageNotFound+from .base import Tool@@ -46,7 +47,14 @@ class DockerTool(Tool):except ImageNotFound:return False- def launch(self, command, volumes: dict = None, timeout: int = 30, environment: dict = None, entrypoint=None):+ def launch(+ self,+ command,+ volumes: dict = None,+ timeout: int = 30,+ environment: dict = None,+ entrypoint=None,+ ):self.install()# Merge default environment with custom environmentenv = {"TERM": "dumb"} # Set terminal type to avoid TTY issues@@ -81,7 +89,7 @@ class DockerTool(Tool):try:error_json = e.response.json()error_detail = f"{str(e)} - Details: {error_json}"- except:+ except Exception:pass# Check if it's a container exit errordiff --git a/flowsint-enrichers/tests/tools/network/asnmap.py b/flowsint-enrichers/tests/tools/network/asnmap.pyindex 12f83fbe..dc9d247f 100644--- a/flowsint-enrichers/tests/tools/network/asnmap.py+++ b/flowsint-enrichers/tests/tools/network/asnmap.py@@ -1,5 +1,6 @@import refrom typing import Dict+from tools.network.asnmap import AsnmapTooltool = AsnmapTool()@@ -23,7 +24,7 @@ def test_image():def test_install():tool.install()- assert tool.is_installed() == True+ assert tool.is_installed() is Truedef test_version():diff --git a/flowsint-enrichers/tests/tools/network/httpx.py b/flowsint-enrichers/tests/tools/network/httpx.pyindex 63af13a6..b0adf4a1 100644--- a/flowsint-enrichers/tests/tools/network/httpx.py+++ b/flowsint-enrichers/tests/tools/network/httpx.py@@ -1,5 +1,6 @@import refrom typing import List+from tools.network.httpx import HttpxTooltool = HttpxTool()@@ -26,7 +27,7 @@ def test_image():def test_install():tool.install()- assert tool.is_installed() == True+ assert tool.is_installed() is Truedef test_version():diff --git a/flowsint-enrichers/tests/tools/network/naabu.py b/flowsint-enrichers/tests/tools/network/naabu.pyindex 2c8986d4..740b8359 100644--- a/flowsint-enrichers/tests/tools/network/naabu.py+++ b/flowsint-enrichers/tests/tools/network/naabu.py@@ -1,5 +1,5 @@import re-from typing import List+from tools.network.naabu import NaabuTooltool = NaabuTool()@@ -26,7 +26,7 @@ def test_image():def test_install():tool.install()- assert tool.is_installed() == True+ assert tool.is_installed() is Truedef test_version():diff --git a/flowsint-enrichers/tests/tools/network/reconcrawl.py b/flowsint-enrichers/tests/tools/network/reconcrawl.pyindex 0bffd125..bd424521 100644--- a/flowsint-enrichers/tests/tools/network/reconcrawl.py+++ b/flowsint-enrichers/tests/tools/network/reconcrawl.py@@ -20,4 +20,4 @@ def test_category():def test_install():tool.install()- assert tool.is_installed() == True+ assert tool.is_installed() is Truediff --git a/flowsint-enrichers/tests/tools/network/subfinder.py b/flowsint-enrichers/tests/tools/network/subfinder.pyindex 3219e214..d8346e68 100644--- a/flowsint-enrichers/tests/tools/network/subfinder.py+++ b/flowsint-enrichers/tests/tools/network/subfinder.py@@ -1,4 +1,5 @@import re+from tools.network.subfinder import SubfinderTooltool = SubfinderTool()@@ -22,7 +23,7 @@ def test_image():def test_install():tool.install()- assert tool.is_installed() == True+ assert tool.is_installed() is Truedef test_version():diff --git a/flowsint-types/src/flowsint_types/asn.py b/flowsint-types/src/flowsint_types/asn.pyindex d7d7a1c2..70ea5998 100644--- a/flowsint-types/src/flowsint_types/asn.py+++ b/flowsint-types/src/flowsint_types/asn.py@@ -1,6 +1,8 @@-from typing import List, Optional, Union, Self-from pydantic import Field, field_validator, model_validatorimport re+from typing import List, Optional, Self++from pydantic import Field, field_validator, model_validator+from .flowsint_base import FlowsintTypefrom .registry import flowsint_type@@ -100,6 +102,6 @@ class ASN(FlowsintType):# Import CIDR here to avoid circular import-from .cidr import CIDR+from .cidr import CIDR # noqa: E402ASN.model_rebuild()diff --git a/flowsint-types/tests/test_registry.py b/flowsint-types/tests/test_registry.pyindex ec269500..5a267936 100644--- a/flowsint-types/tests/test_registry.py+++ b/flowsint-types/tests/test_registry.py@@ -1,4 +1,4 @@-from flowsint_types import Domain, Individual, Ip, get_type+from flowsint_types import Domain, Ip, get_typefrom flowsint_types.registry import TYPE_REGISTRY@@ -20,11 +20,6 @@ def test_get_by_lower_key():assert type is Domain-def test_get_by_lower_key():- type = TYPE_REGISTRY.get_lowercase("domain")- assert type is Domain--def get_type_function():# Probably the main util function used here, as we can pass it either capital or lowercase typestype_lower = get_type("domain")