Skip to content

A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.

fix(python): correctness fixes surfaced by turning flake8 on

dextmorgn24e1e276c8d0
diff --git a/flowsint-api/app/security/permissions.py b/flowsint-api/app/security/permissions.py
index 2650f8e4..16bc0334 100644
--- a/flowsint-api/app/security/permissions.py
+++ b/flowsint-api/app/security/permissions.py
@@ -20,9 +20,6 @@ def can_user(roles: list[Role], actions: list[str]) -> bool:
return False
-from fastapi import HTTPException
-
-
def check_investigation_permission(
user_id: UUID, investigation_id: str, actions: list[str], db
):
diff --git a/flowsint-api/tests/conftest.py b/flowsint-api/tests/conftest.py
index 144fb230..b5980f7b 100644
--- a/flowsint-api/tests/conftest.py
+++ b/flowsint-api/tests/conftest.py
@@ -9,14 +9,14 @@ os.environ.setdefault(
"MASTER_VAULT_KEY_V1", "base64:qnHTmwYb+uoygIw9MsRMY22vS5YPchY+QOi/E79GAvM="
)
-import pytest
-from fastapi.testclient import TestClient
-from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker
-
-from flowsint_core.core.models import Base
-from flowsint_core.core.postgre_db import get_db
-from app.main import app
+import pytest # noqa: E402
+from fastapi.testclient import TestClient # noqa: E402
+from flowsint_core.core.models import Base # noqa: E402
+from flowsint_core.core.postgre_db import get_db # noqa: E402
+from sqlalchemy import create_engine # noqa: E402
+from sqlalchemy.orm import sessionmaker # noqa: E402
+
+from app.main import app # noqa: E402
@pytest.fixture
diff --git a/flowsint-core/src/flowsint_core/core/logger.py b/flowsint-core/src/flowsint_core/core/logger.py
index 21a7a286..d8668dc9 100644
--- a/flowsint-core/src/flowsint_core/core/logger.py
+++ b/flowsint-core/src/flowsint_core/core/logger.py
@@ -13,7 +13,7 @@ import atexit
import threading
import time
from datetime import datetime, timezone
-from queue import Queue
+from queue import Empty, Queue
from typing import Dict, Optional, Union
from uuid import UUID
@@ -132,7 +132,7 @@ class LoggerSingleton:
):
try:
logs_to_insert.append(self._log_queue.get_nowait())
- except:
+ except Empty:
break
if not logs_to_insert:
diff --git a/flowsint-core/tests/core/base.py b/flowsint-core/tests/core/base.py
index 2f9972ab..2e0e7416 100644
--- a/flowsint-core/tests/core/base.py
+++ b/flowsint-core/tests/core/base.py
@@ -1,4 +1,5 @@
from typing import List
+
from flowsint_core.core.enricher_base import Enricher
from flowsint_types import Phone
from flowsint_types.domain import Domain
@@ -46,15 +47,15 @@ def test_correct_preprocess():
assert len(preprocessed) == 3 # 3 valid domains
assert preprocessed[0].domain == "mydomain.com"
assert preprocessed[0].label == "mydomain.com"
- assert preprocessed[0].root == True
+ assert preprocessed[0].root is True
assert preprocessed[1].domain == "blog.mydomain2.com"
assert preprocessed[1].label == "blog.mydomain2.com"
- assert preprocessed[1].root == False
+ assert preprocessed[1].root is False
assert preprocessed[2].domain == "mydomain3.com"
assert preprocessed[2].label == "mydomain3.com"
- assert preprocessed[2].root == True
+ assert preprocessed[2].root is True
def test_incorrect_preprocess():
@@ -67,5 +68,5 @@ def test_incorrect_preprocess():
preprocessed = enricher.preprocess(inputs)
assert len(preprocessed) == 1 # 1 valid domain
assert preprocessed[0].domain == "mydomain.com"
- assert preprocessed[0].root == True
+ assert preprocessed[0].root is True
assert preprocessed[0].label == "mydomain.com"
diff --git a/flowsint-core/tests/import/test_entity_detection.py b/flowsint-core/tests/import/test_entity_detection.py
index caa26887..fbb45d95 100644
--- a/flowsint-core/tests/import/test_entity_detection.py
+++ b/flowsint-core/tests/import/test_entity_detection.py
@@ -1,6 +1,5 @@
-from flowsint_types import Domain, File, Ip, Phone
-
from flowsint_core.imports import detect_type
+from flowsint_types import Domain, File, Ip, Phone
def test_detection_domains():
@@ -15,13 +14,13 @@ def test_detection_domains():
results.append(py_obj)
# domain 1
assert results[0].domain == "mydomain.com"
- assert results[0].root == True
+ assert results[0].root is True
# domain 2
assert results[1].domain == "myseconddomain.com"
- assert results[1].root == True
+ assert results[1].root is True
# domain 3
assert results[2].domain == "blog.subdomain.uk"
- assert results[2].root == False
+ assert results[2].root is False
def test_detection_ips():
@@ -69,7 +68,7 @@ def test_detection_mixed():
# domain
assert isinstance(results[1], Domain)
assert results[1].domain == "my.domain.io"
- assert results[1].root == False
+ assert results[1].root is False
# phone
assert isinstance(results[2], Phone)
assert results[2].number == "+33632233223"
diff --git a/flowsint-core/tests/repositories/test_investigation_repository.py b/flowsint-core/tests/repositories/test_investigation_repository.py
index 31edf697..cf274e45 100644
--- a/flowsint-core/tests/repositories/test_investigation_repository.py
+++ b/flowsint-core/tests/repositories/test_investigation_repository.py
@@ -1,14 +1,16 @@
"""Tests for InvestigationRepository."""
+
from uuid import uuid4
+from flowsint_core.core.models import InvestigationUserRole
+from flowsint_core.core.repositories import InvestigationRepository
+from flowsint_core.core.types import Role
+
from tests.factories import (
InvestigationFactory,
InvestigationUserRoleFactory,
ProfileFactory,
)
-from flowsint_core.core.models import InvestigationUserRole
-from flowsint_core.core.repositories import InvestigationRepository
-from flowsint_core.core.types import Role
class TestInvestigationRepository:
@@ -31,9 +33,7 @@ class TestInvestigationRepository:
self._setup(db_session)
user = ProfileFactory()
inv = InvestigationFactory(owner=user)
- InvestigationUserRoleFactory(
- user=user, investigation=inv, roles=[Role.OWNER]
- )
+ InvestigationUserRoleFactory(user=user, investigation=inv, roles=[Role.OWNER])
# Another user's investigation (should not appear)
other_user = ProfileFactory()
@@ -52,9 +52,7 @@ class TestInvestigationRepository:
self._setup(db_session)
user = ProfileFactory()
inv = InvestigationFactory()
- InvestigationUserRoleFactory(
- user=user, investigation=inv, roles=[Role.VIEWER]
- )
+ InvestigationUserRoleFactory(user=user, investigation=inv, roles=[Role.VIEWER])
repo = InvestigationRepository(db_session)
@@ -99,9 +97,7 @@ class TestInvestigationRepository:
self._setup(db_session)
user = ProfileFactory()
inv = InvestigationFactory()
- role_entry = InvestigationUserRoleFactory(
- user=user, investigation=inv, roles=[Role.EDITOR]
- )
+ InvestigationUserRoleFactory(user=user, investigation=inv, roles=[Role.EDITOR])
repo = InvestigationRepository(db_session)
result = repo.get_user_role(user.id, inv.id)
diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_dehashed.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_dehashed.py
index 80fd8ba8..f4e52d21 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_dehashed.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_dehashed.py
@@ -1,13 +1,14 @@
import json
import os
-import requests
-
from typing import Any, Dict, List, Optional
+
+import requests
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.domain import Domain
from flowsint_types.individual import Individual
-from flowsint_core.core.logger import Logger
+
@flowsint_enricher
class DomainToDehashed(Enricher):
@@ -41,7 +42,7 @@ class DomainToDehashed(Enricher):
"""Declare required parameters for this enricher"""
return [
{
- "name": "DEHASHED_API_KEY", # Get your API key from dehashed.com/api
+ "name": "DEHASHED_API_KEY", # Get your API key from dehashed.com/api
"type": "vaultSecret",
"description": "Your Dehashed API key.",
"required": True,
@@ -63,14 +64,22 @@ class DomainToDehashed(Enricher):
async def scan(self, data: List[InputType]) -> List[OutputType]:
results: List[OutputType] = []
- api_key = self.get_secret("DEHASHED_API_KEY", os.getenv("DEHASHED_API_KEY"))
+ api_key = self.get_secret("DEHASHED_API_KEY", os.getenv("DEHASHED_API_KEY"))
for domain in data:
try:
- headers = {'Dehashed-Api-Key': api_key, 'Content-Type': 'application/json'}
+ headers = {
+ "Dehashed-Api-Key": api_key,
+ "Content-Type": "application/json",
+ }
raw_data = json.dumps({"query": f"domain:{domain.domain}"})
- api_request = requests.post(f'https://api.dehashed.com/v2/search', data=raw_data, headers=headers, timeout=30)
+ api_request = requests.post(
+ "https://api.dehashed.com/v2/search",
+ data=raw_data,
+ headers=headers,
+ timeout=30,
+ )
if api_request.status_code != 200:
if api_request.status_code == 401:
@@ -92,23 +101,30 @@ class DomainToDehashed(Enricher):
try:
response_json = api_request.json()
except Exception as e:
- Logger.error(None, {"message": f"(DomainToDehashed) Failed to parse JSON for {domain.domain}: {e}"})
+ Logger.error(
+ None,
+ {
+ "message": f"(DomainToDehashed) Failed to parse JSON for {domain.domain}: {e}"
+ },
+ )
continue
dehashed_entries = response_json.get("entries", [])
if not dehashed_entries:
Logger.error(
- self.sketch_id,
- {
- "message": f"(DomainToDehashed) Enricher failed for the domain: '{domain.domain}': {api_request.text}"
- },
- )
+ self.sketch_id,
+ {
+ "message": f"(DomainToDehashed) Enricher failed for the domain: '{domain.domain}': {api_request.text}"
+ },
+ )
continue
-
api_balance = response_json.get("balance")
if api_balance:
- Logger.info(self.sketch_id, f'(DomainToDehashed) Your remaining API balance is {api_balance}.')
+ Logger.info(
+ self.sketch_id,
+ f"(DomainToDehashed) Your remaining API balance is {api_balance}.",
+ )
for entry in dehashed_entries:
entry_email = entry.get("email")
@@ -121,18 +137,25 @@ class DomainToDehashed(Enricher):
results.append(
Individual(
- full_name=entry_name[0] if entry_name else None,
- birth_date=entry_dob[0] if entry_dob else None,
- email_addresses=entry_email if entry_email else None,
- phone_numbers=entry_phone if entry_phone else None,
- social_media_profiles=entry_socialmedia if entry_socialmedia else None,
- ip_addresses=entry_ip if entry_ip else None,
- usernames=entry_username if entry_username else None
- )
+ full_name=entry_name[0] if entry_name else None,
+ birth_date=entry_dob[0] if entry_dob else None,
+ email_addresses=entry_email if entry_email else None,
+ phone_numbers=entry_phone if entry_phone else None,
+ social_media_profiles=(
+ entry_socialmedia if entry_socialmedia else None
+ ),
+ ip_addresses=entry_ip if entry_ip else None,
+ usernames=entry_username if entry_username else None,
)
+ )
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"(DomainToDehashed) Exception while querying {domain.domain}: {e}"})
-
+ Logger.error(
+ self.sketch_id,
+ {
+ "message": f"(DomainToDehashed) Exception while querying {domain.domain}: {e}"
+ },
+ )
+
return results
def postprocess(
@@ -158,4 +181,4 @@ class DomainToDehashed(Enricher):
# Make types available at module level for easy access
InputType = DomainToDehashed.InputType
-OutputType = DomainToDehashed.OutputType
\ No newline at end of file
+OutputType = DomainToDehashed.OutputType
diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_dummy_individuals.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_dummy_individuals.py
index 90e62256..9ed864f0 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_dummy_individuals.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_dummy_individuals.py
@@ -1,6 +1,5 @@
-import socket
from typing import Dict, List
-from flowsint_core.core.logger import Logger
+
from flowsint_core.core.enricher_base import Enricher
from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.domain import Domain
@@ -56,7 +55,7 @@ class ToDummyEnricher(Enricher):
item,
"IS_RELATED_T0",
)
- self.log_graph_message(f"Dummy enricher finished.")
+ self.log_graph_message("Dummy enricher finished.")
return results
diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_whois.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_whois.py
index df91076b..ece2a057 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_whois.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_whois.py
@@ -1,12 +1,13 @@
from typing import List
+
import whois
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.domain import Domain
-from flowsint_types.whois import Whois
from flowsint_types.email import Email
from flowsint_types.organization import Organization
-from flowsint_core.core.logger import Logger
+from flowsint_types.whois import Whois
@flowsint_enricher
@@ -29,8 +30,9 @@ class WhoisEnricher(Enricher):
def key(cls) -> str:
return "domain"
- async def scan(self, data: List[InputType]) -> List[OutputType]:
- results: List[OutputType] = []
+ # noqa false positive below: OutputType is a class attr, resolves fine at def-time
+ async def scan(self, data: List[InputType]) -> List[OutputType]: # noqa: F821
+ results: List[OutputType] = [] # noqa: F821
for domain in data:
try:
whois_info = whois.whois(domain.domain)
@@ -109,7 +111,9 @@ class WhoisEnricher(Enricher):
return results
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
for whois_obj in results:
if not self._graph_service:
continue
@@ -122,7 +126,9 @@ class WhoisEnricher(Enricher):
# Create organization node if available
if whois_obj.organization:
self.create_node(whois_obj.organization)
- self.create_relationship(whois_obj.organization, whois_obj.domain, "HAS_DOMAIN")
+ self.create_relationship(
+ whois_obj.organization, whois_obj.domain, "HAS_DOMAIN"
+ )
# Create email node if available
if whois_obj.email:
self.create_node(whois_obj.email)
diff --git a/flowsint-enrichers/src/flowsint_enrichers/email/to_dehashed.py b/flowsint-enrichers/src/flowsint_enrichers/email/to_dehashed.py
index 622e8d16..0a9ca678 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/email/to_dehashed.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/email/to_dehashed.py
@@ -1,13 +1,14 @@
import json
import os
-import requests
-
from typing import Any, Dict, List, Optional
+
+import requests
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.email import Email
from flowsint_types.individual import Individual
-from flowsint_core.core.logger import Logger
+
@flowsint_enricher
class EmailToDehashed(Enricher):
@@ -41,7 +42,7 @@ class EmailToDehashed(Enricher):
"""Declare required parameters for this enricher"""
return [
{
- "name": "DEHASHED_API_KEY", # Get your API key from dehashed.com/api
+ "name": "DEHASHED_API_KEY", # Get your API key from dehashed.com/api
"type": "vaultSecret",
"description": "Your Dehashed API key.",
"required": True,
@@ -63,14 +64,22 @@ class EmailToDehashed(Enricher):
async def scan(self, data: List[InputType]) -> List[OutputType]:
results: List[OutputType] = []
- api_key = self.get_secret("DEHASHED_API_KEY", os.getenv("DEHASHED_API_KEY"))
+ api_key = self.get_secret("DEHASHED_API_KEY", os.getenv("DEHASHED_API_KEY"))
for email in data:
try:
- headers = {'Dehashed-Api-Key': api_key, 'Content-Type': 'application/json'}
+ headers = {
+ "Dehashed-Api-Key": api_key,
+ "Content-Type": "application/json",
+ }
raw_data = json.dumps({"query": f"email:{email.email}"})
- api_request = requests.post(f'https://api.dehashed.com/v2/search', data=raw_data, headers=headers, timeout=30)
+ api_request = requests.post(
+ "https://api.dehashed.com/v2/search",
+ data=raw_data,
+ headers=headers,
+ timeout=30,
+ )
if api_request.status_code != 200:
if api_request.status_code == 401:
@@ -92,23 +101,30 @@ class EmailToDehashed(Enricher):
try:
response_json = api_request.json()
except Exception as e:
- Logger.error(None, {"message": f"(EmailToDehashed) Failed to parse JSON for {email.email}: {e}"})
+ Logger.error(
+ None,
+ {
+ "message": f"(EmailToDehashed) Failed to parse JSON for {email.email}: {e}"
+ },
+ )
continue
dehashed_entries = response_json.get("entries", [])
if not dehashed_entries:
Logger.error(
- self.sketch_id,
- {
- "message": f"(EmailToDehashed) Enricher failed for the email: '{email.email}': {api_request.text}"
- },
- )
+ self.sketch_id,
+ {
+ "message": f"(EmailToDehashed) Enricher failed for the email: '{email.email}': {api_request.text}"
+ },
+ )
continue
-
api_balance = response_json.get("balance")
if api_balance:
- Logger.info(self.sketch_id, f'(EmailToDehashed) Your remaining API balance is {api_balance}.')
+ Logger.info(
+ self.sketch_id,
+ f"(EmailToDehashed) Your remaining API balance is {api_balance}.",
+ )
for entry in dehashed_entries:
entry_email = entry.get("email")
@@ -121,18 +137,25 @@ class EmailToDehashed(Enricher):
results.append(
Individual(
- full_name=entry_name[0] if entry_name else None,
- birth_date=entry_dob[0] if entry_dob else None,
- email_addresses=entry_email if entry_email else None,
- phone_numbers=entry_phone if entry_phone else None,
- social_media_profiles=entry_socialmedia if entry_socialmedia else None,
- ip_addresses=entry_ip if entry_ip else None,
- usernames=entry_username if entry_username else None
- )
+ full_name=entry_name[0] if entry_name else None,
+ birth_date=entry_dob[0] if entry_dob else None,
+ email_addresses=entry_email if entry_email else None,
+ phone_numbers=entry_phone if entry_phone else None,
+ social_media_profiles=(
+ entry_socialmedia if entry_socialmedia else None
+ ),
+ ip_addresses=entry_ip if entry_ip else None,
+ usernames=entry_username if entry_username else None,
)
+ )
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"(EmailToDehashed) Exception while querying email {email.email}: {e}"})
-
+ Logger.error(
+ self.sketch_id,
+ {
+ "message": f"(EmailToDehashed) Exception while querying email {email.email}: {e}"
+ },
+ )
+
return results
def postprocess(
@@ -158,4 +181,4 @@ class EmailToDehashed(Enricher):
# Make types available at module level for easy access
InputType = EmailToDehashed.InputType
-OutputType = EmailToDehashed.OutputType
\ No newline at end of file
+OutputType = EmailToDehashed.OutputType
diff --git a/flowsint-enrichers/src/flowsint_enrichers/ip/to_dehashed.py b/flowsint-enrichers/src/flowsint_enrichers/ip/to_dehashed.py
index 6612f56e..69175c7e 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/ip/to_dehashed.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/ip/to_dehashed.py
@@ -1,13 +1,14 @@
import json
import os
-import requests
-
from typing import Any, Dict, List, Optional
+
+import requests
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
-from flowsint_types.ip import Ip
from flowsint_types.individual import Individual
-from flowsint_core.core.logger import Logger
+from flowsint_types.ip import Ip
+
@flowsint_enricher
class IpToIntelligence(Enricher):
@@ -41,7 +42,7 @@ class IpToIntelligence(Enricher):
"""Declare required parameters for this enricher"""
return [
{
- "name": "DEHASHED_API_KEY", # Get your API key from dehashed.com/api
+ "name": "DEHASHED_API_KEY", # Get your API key from dehashed.com/api
"type": "vaultSecret",
"description": "Your Dehashed API key.",
"required": True,
@@ -63,14 +64,22 @@ class IpToIntelligence(Enricher):
async def scan(self, data: List[InputType]) -> List[OutputType]:
results: List[OutputType] = []
- api_key = self.get_secret("DEHASHED_API_KEY", os.getenv("DEHASHED_API_KEY"))
+ api_key = self.get_secret("DEHASHED_API_KEY", os.getenv("DEHASHED_API_KEY"))
for ip in data:
try:
- headers = {'Dehashed-Api-Key': api_key, 'Content-Type': 'application/json'}
+ headers = {
+ "Dehashed-Api-Key": api_key,
+ "Content-Type": "application/json",
+ }
raw_data = json.dumps({"query": f"ip_address:{ip.address}"})
- api_request = requests.post(f'https://api.dehashed.com/v2/search', data=raw_data, headers=headers, timeout=30)
+ api_request = requests.post(
+ "https://api.dehashed.com/v2/search",
+ data=raw_data,
+ headers=headers,
+ timeout=30,
+ )
if api_request.status_code != 200:
if api_request.status_code == 401:
@@ -92,25 +101,32 @@ class IpToIntelligence(Enricher):
try:
response_json = api_request.json()
except Exception as e:
- Logger.error(None, {"message": f"(IpToIntelligence) Failed to parse JSON for {ip.address}: {e}"})
+ Logger.error(
+ None,
+ {
+ "message": f"(IpToIntelligence) Failed to parse JSON for {ip.address}: {e}"
+ },
+ )
continue
dehashed_entries = response_json.get("entries", [])
if not dehashed_entries:
Logger.error(
- self.sketch_id,
- {
- "message": f"(IpToIntelligence) Enricher failed for the IP address: '{ip.address}': {api_request.text}"
- },
- )
+ self.sketch_id,
+ {
+ "message": f"(IpToIntelligence) Enricher failed for the IP address: '{ip.address}': {api_request.text}"
+ },
+ )
continue
-
api_balance = response_json.get("balance")
if api_balance:
- Logger.info(self.sketch_id, f'(EmailToDehashed) Your remaining API balance is {api_balance}.')
-
- time_took = response_json.get("took")
+ Logger.info(
+ self.sketch_id,
+ f"(EmailToDehashed) Your remaining API balance is {api_balance}.",
+ )
+
+ response_json.get("took")
for entry in dehashed_entries:
entry_email = entry.get("email")
@@ -123,18 +139,25 @@ class IpToIntelligence(Enricher):
results.append(
Individual(
- full_name=entry_name[0] if entry_name else None,
- birth_date=entry_dob[0] if entry_dob else None,
- email_addresses=entry_email if entry_email else None,
- phone_numbers=entry_phone if entry_phone else None,
- social_media_profiles=entry_socialmedia if entry_socialmedia else None,
- ip_addresses=entry_ip if entry_ip else None,
- usernames=entry_username if entry_username else None
- )
+ full_name=entry_name[0] if entry_name else None,
+ birth_date=entry_dob[0] if entry_dob else None,
+ email_addresses=entry_email if entry_email else None,
+ phone_numbers=entry_phone if entry_phone else None,
+ social_media_profiles=(
+ entry_socialmedia if entry_socialmedia else None
+ ),
+ ip_addresses=entry_ip if entry_ip else None,
+ usernames=entry_username if entry_username else None,
)
+ )
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"(IpToIntelligence) Exception while querying {ip.address}: {e}"})
-
+ Logger.error(
+ self.sketch_id,
+ {
+ "message": f"(IpToIntelligence) Exception while querying {ip.address}: {e}"
+ },
+ )
+
return results
def postprocess(
@@ -160,4 +183,4 @@ class IpToIntelligence(Enricher):
# Make types available at module level for easy access
InputType = IpToIntelligence.InputType
-OutputType = IpToIntelligence.OutputType
\ No newline at end of file
+OutputType = IpToIntelligence.OutputType
diff --git a/flowsint-enrichers/src/flowsint_enrichers/ip/to_infos.py b/flowsint-enrichers/src/flowsint_enrichers/ip/to_infos.py
index fe8240df..3f894cf8 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/ip/to_infos.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/ip/to_infos.py
@@ -1,5 +1,6 @@
+from typing import Any, Dict, List
+
import requests
-from typing import List, Dict, Any
from flowsint_core.core.enricher_base import Enricher
from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.ip import Ip
@@ -25,8 +26,9 @@ class IpToInfosEnricher(Enricher):
def key(cls) -> str:
return "address"
- async def scan(self, data: List[InputType]) -> List[OutputType]:
- results: List[OutputType] = []
+ # noqa false positive below: OutputType is a class attr, resolves fine at def-time
+ async def scan(self, data: List[InputType]) -> List[OutputType]: # noqa: F821
+ results: List[OutputType] = [] # noqa: F821
for ip in data:
try:
geo_data = self.get_location_data(ip.address)
@@ -41,7 +43,9 @@ class IpToInfosEnricher(Enricher):
print(f"Error geolocating {ip.address}: {e}")
return results
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
"""Update IP nodes in Neo4j with geolocation information."""
if self._graph_service:
for ip in results:
diff --git a/flowsint-enrichers/src/flowsint_enrichers/ip/to_ports.py b/flowsint-enrichers/src/flowsint_enrichers/ip/to_ports.py
index ad738160..2cc5364f 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/ip/to_ports.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/ip/to_ports.py
@@ -1,10 +1,10 @@
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional
+
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.ip import Ip
from flowsint_types.port import Port
-from flowsint_core.utils import is_valid_ip
-from flowsint_core.core.logger import Logger
from tools.network.naabu import NaabuTool
@@ -133,9 +133,7 @@ class IpToPortsEnricher(Enricher):
try:
Logger.info(
self.sketch_id,
- {
- "message": f"[NAABU] Scanning {ip.address} in {mode} mode..."
- },
+ {"message": f"[NAABU] Scanning {ip.address} in {mode} mode..."},
)
# Launch naabu scan
@@ -197,8 +195,7 @@ class IpToPortsEnricher(Enricher):
ip_address = getattr(port, "_ip_address", None)
if not ip_address:
continue
-
- port_id = f"{ip_address}:{port.number}"
+
self.create_node(port)
# Create relationship from IP to Port
diff --git a/flowsint-enrichers/src/flowsint_enrichers/social/to_dehashed.py b/flowsint-enrichers/src/flowsint_enrichers/social/to_dehashed.py
index 00638792..22860e48 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/social/to_dehashed.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/social/to_dehashed.py
@@ -1,13 +1,14 @@
import json
import os
-import requests
-
from typing import Any, Dict, List, Optional
+
+import requests
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
-from flowsint_types.username import Username
from flowsint_types.individual import Individual
-from flowsint_core.core.logger import Logger
+from flowsint_types.username import Username
+
@flowsint_enricher
class UsernameToDehashed(Enricher):
@@ -41,7 +42,7 @@ class UsernameToDehashed(Enricher):
"""Declare required parameters for this enricher"""
return [
{
- "name": "DEHASHED_API_KEY", # Get your API key from dehashed.com/api
+ "name": "DEHASHED_API_KEY", # Get your API key from dehashed.com/api
"type": "vaultSecret",
"description": "Your Dehashed API key.",
"required": True,
@@ -63,14 +64,22 @@ class UsernameToDehashed(Enricher):
async def scan(self, data: List[InputType]) -> List[OutputType]:
results: List[OutputType] = []
- api_key = self.get_secret("DEHASHED_API_KEY", os.getenv("DEHASHED_API_KEY"))
+ api_key = self.get_secret("DEHASHED_API_KEY", os.getenv("DEHASHED_API_KEY"))
for username in data:
try:
- headers = {'Dehashed-Api-Key': api_key, 'Content-Type': 'application/json'}
+ headers = {
+ "Dehashed-Api-Key": api_key,
+ "Content-Type": "application/json",
+ }
raw_data = json.dumps({"query": f"username:{username.value}"})
- api_request = requests.post(f'https://api.dehashed.com/v2/search', data=raw_data, headers=headers, timeout=30)
+ api_request = requests.post(
+ "https://api.dehashed.com/v2/search",
+ data=raw_data,
+ headers=headers,
+ timeout=30,
+ )
if api_request.status_code != 200:
if api_request.status_code == 401:
@@ -92,23 +101,30 @@ class UsernameToDehashed(Enricher):
try:
response_json = api_request.json()
except Exception as e:
- Logger.error(None, {"message": f"(UsernameToDehashed) Failed to parse JSON for {username.value}: {e}"})
+ Logger.error(
+ None,
+ {
+ "message": f"(UsernameToDehashed) Failed to parse JSON for {username.value}: {e}"
+ },
+ )
continue
dehashed_entries = response_json.get("entries", [])
if not dehashed_entries:
Logger.error(
- self.sketch_id,
- {
- "message": f"(UsernameToDehashed) failed for the username: '{username.value}': {api_request.text}"
- },
- )
+ self.sketch_id,
+ {
+ "message": f"(UsernameToDehashed) failed for the username: '{username.value}': {api_request.text}"
+ },
+ )
continue
-
api_balance = response_json.get("balance")
if api_balance:
- Logger.info(self.sketch_id, f'(UsernameToDehashed) Your remaining API balance is {api_balance}.')
+ Logger.info(
+ self.sketch_id,
+ f"(UsernameToDehashed) Your remaining API balance is {api_balance}.",
+ )
for entry in dehashed_entries:
entry_email = entry.get("email")
@@ -121,18 +137,25 @@ class UsernameToDehashed(Enricher):
results.append(
Individual(
- full_name=entry_name[0] if entry_name else None,
- birth_date=entry_dob[0] if entry_dob else None,
- email_addresses=entry_email if entry_email else None,
- phone_numbers=entry_phone if entry_phone else None,
- social_media_profiles=entry_socialmedia if entry_socialmedia else None,
- ip_addresses=entry_ip if entry_ip else None,
- usernames=entry_username if entry_username else None
- )
+ full_name=entry_name[0] if entry_name else None,
+ birth_date=entry_dob[0] if entry_dob else None,
+ email_addresses=entry_email if entry_email else None,
+ phone_numbers=entry_phone if entry_phone else None,
+ social_media_profiles=(
+ entry_socialmedia if entry_socialmedia else None
+ ),
+ ip_addresses=entry_ip if entry_ip else None,
+ usernames=entry_username if entry_username else None,
)
+ )
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"(UsernameToDehashed) Exception while querying username {username.value}: {e}"})
-
+ Logger.error(
+ self.sketch_id,
+ {
+ "message": f"(UsernameToDehashed) Exception while querying username {username.value}: {e}"
+ },
+ )
+
return results
def postprocess(
@@ -158,4 +181,4 @@ class UsernameToDehashed(Enricher):
# Make types available at module level for easy access
InputType = UsernameToDehashed.InputType
-OutputType = UsernameToDehashed.OutputType
\ No newline at end of file
+OutputType = UsernameToDehashed.OutputType
diff --git a/flowsint-enrichers/src/tools/dockertool.py b/flowsint-enrichers/src/tools/dockertool.py
index 94077da8..ebeee776 100644
--- a/flowsint-enrichers/src/tools/dockertool.py
+++ b/flowsint-enrichers/src/tools/dockertool.py
@@ -1,5 +1,6 @@
-from docker import from_env, DockerClient
-from docker.errors import ImageNotFound, APIError, DockerException
+from docker import from_env
+from docker.errors import APIError, DockerException, ImageNotFound
+
from .base import Tool
@@ -46,7 +47,14 @@ class DockerTool(Tool):
except ImageNotFound:
return False
- def launch(self, command, volumes: dict = None, timeout: int = 30, environment: dict = None, entrypoint=None):
+ def launch(
+ self,
+ command,
+ volumes: dict = None,
+ timeout: int = 30,
+ environment: dict = None,
+ entrypoint=None,
+ ):
self.install()
# Merge default environment with custom environment
env = {"TERM": "dumb"} # Set terminal type to avoid TTY issues
@@ -81,7 +89,7 @@ class DockerTool(Tool):
try:
error_json = e.response.json()
error_detail = f"{str(e)} - Details: {error_json}"
- except:
+ except Exception:
pass
# Check if it's a container exit error
diff --git a/flowsint-enrichers/tests/tools/network/asnmap.py b/flowsint-enrichers/tests/tools/network/asnmap.py
index 12f83fbe..dc9d247f 100644
--- a/flowsint-enrichers/tests/tools/network/asnmap.py
+++ b/flowsint-enrichers/tests/tools/network/asnmap.py
@@ -1,5 +1,6 @@
import re
from typing import Dict
+
from tools.network.asnmap import AsnmapTool
tool = AsnmapTool()
@@ -23,7 +24,7 @@ def test_image():
def test_install():
tool.install()
- assert tool.is_installed() == True
+ assert tool.is_installed() is True
def test_version():
diff --git a/flowsint-enrichers/tests/tools/network/httpx.py b/flowsint-enrichers/tests/tools/network/httpx.py
index 63af13a6..b0adf4a1 100644
--- a/flowsint-enrichers/tests/tools/network/httpx.py
+++ b/flowsint-enrichers/tests/tools/network/httpx.py
@@ -1,5 +1,6 @@
import re
from typing import List
+
from tools.network.httpx import HttpxTool
tool = HttpxTool()
@@ -26,7 +27,7 @@ def test_image():
def test_install():
tool.install()
- assert tool.is_installed() == True
+ assert tool.is_installed() is True
def test_version():
diff --git a/flowsint-enrichers/tests/tools/network/naabu.py b/flowsint-enrichers/tests/tools/network/naabu.py
index 2c8986d4..740b8359 100644
--- a/flowsint-enrichers/tests/tools/network/naabu.py
+++ b/flowsint-enrichers/tests/tools/network/naabu.py
@@ -1,5 +1,5 @@
import re
-from typing import List
+
from tools.network.naabu import NaabuTool
tool = NaabuTool()
@@ -26,7 +26,7 @@ def test_image():
def test_install():
tool.install()
- assert tool.is_installed() == True
+ assert tool.is_installed() is True
def test_version():
diff --git a/flowsint-enrichers/tests/tools/network/reconcrawl.py b/flowsint-enrichers/tests/tools/network/reconcrawl.py
index 0bffd125..bd424521 100644
--- a/flowsint-enrichers/tests/tools/network/reconcrawl.py
+++ b/flowsint-enrichers/tests/tools/network/reconcrawl.py
@@ -20,4 +20,4 @@ def test_category():
def test_install():
tool.install()
- assert tool.is_installed() == True
+ assert tool.is_installed() is True
diff --git a/flowsint-enrichers/tests/tools/network/subfinder.py b/flowsint-enrichers/tests/tools/network/subfinder.py
index 3219e214..d8346e68 100644
--- a/flowsint-enrichers/tests/tools/network/subfinder.py
+++ b/flowsint-enrichers/tests/tools/network/subfinder.py
@@ -1,4 +1,5 @@
import re
+
from tools.network.subfinder import SubfinderTool
tool = SubfinderTool()
@@ -22,7 +23,7 @@ def test_image():
def test_install():
tool.install()
- assert tool.is_installed() == True
+ assert tool.is_installed() is True
def test_version():
diff --git a/flowsint-types/src/flowsint_types/asn.py b/flowsint-types/src/flowsint_types/asn.py
index d7d7a1c2..70ea5998 100644
--- a/flowsint-types/src/flowsint_types/asn.py
+++ b/flowsint-types/src/flowsint_types/asn.py
@@ -1,6 +1,8 @@
-from typing import List, Optional, Union, Self
-from pydantic import Field, field_validator, model_validator
import re
+from typing import List, Optional, Self
+
+from pydantic import Field, field_validator, model_validator
+
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -100,6 +102,6 @@ class ASN(FlowsintType):
# Import CIDR here to avoid circular import
-from .cidr import CIDR
+from .cidr import CIDR # noqa: E402
ASN.model_rebuild()
diff --git a/flowsint-types/tests/test_registry.py b/flowsint-types/tests/test_registry.py
index ec269500..5a267936 100644
--- a/flowsint-types/tests/test_registry.py
+++ b/flowsint-types/tests/test_registry.py
@@ -1,4 +1,4 @@
-from flowsint_types import Domain, Individual, Ip, get_type
+from flowsint_types import Domain, Ip, get_type
from flowsint_types.registry import TYPE_REGISTRY
@@ -20,11 +20,6 @@ def test_get_by_lower_key():
assert type is Domain
-def test_get_by_lower_key():
- type = TYPE_REGISTRY.get_lowercase("domain")
- assert type is Domain
-
-
def get_type_function():
# Probably the main util function used here, as we can pass it either capital or lowercase types
type_lower = get_type("domain")