Skip to content

A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.

style(python): apply black/isort formatting workspace-wide

dextmorgneb30268c5631
diff --git a/flowsint-api/app/api/deps.py b/flowsint-api/app/api/deps.py
index 3dd967c4..6a5ff5fa 100644
--- a/flowsint-api/app/api/deps.py
+++ b/flowsint-api/app/api/deps.py
@@ -1,10 +1,10 @@
from fastapi import Depends, HTTPException, status
from fastapi.security import OAuth2PasswordBearer
-from jose import JWTError, jwt
-from sqlalchemy.orm import Session
from flowsint_core.core.auth import ALGORITHM, AUTH_SECRET
-from flowsint_core.core.postgre_db import get_db
from flowsint_core.core.models import Profile
+from flowsint_core.core.postgre_db import get_db
+from jose import JWTError, jwt
+from sqlalchemy.orm import Session
oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")
diff --git a/flowsint-api/app/api/routes/analysis.py b/flowsint-api/app/api/routes/analysis.py
index f311ec39..58982c8a 100644
--- a/flowsint-api/app/api/routes/analysis.py
+++ b/flowsint-api/app/api/routes/analysis.py
@@ -1,17 +1,18 @@
-from uuid import UUID
-from fastapi import APIRouter, HTTPException, Depends, status
from typing import List
-from sqlalchemy.orm import Session
+from uuid import UUID
-from flowsint_core.core.postgre_db import get_db
+from fastapi import APIRouter, Depends, HTTPException, status
from flowsint_core.core.models import Profile
+from flowsint_core.core.postgre_db import get_db
from flowsint_core.core.services import (
- create_analysis_service,
NotFoundError,
PermissionDeniedError,
+ create_analysis_service,
)
+from sqlalchemy.orm import Session
+
from app.api.deps import get_current_user
-from app.api.schemas.analysis import AnalysisRead, AnalysisCreate, AnalysisUpdate
+from app.api.schemas.analysis import AnalysisCreate, AnalysisRead, AnalysisUpdate
router = APIRouter()
diff --git a/flowsint-api/app/api/routes/auth.py b/flowsint-api/app/api/routes/auth.py
index 82752080..bee6f3e5 100644
--- a/flowsint-api/app/api/routes/auth.py
+++ b/flowsint-api/app/api/routes/auth.py
@@ -1,19 +1,20 @@
-from fastapi import APIRouter, Depends, HTTPException, Query
-from fastapi.security import OAuth2PasswordRequestForm
-from sqlalchemy.orm import Session
-from sqlalchemy.exc import SQLAlchemyError
from typing import List
+from fastapi import APIRouter, Depends, HTTPException, Query
+from fastapi.security import OAuth2PasswordRequestForm
+from flowsint_core.core.models import Profile
+from flowsint_core.core.postgre_db import get_db
from flowsint_core.core.services import (
- create_auth_service,
AuthenticationError,
ConflictError,
DatabaseError,
+ create_auth_service,
)
-from flowsint_core.core.models import Profile
-from flowsint_core.core.postgre_db import get_db
-from app.api.schemas.profile import ProfileCreate, ProfileRead, ProfileUpdate
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.orm import Session
+
from app.api.deps import get_current_user
+from app.api.schemas.profile import ProfileCreate, ProfileRead, ProfileUpdate
router = APIRouter()
diff --git a/flowsint-api/app/api/routes/chat.py b/flowsint-api/app/api/routes/chat.py
index 7b081a5d..3f623e1d 100644
--- a/flowsint-api/app/api/routes/chat.py
+++ b/flowsint-api/app/api/routes/chat.py
@@ -1,4 +1,4 @@
-from typing import Dict, List, Optional
+from typing import List, Optional
from uuid import UUID
from fastapi import APIRouter, Depends, HTTPException, status
diff --git a/flowsint-api/app/api/routes/enricher_templates.py b/flowsint-api/app/api/routes/enricher_templates.py
index d4e5d9f7..142be263 100644
--- a/flowsint-api/app/api/routes/enricher_templates.py
+++ b/flowsint-api/app/api/routes/enricher_templates.py
@@ -16,10 +16,10 @@ from flowsint_core.core.services import (
from flowsint_core.core.template_enricher import TemplateEnricher
from flowsint_core.core.vault import Vault
from flowsint_core.templates.types import Template
+from flowsint_types.registry import get_type as get_type_from_registry
+from flowsint_types.registry import load_all_types
from sqlalchemy.orm import Session
-from flowsint_types.registry import get_type as get_type_from_registry, load_all_types
-
from app.api.deps import get_current_user
from app.api.schemas.enricher_template import (
EnricherTemplateCreate,
diff --git a/flowsint-api/app/api/routes/enrichers.py b/flowsint-api/app/api/routes/enrichers.py
index 7e8dbc75..b2c96053 100644
--- a/flowsint-api/app/api/routes/enrichers.py
+++ b/flowsint-api/app/api/routes/enrichers.py
@@ -9,7 +9,9 @@ from flowsint_core.core.services import (
create_enricher_service,
create_enricher_template_service,
)
-from flowsint_core.core.services.type_registry_service import create_type_registry_service
+from flowsint_core.core.services.type_registry_service import (
+ create_type_registry_service,
+)
from flowsint_enrichers import ENRICHER_REGISTRY, load_all_enrichers
from pydantic import BaseModel
from sqlalchemy.orm import Session
@@ -51,7 +53,9 @@ async def launch_enricher(
# Retrieve nodes from Neo4J by their element IDs
type_registry = create_type_registry_service(db)
resolver = type_registry.build_type_resolver(current_user.id)
- graph_service = create_graph_service(sketch_id=payload.sketch_id, type_resolver=resolver)
+ graph_service = create_graph_service(
+ sketch_id=payload.sketch_id, type_resolver=resolver
+ )
entities = graph_service.get_nodes_by_ids_for_task(payload.node_ids)
# Send deserialized nodes
diff --git a/flowsint-api/app/api/routes/events.py b/flowsint-api/app/api/routes/events.py
index cc891f37..5cc367e7 100644
--- a/flowsint-api/app/api/routes/events.py
+++ b/flowsint-api/app/api/routes/events.py
@@ -1,19 +1,20 @@
-from fastapi import APIRouter, Depends, HTTPException, Request
-from sqlalchemy.orm import Session
-from sse_starlette.sse import EventSourceResponse
-import json
import asyncio
+import json
from datetime import datetime
-from flowsint_core.core.postgre_db import get_db
+from fastapi import APIRouter, Depends, HTTPException, Request
from flowsint_core.core.events import event_emitter
from flowsint_core.core.models import Profile
+from flowsint_core.core.postgre_db import get_db
from flowsint_core.core.services import (
- create_log_service,
+ DatabaseError,
NotFoundError,
PermissionDeniedError,
- DatabaseError,
+ create_log_service,
)
+from sqlalchemy.orm import Session
+from sse_starlette.sse import EventSourceResponse
+
from app.api.deps import get_current_user
router = APIRouter()
@@ -130,7 +131,9 @@ async def stream_sketch_status(
channel = f"{sketch_id}_status"
await event_emitter.subscribe(channel)
try:
- yield json.dumps({"event": "connected", "data": "Connected to status stream"})
+ yield json.dumps(
+ {"event": "connected", "data": "Connected to status stream"}
+ )
while True:
if await request.is_disconnected():
@@ -145,7 +148,9 @@ async def stream_sketch_status(
await asyncio.sleep(0.1)
except asyncio.CancelledError:
- print(f"[EventEmitter] Client disconnected from status stream for sketch_id: {sketch_id}")
+ print(
+ f"[EventEmitter] Client disconnected from status stream for sketch_id: {sketch_id}"
+ )
except Exception as e:
print(f"[EventEmitter] Error in stream_sketch_status: {str(e)}")
finally:
diff --git a/flowsint-api/app/api/routes/investigations.py b/flowsint-api/app/api/routes/investigations.py
index 11e549f9..7e10ed0d 100644
--- a/flowsint-api/app/api/routes/investigations.py
+++ b/flowsint-api/app/api/routes/investigations.py
@@ -1,26 +1,27 @@
-from uuid import UUID
-from fastapi import APIRouter, HTTPException, Depends, status
from typing import List
-from sqlalchemy.orm import Session
+from uuid import UUID
-from flowsint_core.core.types import Role
-from flowsint_core.core.postgre_db import get_db
+from fastapi import APIRouter, Depends, HTTPException, status
from flowsint_core.core.models import Profile
+from flowsint_core.core.postgre_db import get_db
from flowsint_core.core.services import (
- create_investigation_service,
- NotFoundError,
- PermissionDeniedError,
ConflictError,
DatabaseError,
+ NotFoundError,
+ PermissionDeniedError,
+ create_investigation_service,
)
+from flowsint_core.core.types import Role
+from sqlalchemy.orm import Session
+
from app.api.deps import get_current_user
from app.api.schemas.investigation import (
- InvestigationRead,
- InvestigationCreate,
- InvestigationUpdate,
CollaboratorAdd,
- CollaboratorUpdate,
CollaboratorRead,
+ CollaboratorUpdate,
+ InvestigationCreate,
+ InvestigationRead,
+ InvestigationUpdate,
)
from app.api.schemas.sketch import SketchRead
@@ -147,9 +148,7 @@ def delete_investigation(
# ── Collaborator endpoints ───────────────────────────────────────────
- "/{investigation_id}/collaborators", response_model=List[CollaboratorRead]
-)
[email protected]("/{investigation_id}/collaborators", response_model=List[CollaboratorRead])
def get_collaborators(
investigation_id: UUID,
db: Session = Depends(get_db),
diff --git a/flowsint-api/app/api/routes/sketches.py b/flowsint-api/app/api/routes/sketches.py
index d7f20a2e..0762e836 100644
--- a/flowsint-api/app/api/routes/sketches.py
+++ b/flowsint-api/app/api/routes/sketches.py
@@ -11,24 +11,24 @@ from fastapi import (
UploadFile,
status,
)
-from flowsint_core.core.graph import GraphNode
+from flowsint_core.core.graph import GraphNode, create_graph_service
from flowsint_core.core.models import Profile
from flowsint_core.core.postgre_db import get_db
from flowsint_core.core.services import (
- create_sketch_service,
+ DatabaseError,
NotFoundError,
PermissionDeniedError,
ValidationError,
- DatabaseError,
+ create_sketch_service,
+)
+from flowsint_core.core.services.type_registry_service import (
+ create_type_registry_service,
)
-from flowsint_core.core.services.type_registry_service import create_type_registry_service
from flowsint_core.imports import (
EntityMapping,
- ImportService,
- create_import_service,
FileParseResult,
+ create_import_service,
)
-from flowsint_core.core.graph import create_graph_service
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
@@ -93,6 +93,7 @@ class UpdatePositionsInput(BaseModel):
class EntityMappingInput(BaseModel):
"""Pydantic model for parsing entity mapping input from frontend."""
+
id: str
entity_type: str
include: bool = True
@@ -103,6 +104,7 @@ class EntityMappingInput(BaseModel):
class ImportExecuteResponse(BaseModel):
"""Response model for import execution."""
+
status: str
nodes_created: int
nodes_skipped: int
@@ -162,7 +164,9 @@ def update_sketch(
):
service = create_sketch_service(db)
try:
- return service.update(id, current_user.id, payload.model_dump(exclude_unset=True))
+ return service.update(
+ id, current_user.id, payload.model_dump(exclude_unset=True)
+ )
except NotFoundError:
raise HTTPException(status_code=404, detail="Sketch not found")
except PermissionDeniedError:
@@ -237,7 +241,11 @@ def add_edge(
service = create_sketch_service(db)
try:
return service.add_relationship(
- UUID(sketch_id), current_user.id, relation.source, relation.target, relation.label
+ UUID(sketch_id),
+ current_user.id,
+ relation.source,
+ relation.target,
+ relation.label,
)
except NotFoundError:
raise HTTPException(status_code=404, detail="Sketch not found")
@@ -284,7 +292,9 @@ def update_node_positions(
service = create_sketch_service(db)
try:
positions = [pos.model_dump() for pos in data.positions]
- return service.update_node_positions(UUID(sketch_id), current_user.id, positions)
+ return service.update_node_positions(
+ UUID(sketch_id), current_user.id, positions
+ )
except NotFoundError:
raise HTTPException(status_code=404, detail="Sketch not found")
except PermissionDeniedError:
@@ -434,7 +444,9 @@ async def analyze_import_file(
try:
type_registry = create_type_registry_service(db)
resolver = type_registry.build_type_resolver(current_user.id)
- graph_service = create_graph_service(sketch_id=sketch_id, enable_batching=False, type_resolver=resolver)
+ graph_service = create_graph_service(
+ sketch_id=sketch_id, enable_batching=False, type_resolver=resolver
+ )
import_service = create_import_service(graph_service)
result = import_service.analyze_file(
file_content=content,
@@ -494,7 +506,9 @@ async def execute_import(
type_registry = create_type_registry_service(db)
resolver = type_registry.build_type_resolver(current_user.id)
- graph_service = create_graph_service(sketch_id=sketch_id, enable_batching=False, type_resolver=resolver)
+ graph_service = create_graph_service(
+ sketch_id=sketch_id, enable_batching=False, type_resolver=resolver
+ )
import_service = create_import_service(graph_service)
try:
diff --git a/flowsint-api/app/api/routes/types.py b/flowsint-api/app/api/routes/types.py
index 28a38806..c42ae6ca 100644
--- a/flowsint-api/app/api/routes/types.py
+++ b/flowsint-api/app/api/routes/types.py
@@ -1,10 +1,10 @@
from fastapi import APIRouter, Depends
-from pydantic import BaseModel
-from sqlalchemy.orm import Session
-
from flowsint_core.core.models import Profile
from flowsint_core.core.postgre_db import get_db
from flowsint_core.core.services import create_type_registry_service
+from pydantic import BaseModel
+from sqlalchemy.orm import Session
+
from app.api.deps import get_current_user
router = APIRouter()
diff --git a/flowsint-api/app/api/schemas/analysis.py b/flowsint-api/app/api/schemas/analysis.py
index 0ff2f57e..176f8ab0 100644
--- a/flowsint-api/app/api/schemas/analysis.py
+++ b/flowsint-api/app/api/schemas/analysis.py
@@ -1,7 +1,9 @@
-from .base import ORMBase
-from pydantic import UUID4, BaseModel
-from typing import Optional, Any
from datetime import datetime
+from typing import Any, Optional
+
+from pydantic import UUID4, BaseModel
+
+from .base import ORMBase
class AnalysisCreate(BaseModel):
diff --git a/flowsint-api/app/api/schemas/chat.py b/flowsint-api/app/api/schemas/chat.py
index 62f49468..dcf85aa5 100644
--- a/flowsint-api/app/api/schemas/chat.py
+++ b/flowsint-api/app/api/schemas/chat.py
@@ -1,7 +1,9 @@
-from .base import ORMBase
-from pydantic import UUID4, BaseModel
-from typing import List, Optional, Any
from datetime import datetime
+from typing import Any, List, Optional
+
+from pydantic import UUID4, BaseModel
+
+from .base import ORMBase
class ChatMessageRead(BaseModel):
diff --git a/flowsint-api/app/api/schemas/feedback.py b/flowsint-api/app/api/schemas/feedback.py
index 36596a10..4372f868 100644
--- a/flowsint-api/app/api/schemas/feedback.py
+++ b/flowsint-api/app/api/schemas/feedback.py
@@ -1,7 +1,9 @@
-from .base import ORMBase
-from pydantic import UUID4, BaseModel
-from typing import Optional
from datetime import datetime
+from typing import Optional
+
+from pydantic import UUID4, BaseModel
+
+from .base import ORMBase
class FeedbackCreate(BaseModel):
diff --git a/flowsint-api/app/api/schemas/flow.py b/flowsint-api/app/api/schemas/flow.py
index 5912698a..96dcc5c5 100644
--- a/flowsint-api/app/api/schemas/flow.py
+++ b/flowsint-api/app/api/schemas/flow.py
@@ -1,8 +1,9 @@
-from .base import ORMBase
-from pydantic import UUID4, BaseModel
-from typing import Optional
from datetime import datetime
-from typing import List, Optional, Dict, Any
+from typing import Any, Dict, List, Optional
+
+from pydantic import UUID4, BaseModel
+
+from .base import ORMBase
class FlowCreate(BaseModel):
diff --git a/flowsint-api/app/api/schemas/investigation.py b/flowsint-api/app/api/schemas/investigation.py
index 52b0d04d..954e0a87 100644
--- a/flowsint-api/app/api/schemas/investigation.py
+++ b/flowsint-api/app/api/schemas/investigation.py
@@ -1,10 +1,12 @@
-from .base import ORMBase
-from pydantic import UUID4, BaseModel
-from typing import Optional
from datetime import datetime
-from .sketch import SketchRead
+from typing import Optional
+
+from pydantic import UUID4, BaseModel
+
from .analysis import AnalysisRead
+from .base import ORMBase
from .profile import ProfileRead
+from .sketch import SketchRead
class InvestigationCreate(BaseModel):
diff --git a/flowsint-api/app/api/schemas/investigation_profiles.py b/flowsint-api/app/api/schemas/investigation_profiles.py
index fd56ca7e..165cc97a 100644
--- a/flowsint-api/app/api/schemas/investigation_profiles.py
+++ b/flowsint-api/app/api/schemas/investigation_profiles.py
@@ -1,7 +1,9 @@
-from .base import ORMBase
-from pydantic import UUID4, BaseModel
-from typing import Optional
from datetime import datetime
+from typing import Optional
+
+from pydantic import UUID4, BaseModel
+
+from .base import ORMBase
class InvestigationProfileCreate(BaseModel):
diff --git a/flowsint-api/app/api/schemas/profile.py b/flowsint-api/app/api/schemas/profile.py
index 1a286e0c..19be1ada 100644
--- a/flowsint-api/app/api/schemas/profile.py
+++ b/flowsint-api/app/api/schemas/profile.py
@@ -1,7 +1,9 @@
-from .base import ORMBase
-from pydantic import UUID4, BaseModel, ConfigDict, EmailStr
from typing import Optional
+from pydantic import UUID4, BaseModel, ConfigDict, EmailStr
+
+from .base import ORMBase
+
class ProfileCreate(BaseModel):
email: EmailStr
diff --git a/flowsint-api/app/api/schemas/scan.py b/flowsint-api/app/api/schemas/scan.py
index f1e70f9b..6b8df1d9 100644
--- a/flowsint-api/app/api/schemas/scan.py
+++ b/flowsint-api/app/api/schemas/scan.py
@@ -1,7 +1,8 @@
-from typing import List, Optional, Any
-from .base import ORMBase
+from typing import Any, List, Optional
+
from pydantic import UUID4, BaseModel
-from typing import Optional
+
+from .base import ORMBase
class ScanCreate(BaseModel):
diff --git a/flowsint-api/app/api/schemas/sketch.py b/flowsint-api/app/api/schemas/sketch.py
index aeba81fd..7cbc9d27 100644
--- a/flowsint-api/app/api/schemas/sketch.py
+++ b/flowsint-api/app/api/schemas/sketch.py
@@ -1,9 +1,11 @@
-from .base import ORMBase
-from pydantic import UUID4, BaseModel
-from typing import Optional
from datetime import datetime
+from typing import Optional
from uuid import UUID
+from pydantic import UUID4, BaseModel
+
+from .base import ORMBase
+
class SketchCreate(BaseModel):
title: str
diff --git a/flowsint-api/app/api/sketch_utils.py b/flowsint-api/app/api/sketch_utils.py
index 180b9e80..781b7f31 100644
--- a/flowsint-api/app/api/sketch_utils.py
+++ b/flowsint-api/app/api/sketch_utils.py
@@ -1,12 +1,13 @@
"""Utilities for sketch operations, including automatic timestamp updates."""
-from functools import wraps
from datetime import datetime
+from functools import wraps
from typing import Callable
from uuid import UUID
+
from fastapi import BackgroundTasks
+from flowsint_core.core.models import Investigation, Sketch
from sqlalchemy.orm import Session
-from flowsint_core.core.models import Sketch, Investigation
def update_sketch_last_modified(db: Session, sketch_id: str | UUID) -> None:
@@ -31,9 +32,11 @@ def update_sketch_last_modified(db: Session, sketch_id: str | UUID) -> None:
# Update parent investigation timestamp if it exists
if sketch.investigation_id:
- investigation = db.query(Investigation).filter(
- Investigation.id == sketch.investigation_id
- ).first()
+ investigation = (
+ db.query(Investigation)
+ .filter(Investigation.id == sketch.investigation_id)
+ .first()
+ )
if investigation:
investigation.last_updated_at = current_time
@@ -71,6 +74,7 @@ def update_sketch_timestamp(func: Callable) -> Callable:
- Route must have 'background_tasks: BackgroundTasks' parameter
- Route must have 'db: Session' parameter
"""
+
@wraps(func)
async def async_wrapper(*args, **kwargs):
# Extract required dependencies from kwargs
@@ -79,9 +83,13 @@ def update_sketch_timestamp(func: Callable) -> Callable:
db: Session = kwargs.get("db")
if not sketch_id:
- raise ValueError("sketch_id parameter is required for @update_sketch_timestamp")
+ raise ValueError(
+ "sketch_id parameter is required for @update_sketch_timestamp"
+ )
if not background_tasks:
- raise ValueError("background_tasks parameter is required for @update_sketch_timestamp")
+ raise ValueError(
+ "background_tasks parameter is required for @update_sketch_timestamp"
+ )
if not db:
raise ValueError("db parameter is required for @update_sketch_timestamp")
@@ -99,9 +107,13 @@ def update_sketch_timestamp(func: Callable) -> Callable:
db: Session = kwargs.get("db")
if not sketch_id:
- raise ValueError("sketch_id parameter is required for @update_sketch_timestamp")
+ raise ValueError(
+ "sketch_id parameter is required for @update_sketch_timestamp"
+ )
if not background_tasks:
- raise ValueError("background_tasks parameter is required for @update_sketch_timestamp")
+ raise ValueError(
+ "background_tasks parameter is required for @update_sketch_timestamp"
+ )
if not db:
raise ValueError("db parameter is required for @update_sketch_timestamp")
@@ -113,6 +125,7 @@ def update_sketch_timestamp(func: Callable) -> Callable:
# Return the appropriate wrapper based on whether the function is async
import inspect
+
if inspect.iscoroutinefunction(func):
return async_wrapper
else:
diff --git a/flowsint-api/app/main.py b/flowsint-api/app/main.py
index e45875fe..629525b2 100644
--- a/flowsint-api/app/main.py
+++ b/flowsint-api/app/main.py
@@ -4,19 +4,21 @@ from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
# Routes to be included
-from app.api.routes import auth
-from app.api.routes import investigations
-from app.api.routes import sketches
-from app.api.routes import enrichers
-from app.api.routes import flows
-from app.api.routes import events
-from app.api.routes import analysis
-from app.api.routes import chat
-from app.api.routes import scan
-from app.api.routes import keys
-from app.api.routes import types
-from app.api.routes import custom_types
-from app.api.routes import enricher_templates
+from app.api.routes import (
+ analysis,
+ auth,
+ chat,
+ custom_types,
+ enricher_templates,
+ enrichers,
+ events,
+ flows,
+ investigations,
+ keys,
+ scan,
+ sketches,
+ types,
+)
# Comma-separated list of allowed origins, e.g. "https://app.example.com,https://staging.example.com"
# Falls back to localhost dev origin when unset. Never use "*" with allow_credentials=True.
@@ -57,5 +59,11 @@ app.include_router(chat.router, prefix="/api/chats", tags=["chats"])
app.include_router(scan.router, prefix="/api/scans", tags=["scans"])
app.include_router(keys.router, prefix="/api/keys", tags=["keys"])
app.include_router(types.router, prefix="/api/types", tags=["types"])
-app.include_router(custom_types.router, prefix="/api/custom-types", tags=["custom-types"])
-app.include_router(enricher_templates.router, prefix="/api/enrichers/templates", tags=["enricher-templates"])
+app.include_router(
+ custom_types.router, prefix="/api/custom-types", tags=["custom-types"]
+)
+app.include_router(
+ enricher_templates.router,
+ prefix="/api/enrichers/templates",
+ tags=["enricher-templates"],
+)
diff --git a/flowsint-api/app/utils.py b/flowsint-api/app/utils.py
index 46b0a5e1..b4b35a23 100644
--- a/flowsint-api/app/utils.py
+++ b/flowsint-api/app/utils.py
@@ -1,15 +1,13 @@
-from urllib.parse import urlparse
-import phonenumbers
+import inspect
import ipaddress
-from phonenumbers import NumberParseException
-from pydantic import TypeAdapter, BaseModel
-from urllib.parse import urlparse
import re
-import ssl
import socket
-from typing import Dict, Any, List, Type
-import inspect
-from typing import Any, Dict, Type
+import ssl
+from typing import Any, Dict, List, Type
+from urllib.parse import urlparse
+
+import phonenumbers
+from phonenumbers import NumberParseException
from pydantic import BaseModel, TypeAdapter
@@ -48,7 +46,7 @@ def is_valid_domain(url_or_domain: str) -> str:
return False
return True
- except Exception as e:
+ except Exception:
return False
@@ -289,6 +287,7 @@ def get_inline_relationships(nodes: List[Any], edges: List[Any]) -> List[str]:
def to_json_serializable(obj):
"""Convert any object to a JSON-serializable format."""
import json
+
from pydantic import BaseModel
try:
diff --git a/flowsint-api/app/utils/custom_types.py b/flowsint-api/app/utils/custom_types.py
index eac344a8..0a50d5fb 100644
--- a/flowsint-api/app/utils/custom_types.py
+++ b/flowsint-api/app/utils/custom_types.py
@@ -1,11 +1,13 @@
"""Utilities for handling custom types: validation and dynamic model creation."""
+
import hashlib
import json
from typing import Any, Dict, Type
-from pydantic import BaseModel, create_model, EmailStr, Field, field_validator
-from jsonschema import Draft7Validator, ValidationError as JSONSchemaValidationError
-from fastapi import HTTPException
+from fastapi import HTTPException
+from jsonschema import Draft7Validator
+from jsonschema import ValidationError as JSONSchemaValidationError
+from pydantic import BaseModel, EmailStr, Field, create_model
# Mapping of JSON Schema types to Python types
TYPE_MAP = {
@@ -44,10 +46,7 @@ def validate_json_schema(schema: Dict[str, Any]) -> None:
try:
Draft7Validator.check_schema(schema)
except JSONSchemaValidationError as e:
- raise HTTPException(
- status_code=400,
- detail=f"Invalid JSON Schema: {e.message}"
- )
+ raise HTTPException(status_code=400, detail=f"Invalid JSON Schema: {e.message}")
# Additional security checks
_check_schema_security(schema)
@@ -68,7 +67,7 @@ def _check_schema_security(schema: Dict[str, Any]) -> None:
if schema_type and schema_type not in ALLOWED_TYPES:
raise HTTPException(
status_code=400,
- detail=f"Type '{schema_type}' is not allowed. Allowed types: {ALLOWED_TYPES}"
+ detail=f"Type '{schema_type}' is not allowed. Allowed types: {ALLOWED_TYPES}",
)
# Check properties recursively
@@ -79,11 +78,13 @@ def _check_schema_security(schema: Dict[str, Any]) -> None:
if prop_type and prop_type not in ALLOWED_TYPES:
raise HTTPException(
status_code=400,
- detail=f"Type '{prop_type}' in property '{prop_name}' is not allowed"
+ detail=f"Type '{prop_type}' in property '{prop_name}' is not allowed",
)
-def validate_payload_against_schema(payload: Dict[str, Any], schema: Dict[str, Any]) -> tuple[bool, list[str]]:
+def validate_payload_against_schema(
+ payload: Dict[str, Any], schema: Dict[str, Any]
+) -> tuple[bool, list[str]]:
"""
Validate a payload against a JSON Schema.
@@ -121,7 +122,9 @@ def calculate_schema_checksum(schema: Dict[str, Any]) -> str:
return hashlib.sha256(schema_str.encode()).hexdigest()
-def jsonschema_to_pydantic(schema: Dict[str, Any], model_name: str = "DynamicModel") -> Type[BaseModel]:
+def jsonschema_to_pydantic(
+ schema: Dict[str, Any], model_name: str = "DynamicModel"
+) -> Type[BaseModel]:
"""
Convert a JSON Schema to a Pydantic model.
@@ -174,7 +177,7 @@ def jsonschema_to_pydantic(schema: Dict[str, Any], model_name: str = "DynamicMod
except Exception as e:
raise HTTPException(
status_code=400,
- detail=f"Failed to convert JSON Schema to Pydantic model: {str(e)}"
+ detail=f"Failed to convert JSON Schema to Pydantic model: {str(e)}",
)
diff --git a/flowsint-api/tests/test_events_auth.py b/flowsint-api/tests/test_events_auth.py
index 58e8bbc3..b91ed56c 100644
--- a/flowsint-api/tests/test_events_auth.py
+++ b/flowsint-api/tests/test_events_auth.py
@@ -2,8 +2,6 @@
import importlib
-import pytest
-
from flowsint_core.core.auth import create_access_token
from flowsint_core.core.models import Profile
diff --git a/flowsint-core/src/flowsint_core/core/auth.py b/flowsint-core/src/flowsint_core/core/auth.py
index a0885a81..04baf6ec 100644
--- a/flowsint-core/src/flowsint_core/core/auth.py
+++ b/flowsint-core/src/flowsint_core/core/auth.py
@@ -1,8 +1,9 @@
-from passlib.context import CryptContext
-from datetime import datetime, timedelta, timezone
-from jose import jwt
import os
+from datetime import datetime, timedelta, timezone
+
from dotenv import load_dotenv
+from jose import jwt
+from passlib.context import CryptContext
# Load environment variables first
load_dotenv()
diff --git a/flowsint-core/src/flowsint_core/core/celery.py b/flowsint-core/src/flowsint_core/core/celery.py
index bf527108..3da104c4 100644
--- a/flowsint-core/src/flowsint_core/core/celery.py
+++ b/flowsint-core/src/flowsint_core/core/celery.py
@@ -1,4 +1,5 @@
from celery import Celery
+
from .config import settings
celery = Celery(
diff --git a/flowsint-core/src/flowsint_core/core/config.py b/flowsint-core/src/flowsint_core/core/config.py
index cc46c81b..5f2984ce 100644
--- a/flowsint-core/src/flowsint_core/core/config.py
+++ b/flowsint-core/src/flowsint_core/core/config.py
@@ -1,4 +1,5 @@
import os
+
from dotenv import load_dotenv
load_dotenv()
diff --git a/flowsint-core/src/flowsint_core/core/enricher_base.py b/flowsint-core/src/flowsint_core/core/enricher_base.py
index ed34a46c..767c8ea2 100644
--- a/flowsint-core/src/flowsint_core/core/enricher_base.py
+++ b/flowsint-core/src/flowsint_core/core/enricher_base.py
@@ -212,7 +212,6 @@ class Enricher(ABC):
@abstractmethod
def key(cls) -> str:
"""Primary key on which the enricher operates (e.g. domain, IP, etc.)"""
- pass
@classmethod
def documentation(cls) -> str:
diff --git a/flowsint-core/src/flowsint_core/core/events.py b/flowsint-core/src/flowsint_core/core/events.py
index c2b633b6..352861d8 100644
--- a/flowsint-core/src/flowsint_core/core/events.py
+++ b/flowsint-core/src/flowsint_core/core/events.py
@@ -1,11 +1,12 @@
-from typing import Dict, Set
-import uuid
-from fastapi import FastAPI
-import json
import asyncio
+import json
+import os
+import uuid
+from typing import Dict
from uuid import UUID
+
import redis.asyncio as redis
-import os
+from fastapi import FastAPI
class EventEmitter:
diff --git a/flowsint-core/src/flowsint_core/core/graph/serializer.py b/flowsint-core/src/flowsint_core/core/graph/serializer.py
index 2dbad02e..8070f46a 100644
--- a/flowsint-core/src/flowsint_core/core/graph/serializer.py
+++ b/flowsint-core/src/flowsint_core/core/graph/serializer.py
@@ -7,11 +7,10 @@ into Neo4j-compatible primitive types, following the Single Responsibility Princ
from typing import Any, Callable, Dict, List, Optional, Type
+from flowsint_core.utils import flatten, unflatten
from flowsint_types import FlowsintType
from pydantic import BaseModel, ValidationError
-from flowsint_core.utils import flatten, unflatten
-
from .types import GraphEdge, GraphNode, NodeMetadata
# Callable that resolves a type name to a FlowsintType subclass (or None).
@@ -39,9 +38,11 @@ class GraphSerializer:
cleaned[key] = cleaned_nested
elif isinstance(value, list):
cleaned_list = [
- GraphSerializer._clean_empty_values(item)
- if isinstance(item, dict)
- else item
+ (
+ GraphSerializer._clean_empty_values(item)
+ if isinstance(item, dict)
+ else item
+ )
for item in value
if item != "" and item is not None
]
diff --git a/flowsint-core/src/flowsint_core/core/llm/__init__.py b/flowsint-core/src/flowsint_core/core/llm/__init__.py
index 1be6bc9a..b96f98eb 100644
--- a/flowsint-core/src/flowsint_core/core/llm/__init__.py
+++ b/flowsint-core/src/flowsint_core/core/llm/__init__.py
@@ -1,5 +1,5 @@
-from .types import ChatMessage, MessageRole
-from .protocol import LLMProvider
from .factory import create_llm_provider
+from .protocol import LLMProvider
+from .types import ChatMessage, MessageRole
__all__ = ["ChatMessage", "MessageRole", "LLMProvider", "create_llm_provider"]
diff --git a/flowsint-core/src/flowsint_core/core/llm/factory.py b/flowsint-core/src/flowsint_core/core/llm/factory.py
index 721d84f6..ef00fbfe 100644
--- a/flowsint-core/src/flowsint_core/core/llm/factory.py
+++ b/flowsint-core/src/flowsint_core/core/llm/factory.py
@@ -3,7 +3,6 @@ from typing import Optional
from .protocol import LLMProvider
-
_SUPPORTED_PROVIDERS = ("mistral", "openai")
_DEFAULT_API_KEY_ENV = {
diff --git a/flowsint-core/src/flowsint_core/core/llm/providers/mistral.py b/flowsint-core/src/flowsint_core/core/llm/providers/mistral.py
index 7829fc4b..66ff2326 100644
--- a/flowsint-core/src/flowsint_core/core/llm/providers/mistral.py
+++ b/flowsint-core/src/flowsint_core/core/llm/providers/mistral.py
@@ -2,7 +2,6 @@ from typing import AsyncIterator, List
from ..types import ChatMessage, MessageRole
-
_ROLE_MAP = {
MessageRole.SYSTEM: "SystemMessage",
MessageRole.USER: "UserMessage",
@@ -18,7 +17,7 @@ class MistralProvider:
self._model = model
def _build_messages(self, messages: List[ChatMessage]):
- from mistralai.models import UserMessage, AssistantMessage, SystemMessage
+ from mistralai.models import AssistantMessage, SystemMessage, UserMessage
type_map = {
MessageRole.SYSTEM: SystemMessage,
diff --git a/flowsint-core/src/flowsint_core/core/logger_old.py b/flowsint-core/src/flowsint_core/core/logger_old.py
index eb6905b1..e103d1e9 100644
--- a/flowsint-core/src/flowsint_core/core/logger_old.py
+++ b/flowsint-core/src/flowsint_core/core/logger_old.py
@@ -1,10 +1,11 @@
+import logging
+from typing import Dict, Union
from uuid import UUID
-from typing import Any, Dict, Union
-from .models import Log
+
from ..tasks.event import emit_event_task
-from .postgre_db import get_db
from .enums import EventLevel
-import logging
+from .models import Log
+from .postgre_db import get_db
logger = logging.getLogger(__name__)
diff --git a/flowsint-core/src/flowsint_core/core/logger_protocols.py b/flowsint-core/src/flowsint_core/core/logger_protocols.py
index f6d71c48..1e21f171 100644
--- a/flowsint-core/src/flowsint_core/core/logger_protocols.py
+++ b/flowsint-core/src/flowsint_core/core/logger_protocols.py
@@ -2,10 +2,11 @@
Protocols and interfaces for the Logger system.
Following SOLID principles with Dependency Inversion.
"""
-from typing import Protocol, Any, Dict
-from uuid import UUID
-from .models import Log
+
+from typing import Any, Dict, Protocol
+
from .enums import EventLevel
+from .models import Log
class DatabaseSession(Protocol):
@@ -35,7 +36,9 @@ class DatabaseSession(Protocol):
class EventEmitter(Protocol):
"""Protocol for event emission (Redis pub/sub)."""
- def emit(self, log_id: str, sketch_id: str, level: EventLevel, content: Dict) -> None:
+ def emit(
+ self, log_id: str, sketch_id: str, level: EventLevel, content: Dict
+ ) -> None:
"""Emit a log event to Redis for real-time display."""
...
@@ -47,6 +50,8 @@ class LogStorage(Protocol):
"""Store a single log entry."""
...
- def store_logs_batch(self, logs_data: list[tuple[str, EventLevel, Dict]]) -> list[Log]:
+ def store_logs_batch(
+ self, logs_data: list[tuple[str, EventLevel, Dict]]
+ ) -> list[Log]:
"""Store multiple log entries in a batch."""
...
diff --git a/flowsint-core/src/flowsint_core/core/models.py b/flowsint-core/src/flowsint_core/core/models.py
index daf7d269..86ec51d5 100644
--- a/flowsint-core/src/flowsint_core/core/models.py
+++ b/flowsint-core/src/flowsint_core/core/models.py
@@ -2,11 +2,16 @@ import json
import uuid
from datetime import datetime, timezone
+from flowsint_core.core.enums import EventLevel
+from flowsint_core.core.types import Role
from sqlalchemy import (
JSON,
Boolean,
Column,
DateTime,
+)
+from sqlalchemy import Enum as SQLEnum
+from sqlalchemy import (
Float,
ForeignKey,
Index,
@@ -17,15 +22,9 @@ from sqlalchemy import (
Uuid,
func,
)
-from sqlalchemy import (
- Enum as SQLEnum,
-)
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
from sqlalchemy.types import TypeDecorator
-from flowsint_core.core.enums import EventLevel
-from flowsint_core.core.types import Role
-
class RoleListType(TypeDecorator):
"""Stores a list of Role enums as a JSON string. Portable across dialects."""
diff --git a/flowsint-core/src/flowsint_core/core/orchestrator.py b/flowsint-core/src/flowsint_core/core/orchestrator.py
index 8c55fcc9..d3c171e9 100644
--- a/flowsint-core/src/flowsint_core/core/orchestrator.py
+++ b/flowsint-core/src/flowsint_core/core/orchestrator.py
@@ -1,15 +1,17 @@
-from typing import List, Dict, Any
-from datetime import datetime
+import asyncio
+import json
+import os
import time
+from datetime import datetime
+from typing import Any, Dict, List
+
+from flowsint_enrichers import ENRICHER_REGISTRY
from pydantic import ValidationError
+
+from ..utils import to_json_serializable
from .enricher_base import Enricher
-from flowsint_enrichers import ENRICHER_REGISTRY
-from .types import FlowBranch, FlowStep
from .logger import Logger
-from ..utils import to_json_serializable
-import asyncio
-import json
-import os
+from .types import FlowBranch, FlowStep
class FlowOrchestrator(Enricher):
@@ -356,7 +358,7 @@ class FlowOrchestrator(Enricher):
# Cache for enricher results to avoid recomputation
enricher_results_cache = {}
- total_steps = sum(len(branch.steps) for branch in self.enricher_branches)
+ sum(len(branch.steps) for branch in self.enricher_branches)
completed_steps = 0
# Process each branch
diff --git a/flowsint-core/src/flowsint_core/core/postgre_db.py b/flowsint-core/src/flowsint_core/core/postgre_db.py
index 516dea1a..d1a8cc0e 100644
--- a/flowsint-core/src/flowsint_core/core/postgre_db.py
+++ b/flowsint-core/src/flowsint_core/core/postgre_db.py
@@ -1,7 +1,8 @@
-from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker, declarative_base, Session
import os
+
from dotenv import load_dotenv
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session, declarative_base, sessionmaker
load_dotenv()
diff --git a/flowsint-core/src/flowsint_core/core/repositories/__init__.py b/flowsint-core/src/flowsint_core/core/repositories/__init__.py
index 4fbcd2a0..1bd3de20 100644
--- a/flowsint-core/src/flowsint_core/core/repositories/__init__.py
+++ b/flowsint-core/src/flowsint_core/core/repositories/__init__.py
@@ -1,16 +1,17 @@
"""Repository layer for database operations."""
-from .base import BaseRepository
-from .profile_repository import ProfileRepository
-from .investigation_repository import InvestigationRepository
-from .sketch_repository import SketchRepository
+
from .analysis_repository import AnalysisRepository
+from .base import BaseRepository
from .chat_repository import ChatRepository
-from .scan_repository import ScanRepository
-from .log_repository import LogRepository
-from .key_repository import KeyRepository
-from .flow_repository import FlowRepository
from .custom_type_repository import CustomTypeRepository
from .enricher_template_repository import EnricherTemplateRepository
+from .flow_repository import FlowRepository
+from .investigation_repository import InvestigationRepository
+from .key_repository import KeyRepository
+from .log_repository import LogRepository
+from .profile_repository import ProfileRepository
+from .scan_repository import ScanRepository
+from .sketch_repository import SketchRepository
__all__ = [
"BaseRepository",
diff --git a/flowsint-core/src/flowsint_core/core/repositories/analysis_repository.py b/flowsint-core/src/flowsint_core/core/repositories/analysis_repository.py
index d8b27365..3ddfcd19 100644
--- a/flowsint-core/src/flowsint_core/core/repositories/analysis_repository.py
+++ b/flowsint-core/src/flowsint_core/core/repositories/analysis_repository.py
@@ -1,4 +1,5 @@
"""Repository for Analysis model."""
+
from typing import List, Optional
from uuid import UUID
diff --git a/flowsint-core/src/flowsint_core/core/repositories/base.py b/flowsint-core/src/flowsint_core/core/repositories/base.py
index e35183f6..ae01d7d1 100644
--- a/flowsint-core/src/flowsint_core/core/repositories/base.py
+++ b/flowsint-core/src/flowsint_core/core/repositories/base.py
@@ -1,4 +1,5 @@
"""Base repository providing common database operations."""
+
from typing import Generic, List, Optional, Set, Type, TypeVar
from uuid import UUID
diff --git a/flowsint-core/src/flowsint_core/core/repositories/chat_repository.py b/flowsint-core/src/flowsint_core/core/repositories/chat_repository.py
index 84a5755f..2d42769f 100644
--- a/flowsint-core/src/flowsint_core/core/repositories/chat_repository.py
+++ b/flowsint-core/src/flowsint_core/core/repositories/chat_repository.py
@@ -1,4 +1,5 @@
"""Repository for Chat and ChatMessage models."""
+
from typing import List, Optional
from uuid import UUID
@@ -25,9 +26,7 @@ class ChatRepository(BaseRepository[Chat]):
.all()
)
- def get_by_id_and_owner(
- self, chat_id: UUID, owner_id: UUID
- ) -> Optional[Chat]:
+ def get_by_id_and_owner(self, chat_id: UUID, owner_id: UUID) -> Optional[Chat]:
return (
self._db.query(Chat)
.filter(Chat.id == chat_id, Chat.owner_id == owner_id)
diff --git a/flowsint-core/src/flowsint_core/core/repositories/custom_type_repository.py b/flowsint-core/src/flowsint_core/core/repositories/custom_type_repository.py
index 436d1d81..9615b965 100644
--- a/flowsint-core/src/flowsint_core/core/repositories/custom_type_repository.py
+++ b/flowsint-core/src/flowsint_core/core/repositories/custom_type_repository.py
@@ -1,4 +1,5 @@
"""Repository for CustomType model."""
+
from typing import List, Optional
from uuid import UUID
@@ -31,9 +32,7 @@ class CustomTypeRepository(BaseRepository[CustomType]):
.first()
)
- def get_by_name_and_owner(
- self, name: str, owner_id: UUID
- ) -> Optional[CustomType]:
+ def get_by_name_and_owner(self, name: str, owner_id: UUID) -> Optional[CustomType]:
return (
self._db.query(CustomType)
.filter(CustomType.owner_id == owner_id, CustomType.name == name)
diff --git a/flowsint-core/src/flowsint_core/core/repositories/enricher_template_repository.py b/flowsint-core/src/flowsint_core/core/repositories/enricher_template_repository.py
index 46d279ac..2cdb48e1 100644
--- a/flowsint-core/src/flowsint_core/core/repositories/enricher_template_repository.py
+++ b/flowsint-core/src/flowsint_core/core/repositories/enricher_template_repository.py
@@ -1,4 +1,5 @@
"""Repository for EnricherTemplate model."""
+
from typing import List, Optional
from uuid import UUID
diff --git a/flowsint-core/src/flowsint_core/core/repositories/flow_repository.py b/flowsint-core/src/flowsint_core/core/repositories/flow_repository.py
index ba63e80c..083c9866 100644
--- a/flowsint-core/src/flowsint_core/core/repositories/flow_repository.py
+++ b/flowsint-core/src/flowsint_core/core/repositories/flow_repository.py
@@ -1,6 +1,6 @@
"""Repository for Flow model."""
+
from typing import List, Optional
-from uuid import UUID
from ..models import Flow
from .base import BaseRepository
diff --git a/flowsint-core/src/flowsint_core/core/repositories/investigation_repository.py b/flowsint-core/src/flowsint_core/core/repositories/investigation_repository.py
index a9ab2c85..bcfce8ee 100644
--- a/flowsint-core/src/flowsint_core/core/repositories/investigation_repository.py
+++ b/flowsint-core/src/flowsint_core/core/repositories/investigation_repository.py
@@ -1,4 +1,5 @@
"""Repository for Investigation and InvestigationUserRole models."""
+
from typing import List, Optional
from uuid import UUID
@@ -31,9 +32,7 @@ class InvestigationRepository(BaseRepository[Investigation]):
.all()
)
- def get_with_relations(
- self, investigation_id: UUID
- ) -> Optional[Investigation]:
+ def get_with_relations(self, investigation_id: UUID) -> Optional[Investigation]:
return (
self._db.query(Investigation)
.options(
@@ -70,9 +69,7 @@ class InvestigationRepository(BaseRepository[Investigation]):
self._db.add(role_entry)
return role_entry
- def get_collaborators(
- self, investigation_id: UUID
- ) -> List[InvestigationUserRole]:
+ def get_collaborators(self, investigation_id: UUID) -> List[InvestigationUserRole]:
return (
self._db.query(InvestigationUserRole)
.options(selectinload(InvestigationUserRole.user))
@@ -88,9 +85,7 @@ class InvestigationRepository(BaseRepository[Investigation]):
entry.roles = roles
return entry
- def remove_user_role(
- self, user_id: UUID, investigation_id: UUID
- ) -> bool:
+ def remove_user_role(self, user_id: UUID, investigation_id: UUID) -> bool:
entry = self.get_user_role(user_id, investigation_id)
if entry:
self._db.delete(entry)
diff --git a/flowsint-core/src/flowsint_core/core/repositories/log_repository.py b/flowsint-core/src/flowsint_core/core/repositories/log_repository.py
index 8eab27e8..90167151 100644
--- a/flowsint-core/src/flowsint_core/core/repositories/log_repository.py
+++ b/flowsint-core/src/flowsint_core/core/repositories/log_repository.py
@@ -1,4 +1,5 @@
"""Repository for Log model."""
+
from datetime import datetime, timedelta, timezone
from typing import List, Optional
from uuid import UUID
diff --git a/flowsint-core/src/flowsint_core/core/repositories/profile_repository.py b/flowsint-core/src/flowsint_core/core/repositories/profile_repository.py
index e484c0e7..ee5f7a29 100644
--- a/flowsint-core/src/flowsint_core/core/repositories/profile_repository.py
+++ b/flowsint-core/src/flowsint_core/core/repositories/profile_repository.py
@@ -1,4 +1,5 @@
"""Repository for Profile model."""
+
from typing import Optional
from ..models import Profile
diff --git a/flowsint-core/src/flowsint_core/core/repositories/sketch_repository.py b/flowsint-core/src/flowsint_core/core/repositories/sketch_repository.py
index d666bda9..07694eeb 100644
--- a/flowsint-core/src/flowsint_core/core/repositories/sketch_repository.py
+++ b/flowsint-core/src/flowsint_core/core/repositories/sketch_repository.py
@@ -1,4 +1,5 @@
"""Repository for Sketch model."""
+
from typing import List
from uuid import UUID
diff --git a/flowsint-core/src/flowsint_core/core/services/analysis_service.py b/flowsint-core/src/flowsint_core/core/services/analysis_service.py
index b5842012..3b8994fb 100644
--- a/flowsint-core/src/flowsint_core/core/services/analysis_service.py
+++ b/flowsint-core/src/flowsint_core/core/services/analysis_service.py
@@ -11,7 +11,7 @@ from sqlalchemy.orm import Session
from ..models import Analysis
from ..repositories import AnalysisRepository, InvestigationRepository
from .base import BaseService
-from .exceptions import NotFoundError, PermissionDeniedError
+from .exceptions import NotFoundError
class AnalysisService(BaseService):
diff --git a/flowsint-core/src/flowsint_core/core/services/auth_service.py b/flowsint-core/src/flowsint_core/core/services/auth_service.py
index 5b4a7af7..262cce6b 100644
--- a/flowsint-core/src/flowsint_core/core/services/auth_service.py
+++ b/flowsint-core/src/flowsint_core/core/services/auth_service.py
@@ -2,15 +2,16 @@
Authentication service for user login and registration.
"""
-from typing import Dict, Any
-from sqlalchemy.orm import Session
+from typing import Any, Dict
+
from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session
+from ..auth import create_access_token, get_password_hash, verify_password
from ..models import Profile
-from ..auth import verify_password, create_access_token, get_password_hash
from ..repositories import ProfileRepository
from .base import BaseService
-from .exceptions import AuthenticationError, ConflictError, DatabaseError
+from .exceptions import AuthenticationError, ConflictError
class AuthService(BaseService):
diff --git a/flowsint-core/src/flowsint_core/core/services/base.py b/flowsint-core/src/flowsint_core/core/services/base.py
index 03e585f7..61466c95 100644
--- a/flowsint-core/src/flowsint_core/core/services/base.py
+++ b/flowsint-core/src/flowsint_core/core/services/base.py
@@ -5,11 +5,11 @@ Base service class providing common functionality for all services.
from typing import List
from uuid import UUID
-from sqlalchemy.orm import Session
from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.orm import Session
-from .exceptions import DatabaseError, PermissionDeniedError
from ..types import Role
+from .exceptions import DatabaseError, PermissionDeniedError
class BaseService:
@@ -54,7 +54,12 @@ class BaseService:
for action in actions:
if role == Role.OWNER:
return True
- if role == Role.ADMIN and action in ["read", "create", "update", "manage"]:
+ if role == Role.ADMIN and action in [
+ "read",
+ "create",
+ "update",
+ "manage",
+ ]:
return True
if role == Role.EDITOR and action in ["read", "create", "update"]:
return True
diff --git a/flowsint-core/src/flowsint_core/core/services/chat_service.py b/flowsint-core/src/flowsint_core/core/services/chat_service.py
index 8364acc7..c15fb2db 100644
--- a/flowsint-core/src/flowsint_core/core/services/chat_service.py
+++ b/flowsint-core/src/flowsint_core/core/services/chat_service.py
@@ -15,7 +15,7 @@ from ..llm import LLMProvider, MessageRole, create_llm_provider
from ..models import Chat, ChatMessage
from ..repositories import ChatRepository
from .base import BaseService
-from .exceptions import DatabaseError, NotFoundError, PermissionDeniedError
+from .exceptions import NotFoundError
DEFAULT_SYSTEM_PROMPT = (
"You are a CTI/OSINT investigator and you are trying to investigate on a "
@@ -26,7 +26,6 @@ DEFAULT_SYSTEM_PROMPT = (
)
-
class ChatService(BaseService):
"""
Service for chat CRUD operations and AI message streaming.
diff --git a/flowsint-core/src/flowsint_core/core/services/enricher_template_service.py b/flowsint-core/src/flowsint_core/core/services/enricher_template_service.py
index a6b19292..ae5c9e15 100644
--- a/flowsint-core/src/flowsint_core/core/services/enricher_template_service.py
+++ b/flowsint-core/src/flowsint_core/core/services/enricher_template_service.py
@@ -67,9 +67,7 @@ class EnricherTemplateService(BaseService):
raise NotFoundError("Template not found")
return template
- def get_owned_template(
- self, template_id: UUID, owner_id: UUID
- ) -> EnricherTemplate:
+ def get_owned_template(self, template_id: UUID, owner_id: UUID) -> EnricherTemplate:
template = self._repo.get_by_id_and_owner(template_id, owner_id)
if not template:
raise NotFoundError("Template not found")
diff --git a/flowsint-core/src/flowsint_core/core/services/flow_service.py b/flowsint-core/src/flowsint_core/core/services/flow_service.py
index 11048957..8fea5771 100644
--- a/flowsint-core/src/flowsint_core/core/services/flow_service.py
+++ b/flowsint-core/src/flowsint_core/core/services/flow_service.py
@@ -16,7 +16,7 @@ from ..repositories import (
SketchRepository,
)
from .base import BaseService
-from .exceptions import NotFoundError, PermissionDeniedError
+from .exceptions import NotFoundError
class FlowService(BaseService):
diff --git a/flowsint-core/src/flowsint_core/core/services/investigation_service.py b/flowsint-core/src/flowsint_core/core/services/investigation_service.py
index d460e81a..cde3b46c 100644
--- a/flowsint-core/src/flowsint_core/core/services/investigation_service.py
+++ b/flowsint-core/src/flowsint_core/core/services/investigation_service.py
@@ -2,27 +2,27 @@
Investigation service for managing investigations and user roles.
"""
+from datetime import datetime, timezone
from typing import List, Optional
from uuid import UUID, uuid4
-from datetime import datetime, timezone
from sqlalchemy.orm import Session
-from ..models import Investigation, InvestigationUserRole, Sketch, Analysis
-from ..types import Role
from ..graph import create_graph_service
+from ..models import Investigation, InvestigationUserRole, Sketch
from ..repositories import (
- InvestigationRepository,
- SketchRepository,
AnalysisRepository,
+ InvestigationRepository,
ProfileRepository,
+ SketchRepository,
)
+from ..types import Role
from .base import BaseService
from .exceptions import (
- NotFoundError,
- PermissionDeniedError,
ConflictError,
DatabaseError,
+ NotFoundError,
+ PermissionDeniedError,
)
diff --git a/flowsint-core/src/flowsint_core/core/services/log_service.py b/flowsint-core/src/flowsint_core/core/services/log_service.py
index 3ee73d95..cbcf19fe 100644
--- a/flowsint-core/src/flowsint_core/core/services/log_service.py
+++ b/flowsint-core/src/flowsint_core/core/services/log_service.py
@@ -2,18 +2,22 @@
Log service for managing event logs.
"""
+from datetime import datetime
from typing import List, Optional
from uuid import UUID
-from datetime import datetime, timedelta
from sqlalchemy.orm import Session
from ..models import Scan
+from ..repositories import (
+ InvestigationRepository,
+ LogRepository,
+ ScanRepository,
+ SketchRepository,
+)
from ..types import Event
-from ..enums import EventLevel
-from ..repositories import LogRepository, SketchRepository, ScanRepository, InvestigationRepository
from .base import BaseService
-from .exceptions import NotFoundError, PermissionDeniedError, DatabaseError
+from .exceptions import DatabaseError, NotFoundError
class LogService(BaseService):
diff --git a/flowsint-core/src/flowsint_core/core/services/scan_service.py b/flowsint-core/src/flowsint_core/core/services/scan_service.py
index 0353b271..48b87e87 100644
--- a/flowsint-core/src/flowsint_core/core/services/scan_service.py
+++ b/flowsint-core/src/flowsint_core/core/services/scan_service.py
@@ -10,7 +10,7 @@ from sqlalchemy.orm import Session
from ..models import Scan
from ..repositories import InvestigationRepository, ScanRepository, SketchRepository
from .base import BaseService
-from .exceptions import NotFoundError, PermissionDeniedError
+from .exceptions import NotFoundError
class ScanService(BaseService):
diff --git a/flowsint-core/src/flowsint_core/core/services/sketch_service.py b/flowsint-core/src/flowsint_core/core/services/sketch_service.py
index c396b4eb..5b981a3b 100644
--- a/flowsint-core/src/flowsint_core/core/services/sketch_service.py
+++ b/flowsint-core/src/flowsint_core/core/services/sketch_service.py
@@ -10,14 +10,12 @@ from uuid import UUID
from sqlalchemy.orm import Session
from ..graph import GraphNode, create_graph_service
-from ..graph.types import GraphData
from ..models import Sketch
from ..repositories import InvestigationRepository, SketchRepository
from .base import BaseService
from .exceptions import (
DatabaseError,
NotFoundError,
- PermissionDeniedError,
ValidationError,
)
@@ -115,7 +113,7 @@ class SketchService(BaseService):
def get_graph(
self, sketch_id: UUID, user_id: UUID, format: Optional[str] = None
) -> Dict[str, Any]:
- sketch = self._get_sketch_with_permission(sketch_id, user_id, ["read"])
+ self._get_sketch_with_permission(sketch_id, user_id, ["read"])
resolver = (
self._type_registry.build_type_resolver(user_id)
diff --git a/flowsint-core/src/flowsint_core/core/services/template_generator_service.py b/flowsint-core/src/flowsint_core/core/services/template_generator_service.py
index 0ed03e56..cbac1c85 100644
--- a/flowsint-core/src/flowsint_core/core/services/template_generator_service.py
+++ b/flowsint-core/src/flowsint_core/core/services/template_generator_service.py
@@ -11,14 +11,13 @@ from typing import Any, Dict, Optional
from uuid import UUID
import yaml
+from flowsint_core.templates.types import Template
from sqlalchemy.orm import Session
from ..llm import ChatMessage, MessageRole, create_llm_provider
from .base import BaseService
from .exceptions import ValidationError
from .vault_service import VaultService
-from flowsint_core.templates.types import Template
-
_SYSTEM_PROMPT = """\
You are a YAML template generator for Flowsint enrichers. Given a user's description, \
diff --git a/flowsint-core/src/flowsint_core/core/services/type_registry_service.py b/flowsint-core/src/flowsint_core/core/services/type_registry_service.py
index 417ad7ab..a64b32a9 100644
--- a/flowsint-core/src/flowsint_core/core/services/type_registry_service.py
+++ b/flowsint-core/src/flowsint_core/core/services/type_registry_service.py
@@ -195,9 +195,7 @@ class TypeRegistryService(BaseService):
label_key = (
required[0]
if required
- else list(properties.keys())[0]
- if properties
- else "value"
+ else list(properties.keys())[0] if properties else "value"
)
custom_type_obj = {
diff --git a/flowsint-core/src/flowsint_core/core/template_enricher.py b/flowsint-core/src/flowsint_core/core/template_enricher.py
index 9c84aa9e..0f9207e0 100644
--- a/flowsint-core/src/flowsint_core/core/template_enricher.py
+++ b/flowsint-core/src/flowsint_core/core/template_enricher.py
@@ -48,8 +48,6 @@ import xml.etree.ElementTree as ET
from typing import Any, Dict, List, Optional
import httpx
-from flowsint_types import FlowsintType, get_type
-
from flowsint_core.core.enricher_base import Enricher
from flowsint_core.core.logger import Logger
from flowsint_core.templates.loader.yaml_loader import (
@@ -59,13 +57,12 @@ from flowsint_core.templates.loader.yaml_loader import (
validate_url_safe,
)
from flowsint_core.templates.types import Template, TemplateRetryConfig
+from flowsint_types import FlowsintType, get_type
class TemplateEnricherError(Exception):
"""Base exception for template enricher errors."""
- pass
-
class TemplateEnricher(Enricher):
"""
diff --git a/flowsint-core/src/flowsint_core/core/vault.py b/flowsint-core/src/flowsint_core/core/vault.py
index c48e9596..8d5bf21d 100644
--- a/flowsint-core/src/flowsint_core/core/vault.py
+++ b/flowsint-core/src/flowsint_core/core/vault.py
@@ -1,16 +1,17 @@
+import base64
import os
-from typing import Protocol, Optional
import uuid
-from sqlalchemy.orm import Session
-from sqlalchemy import select
-from .models import Key
-from datetime import datetime, UTC
-import base64
-from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+from datetime import UTC, datetime
+from typing import Optional, Protocol
+
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
-
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
from dotenv import load_dotenv
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from .models import Key
load_dotenv()
diff --git a/flowsint-core/src/flowsint_core/imports/json/parse_json.py b/flowsint-core/src/flowsint_core/imports/json/parse_json.py
index cecf62f5..4ce342e7 100644
--- a/flowsint-core/src/flowsint_core/imports/json/parse_json.py
+++ b/flowsint-core/src/flowsint_core/imports/json/parse_json.py
@@ -1,9 +1,8 @@
import json
from typing import Dict, List, Optional
-from flowsint_types import Individual
-
from flowsint_core.core.graph.serializer import TypeResolver
+from flowsint_types import Individual
from ..types import Edge, Entity, EntityPreview, FileParseResult
from ..utils import create_entity_preview
@@ -117,7 +116,10 @@ def _parse_node(
return create_entity_preview(label)
# Try to resolve the type via the registry service
if not type_resolver:
- from flowsint_core.core.services.type_registry_service import local_type_resolver
+ from flowsint_core.core.services.type_registry_service import (
+ local_type_resolver,
+ )
+
type_resolver = local_type_resolver
DetectedType = type_resolver(type.lower())
if not DetectedType:
diff --git a/flowsint-core/src/flowsint_core/tasks/enricher.py b/flowsint-core/src/flowsint_core/tasks/enricher.py
index 1238f057..4e9d79e9 100644
--- a/flowsint-core/src/flowsint_core/tasks/enricher.py
+++ b/flowsint-core/src/flowsint_core/tasks/enricher.py
@@ -3,11 +3,10 @@ import uuid
from typing import List, Optional
from celery import states
+from flowsint_core.utils import to_json_serializable
from flowsint_enrichers import ENRICHER_REGISTRY, load_all_enrichers
from sqlalchemy.orm import Session
-from flowsint_core.utils import to_json_serializable
-
from ..core.celery import celery
from ..core.enums import EventLevel
from ..core.logger import Logger
@@ -119,15 +118,11 @@ def run_template_enricher(
try:
vault = create_vault_service(session).for_user(uuid.UUID(owner_id))
except Exception as e:
- Logger.error(
- sketch_id, {"message": f"Failed to create vault: {str(e)}"}
- )
+ Logger.error(sketch_id, {"message": f"Failed to create vault: {str(e)}"})
# Load template from database
template_service = create_enricher_template_service(session)
- db_template = template_service.find_by_name(
- template_name, uuid.UUID(owner_id)
- )
+ db_template = template_service.find_by_name(template_name, uuid.UUID(owner_id))
if not db_template:
raise ValueError(
f"Template '{template_name}' not found for user {owner_id}"
@@ -155,11 +150,7 @@ def run_template_enricher(
error_logs = f"An error occurred: {str(ex)}"
print(f"Error in template task: {error_logs}")
- scan = (
- session.query(Scan)
- .filter(Scan.id == uuid.UUID(self.request.id))
- .first()
- )
+ scan = session.query(Scan).filter(Scan.id == uuid.UUID(self.request.id)).first()
if scan:
scan.status = EventLevel.FAILED
scan.error = error_logs
diff --git a/flowsint-core/src/flowsint_core/tasks/event.py b/flowsint-core/src/flowsint_core/tasks/event.py
index b79b0fe1..2399496c 100644
--- a/flowsint-core/src/flowsint_core/tasks/event.py
+++ b/flowsint-core/src/flowsint_core/tasks/event.py
@@ -21,7 +21,7 @@ def emit_event_task(log_id: str, sketch_id: str, log_type: EventLevel, content:
).model_dump_json()
redis_client = redis.from_url(os.environ["REDIS_URL"])
redis_client.publish(sketch_id, event)
- except Exception as e:
+ except Exception:
raise
@@ -37,5 +37,5 @@ def emit_status_event_task(
redis_client = redis.from_url(os.environ["REDIS_URL"])
# Publish to status channel
redis_client.publish(f"{sketch_id}_status", event)
- except Exception as e:
+ except Exception:
raise
diff --git a/flowsint-core/src/flowsint_core/templates/loader/yaml_loader.py b/flowsint-core/src/flowsint_core/templates/loader/yaml_loader.py
index 69462c92..d206fa22 100644
--- a/flowsint-core/src/flowsint_core/templates/loader/yaml_loader.py
+++ b/flowsint-core/src/flowsint_core/templates/loader/yaml_loader.py
@@ -4,7 +4,6 @@ from typing import Any, Optional, Set
from urllib.parse import urlparse
import yaml
-
from flowsint_core.core.graph.serializer import TypeResolver
from flowsint_core.templates.types import Template
@@ -44,14 +43,10 @@ BLOCKED_HOSTNAMES: Set[str] = {
class SSRFError(Exception):
"""Raised when a URL is blocked due to SSRF protection."""
- pass
-
class TemplateRenderError(Exception):
"""Raised when template rendering fails."""
- pass
-
def is_ip_blocked(ip_str: str) -> bool:
"""Check if an IP address is in a blocked range."""
@@ -134,7 +129,10 @@ class YamlLoader:
raise ValueError("Missing 'input.type' property in the yaml.")
if not type_resolver:
- from flowsint_core.core.services.type_registry_service import local_type_resolver
+ from flowsint_core.core.services.type_registry_service import (
+ local_type_resolver,
+ )
+
type_resolver = local_type_resolver
DetectedType = type_resolver(input_type)
@@ -221,9 +219,11 @@ class YamlLoader:
result[key] = YamlLoader.render_dict(value, values, sanitize)
elif isinstance(value, list):
result[key] = [
- YamlLoader.render_template(item, values, sanitize)
- if isinstance(item, str)
- else item
+ (
+ YamlLoader.render_template(item, values, sanitize)
+ if isinstance(item, str)
+ else item
+ )
for item in value
]
else:
diff --git a/flowsint-core/tests/conftest.py b/flowsint-core/tests/conftest.py
index 5cd0cfc6..66cb555e 100644
--- a/flowsint-core/tests/conftest.py
+++ b/flowsint-core/tests/conftest.py
@@ -1,15 +1,12 @@
"""Pytest configuration for flowsint-core tests."""
-import os
from unittest.mock import MagicMock
import pytest
-
+from flowsint_core.core.models import Base
from sqlalchemy import create_engine, event
from sqlalchemy.orm import sessionmaker
-from flowsint_core.core.models import Base
-
@pytest.fixture(autouse=True)
def setup_test_environment(monkeypatch):
diff --git a/flowsint-core/tests/core/graph/in_memory_graph_repository.py b/flowsint-core/tests/core/graph/in_memory_graph_repository.py
index 84d4cd1a..55b9382d 100644
--- a/flowsint-core/tests/core/graph/in_memory_graph_repository.py
+++ b/flowsint-core/tests/core/graph/in_memory_graph_repository.py
@@ -84,8 +84,14 @@ class InMemoryGraphRepository:
# Also soft delete related edges
for edge in self._edges.values():
- if edge.get("source") == node_id or edge.get("target") == node_id:
- if edge.get("sketch_id") == sketch_id and edge.get("deleted_at") is None:
+ if (
+ edge.get("source") == node_id
+ or edge.get("target") == node_id
+ ):
+ if (
+ edge.get("sketch_id") == sketch_id
+ and edge.get("deleted_at") is None
+ ):
edge["deleted_at"] = deleted_at
self._nodes[node_id]["deleted_at"] = deleted_at
@@ -116,7 +122,10 @@ class InMemoryGraphRepository:
for node_id in node_ids:
if node_id in self._nodes:
node = self._nodes[node_id]
- if node.get("sketch_id") == sketch_id and node.get("deleted_at") is None:
+ if (
+ node.get("sketch_id") == sketch_id
+ and node.get("deleted_at") is None
+ ):
result.append({"data": node})
return result
@@ -142,9 +151,9 @@ class InMemoryGraphRepository:
def create_relationship(self, rel_obj: Dict[str, Any], sketch_id: str) -> None:
"""Create a relationship between two nodes."""
- from_type = rel_obj.get("from_type")
+ rel_obj.get("from_type")
from_label = rel_obj.get("from_label")
- to_type = rel_obj.get("to_type")
+ rel_obj.get("to_type")
to_label = rel_obj.get("to_label")
rel_label = rel_obj.get("rel_label", "RELATED_TO")
diff --git a/flowsint-core/tests/core/graph/test_graph_repository.py b/flowsint-core/tests/core/graph/test_graph_repository.py
index 9ca97a03..04516946 100644
--- a/flowsint-core/tests/core/graph/test_graph_repository.py
+++ b/flowsint-core/tests/core/graph/test_graph_repository.py
@@ -1,9 +1,8 @@
"""Tests for Neo4jGraphRepository."""
-import pytest
-from unittest.mock import MagicMock, patch, call
-from datetime import datetime, timezone
+from unittest.mock import MagicMock, patch
+import pytest
from flowsint_core.core.graph import Neo4jGraphRepository
@@ -56,7 +55,9 @@ class TestCreateNode:
def test_create_node_no_connection(self):
repo = repo_without_connection()
- result = repo.create_node({"nodeLabel": "test", "nodeType": "domain"}, "sketch-1")
+ result = repo.create_node(
+ {"nodeLabel": "test", "nodeType": "domain"}, "sketch-1"
+ )
assert result is None
diff --git a/flowsint-core/tests/core/graph/test_graph_service.py b/flowsint-core/tests/core/graph/test_graph_service.py
index c70adf37..cbff32e7 100644
--- a/flowsint-core/tests/core/graph/test_graph_service.py
+++ b/flowsint-core/tests/core/graph/test_graph_service.py
@@ -1,16 +1,16 @@
"""Tests for GraphService using dependency injection."""
-import pytest
from unittest.mock import MagicMock, patch
-from flowsint_types import Domain, Ip
+import pytest
from flowsint_core.core.graph import (
- GraphService,
- create_graph_service,
- GraphNode,
GraphData,
+ GraphNode,
+ GraphService,
NodeMetadata,
+ create_graph_service,
)
+from flowsint_types import Domain, Ip
from .in_memory_graph_repository import InMemoryGraphRepository
@@ -148,7 +148,8 @@ class TestCreateNodeFromFlowsintType:
)
with pytest.raises(
- Exception, match="create_node_from_flowsint_type method takes a FlowsintType"
+ Exception,
+ match="create_node_from_flowsint_type method takes a FlowsintType",
):
service.create_node_from_flowsint_type(node)
@@ -252,9 +253,7 @@ class TestCreateRelationship:
sketch_id="sketch-1", repository=mock_repo, enable_batching=True
)
- service.create_relationship(
- Domain(domain="a.com"), Domain(domain="b.com")
- )
+ service.create_relationship(Domain(domain="a.com"), Domain(domain="b.com"))
mock_repo.add_to_batch.assert_called_once()
mock_repo.create_relationship.assert_not_called()
@@ -551,7 +550,8 @@ class TestCreateGraphServiceFactory:
mock_repo = MagicMock()
with patch(
- "flowsint_core.core.graph.service.Neo4jGraphRepository", return_value=mock_repo
+ "flowsint_core.core.graph.service.Neo4jGraphRepository",
+ return_value=mock_repo,
) as MockRepoClass:
with patch("flowsint_core.core.logger.Logger") as MockLogger:
service = create_graph_service(
@@ -570,7 +570,8 @@ class TestCreateGraphServiceFactory:
mock_repo = MagicMock()
with patch(
- "flowsint_core.core.graph.service.Neo4jGraphRepository", return_value=mock_repo
+ "flowsint_core.core.graph.service.Neo4jGraphRepository",
+ return_value=mock_repo,
):
with patch("flowsint_core.core.logger.Logger"):
service = create_graph_service(sketch_id="sketch-1")
diff --git a/flowsint-core/tests/core/test_enricher_base_simplified_api.py b/flowsint-core/tests/core/test_enricher_base_simplified_api.py
index 3a2d9cd7..2f54bb8d 100644
--- a/flowsint-core/tests/core/test_enricher_base_simplified_api.py
+++ b/flowsint-core/tests/core/test_enricher_base_simplified_api.py
@@ -2,13 +2,10 @@
from typing import List
-import pytest
+from flowsint_core.core.enricher_base import Enricher
from flowsint_types.domain import Domain
-from flowsint_types.email import Email
from flowsint_types.individual import Individual
-from flowsint_core.core.enricher_base import Enricher
-
class MockEnricher(Enricher):
"""Simple enricher for testing."""
diff --git a/flowsint-core/tests/core/test_logger_singleton.py b/flowsint-core/tests/core/test_logger_singleton.py
index 31ff59b0..96119481 100644
--- a/flowsint-core/tests/core/test_logger_singleton.py
+++ b/flowsint-core/tests/core/test_logger_singleton.py
@@ -9,14 +9,15 @@ Tests cover:
- Queue management
- Shutdown behavior
"""
-import pytest
+
import threading
import time
-from unittest.mock import Mock, patch, MagicMock
+from unittest.mock import Mock, patch
from uuid import uuid4
-from flowsint_core.core.logger import LoggerSingleton
+import pytest
from flowsint_core.core.enums import EventLevel
+from flowsint_core.core.logger import LoggerSingleton
@pytest.fixture
@@ -35,17 +36,18 @@ def mock_db_session():
@pytest.fixture
def mock_get_db(mock_db_session):
"""Mock the get_db generator."""
+
def mock_generator():
yield mock_db_session
- with patch('flowsint_core.core.logger.get_db', mock_generator):
+ with patch("flowsint_core.core.logger.get_db", mock_generator):
yield mock_generator
@pytest.fixture
def mock_emit_event():
"""Mock the emit_event_task."""
- with patch('flowsint_core.core.logger.emit_event_task') as mock:
+ with patch("flowsint_core.core.logger.emit_event_task") as mock:
mock.apply = Mock()
yield mock
@@ -58,7 +60,7 @@ def logger_instance(mock_get_db, mock_emit_event):
logger = LoggerSingleton(
batch_size=5,
flush_interval=0.5,
- auto_start=False # Don't start worker for most tests
+ auto_start=False, # Don't start worker for most tests
)
yield logger
# Cleanup
@@ -113,7 +115,7 @@ class TestLoggingMethods:
# Event should be emitted immediately
mock_emit_event.apply.assert_called_once()
- call_args = mock_emit_event.apply.call_args[1]['args']
+ call_args = mock_emit_event.apply.call_args[1]["args"]
assert call_args[1] == sketch_id
assert call_args[2] == EventLevel.INFO
assert call_args[3] == message
@@ -128,7 +130,7 @@ class TestLoggingMethods:
logger_instance.error(sketch_id, message)
- call_args = mock_emit_event.apply.call_args[1]['args']
+ call_args = mock_emit_event.apply.call_args[1]["args"]
assert call_args[2] == EventLevel.FAILED
def test_warn_logs_correctly(self, logger_instance, mock_emit_event):
@@ -138,7 +140,7 @@ class TestLoggingMethods:
logger_instance.warn(sketch_id, message)
- call_args = mock_emit_event.apply.call_args[1]['args']
+ call_args = mock_emit_event.apply.call_args[1]["args"]
assert call_args[2] == EventLevel.WARNING
def test_debug_logs_correctly(self, logger_instance, mock_emit_event):
@@ -148,7 +150,7 @@ class TestLoggingMethods:
logger_instance.debug(sketch_id, message)
- call_args = mock_emit_event.apply.call_args[1]['args']
+ call_args = mock_emit_event.apply.call_args[1]["args"]
assert call_args[2] == EventLevel.DEBUG
def test_success_logs_correctly(self, logger_instance, mock_emit_event):
@@ -158,7 +160,7 @@ class TestLoggingMethods:
logger_instance.success(sketch_id, message)
- call_args = mock_emit_event.apply.call_args[1]['args']
+ call_args = mock_emit_event.apply.call_args[1]["args"]
assert call_args[2] == EventLevel.SUCCESS
def test_completed_logs_correctly(self, logger_instance, mock_emit_event):
@@ -168,7 +170,7 @@ class TestLoggingMethods:
logger_instance.completed(sketch_id, message)
- call_args = mock_emit_event.apply.call_args[1]['args']
+ call_args = mock_emit_event.apply.call_args[1]["args"]
assert call_args[2] == EventLevel.COMPLETED
def test_pending_logs_correctly(self, logger_instance, mock_emit_event):
@@ -178,7 +180,7 @@ class TestLoggingMethods:
logger_instance.pending(sketch_id, message)
- call_args = mock_emit_event.apply.call_args[1]['args']
+ call_args = mock_emit_event.apply.call_args[1]["args"]
assert call_args[2] == EventLevel.PENDING
def test_graph_append_logs_correctly(self, logger_instance, mock_emit_event):
@@ -188,7 +190,7 @@ class TestLoggingMethods:
logger_instance.graph_append(sketch_id, message)
- call_args = mock_emit_event.apply.call_args[1]['args']
+ call_args = mock_emit_event.apply.call_args[1]["args"]
assert call_args[2] == EventLevel.GRAPH_APPEND
@@ -239,13 +241,15 @@ class TestBatchInsertion:
mock_db_session.add_all.assert_called_once()
mock_db_session.commit.assert_called_once()
- def test_batch_worker_flushes_periodically(self, mock_get_db, mock_emit_event, mock_db_session):
+ def test_batch_worker_flushes_periodically(
+ self, mock_get_db, mock_emit_event, mock_db_session
+ ):
"""Test that batch worker flushes logs periodically."""
LoggerSingleton._instance = None
logger = LoggerSingleton(
batch_size=100, # High batch size
flush_interval=0.3, # Short interval for testing
- auto_start=True # Start worker
+ auto_start=True, # Start worker
)
sketch_id = str(uuid4())
@@ -324,7 +328,9 @@ class TestThreadSafety:
class TestShutdown:
"""Test shutdown behavior."""
- def test_shutdown_flushes_pending_logs(self, mock_get_db, mock_emit_event, mock_db_session):
+ def test_shutdown_flushes_pending_logs(
+ self, mock_get_db, mock_emit_event, mock_db_session
+ ):
"""Test that shutdown flushes all pending logs."""
LoggerSingleton._instance = None
logger = LoggerSingleton(batch_size=100, auto_start=False)
@@ -378,7 +384,9 @@ class TestErrorHandling:
# Logger should still be operational
logger_instance.info(sketch_id, {"message": "After error"})
- def test_event_emission_error_does_not_crash(self, logger_instance, mock_emit_event):
+ def test_event_emission_error_does_not_crash(
+ self, logger_instance, mock_emit_event
+ ):
"""Test that event emission errors don't crash the logger."""
mock_emit_event.apply.side_effect = Exception("Event emission error")
diff --git a/flowsint-core/tests/factories.py b/flowsint-core/tests/factories.py
index 31aeda7d..d6b5e076 100644
--- a/flowsint-core/tests/factories.py
+++ b/flowsint-core/tests/factories.py
@@ -1,10 +1,10 @@
"""FactoryBoy factories for all models."""
+
from datetime import datetime, timezone
from uuid import uuid4
import factory
from factory.alchemy import SQLAlchemyModelFactory
-
from flowsint_core.core.enums import EventLevel
from flowsint_core.core.models import (
Analysis,
diff --git a/flowsint-core/tests/import/json/test_json_import.py b/flowsint-core/tests/import/json/test_json_import.py
index fded7168..97acdb42 100644
--- a/flowsint-core/tests/import/json/test_json_import.py
+++ b/flowsint-core/tests/import/json/test_json_import.py
@@ -91,14 +91,15 @@ def test_json_import_preserves_apostrophes_in_string_values():
assert "Individual" in results.entities
labels = {
- str(preview.obj.nodeLabel)
- for preview in results.entities["Individual"].results
+ str(preview.obj.nodeLabel) for preview in results.entities["Individual"].results
}
assert "Sarah O'Brien" in labels
def test_json_import_still_accepts_single_quoted_payload():
"""Python-dict-style payloads (single-quoted) remain supported as fallback."""
- single_quoted = b"{'nodes': [{'id': '1', 'label': 'Alice', 'type': 'individual'}], 'edges': []}"
+ single_quoted = (
+ b"{'nodes': [{'id': '1', 'label': 'Alice', 'type': 'individual'}], 'edges': []}"
+ )
results = parse_json(single_quoted, max_preview_rows=100)
assert "Individual" in results.entities
diff --git a/flowsint-core/tests/import/test_file_parser.py b/flowsint-core/tests/import/test_file_parser.py
index 41f91b5b..15d810f9 100644
--- a/flowsint-core/tests/import/test_file_parser.py
+++ b/flowsint-core/tests/import/test_file_parser.py
@@ -1,12 +1,9 @@
-from unittest.mock import patch
-
-from flowsint_types import Domain, Ip, Username
-
from flowsint_core.imports import (
EntityPreview,
FileParseResult,
parse_import_file,
)
+from flowsint_types import Domain, Ip, Username
def test_parse_import_file_txt_basic_with_mocked_detect_type():
diff --git a/flowsint-core/tests/import/txt/test_txt_import.py b/flowsint-core/tests/import/txt/test_txt_import.py
index 833830c2..140a1c8c 100644
--- a/flowsint-core/tests/import/txt/test_txt_import.py
+++ b/flowsint-core/tests/import/txt/test_txt_import.py
@@ -4,10 +4,9 @@ Tests automatic entity detection and parsing of plain text files.
"""
import pytest
-from flowsint_core.imports.txt.parse_txt import parse_txt
-from flowsint_types import CIDR, Domain, Email, Ip, Phone, Username, Website
-
from flowsint_core.imports import parse_import_file
+from flowsint_core.imports.txt.parse_txt import parse_txt
+from flowsint_types import Domain, Email, Ip
class TestBasicTxtParsing:
diff --git a/flowsint-core/tests/repositories/test_analysis_repository.py b/flowsint-core/tests/repositories/test_analysis_repository.py
index adea30ae..18d7880d 100644
--- a/flowsint-core/tests/repositories/test_analysis_repository.py
+++ b/flowsint-core/tests/repositories/test_analysis_repository.py
@@ -1,5 +1,7 @@
"""Tests for AnalysisRepository."""
-from uuid import uuid4
+
+from flowsint_core.core.repositories import AnalysisRepository
+from flowsint_core.core.types import Role
from tests.factories import (
AnalysisFactory,
@@ -7,8 +9,6 @@ from tests.factories import (
InvestigationUserRoleFactory,
ProfileFactory,
)
-from flowsint_core.core.repositories import AnalysisRepository
-from flowsint_core.core.types import Role
class TestAnalysisRepository:
@@ -32,9 +32,7 @@ class TestAnalysisRepository:
self._setup(db_session)
user = ProfileFactory()
inv = InvestigationFactory(owner=user)
- InvestigationUserRoleFactory(
- user=user, investigation=inv, roles=[Role.OWNER]
- )
+ InvestigationUserRoleFactory(user=user, investigation=inv, roles=[Role.OWNER])
AnalysisFactory(investigation=inv, owner_id=user.id)
# Another user's investigation — should not be accessible
diff --git a/flowsint-core/tests/repositories/test_chat_repository.py b/flowsint-core/tests/repositories/test_chat_repository.py
index 23635152..4d06f628 100644
--- a/flowsint-core/tests/repositories/test_chat_repository.py
+++ b/flowsint-core/tests/repositories/test_chat_repository.py
@@ -1,15 +1,17 @@
"""Tests for ChatRepository."""
+
from datetime import datetime, timezone
from uuid import uuid4
+from flowsint_core.core.models import ChatMessage
+from flowsint_core.core.repositories import ChatRepository
+
from tests.factories import (
ChatFactory,
ChatMessageFactory,
InvestigationFactory,
ProfileFactory,
)
-from flowsint_core.core.models import ChatMessage
-from flowsint_core.core.repositories import ChatRepository
class TestChatRepository:
diff --git a/flowsint-core/tests/repositories/test_custom_type_repository.py b/flowsint-core/tests/repositories/test_custom_type_repository.py
index 02f02e23..c3780cab 100644
--- a/flowsint-core/tests/repositories/test_custom_type_repository.py
+++ b/flowsint-core/tests/repositories/test_custom_type_repository.py
@@ -1,7 +1,9 @@
"""Tests for CustomTypeRepository."""
-from tests.factories import CustomTypeFactory, ProfileFactory
+
from flowsint_core.core.repositories import CustomTypeRepository
+from tests.factories import CustomTypeFactory, ProfileFactory
+
class TestCustomTypeRepository:
def _setup(self, db_session):
diff --git a/flowsint-core/tests/repositories/test_flow_repository.py b/flowsint-core/tests/repositories/test_flow_repository.py
index b42be653..af5a7064 100644
--- a/flowsint-core/tests/repositories/test_flow_repository.py
+++ b/flowsint-core/tests/repositories/test_flow_repository.py
@@ -1,7 +1,9 @@
"""Tests for FlowRepository."""
-from tests.factories import FlowFactory
+
from flowsint_core.core.repositories import FlowRepository
+from tests.factories import FlowFactory
+
class TestFlowRepository:
def _setup(self, db_session):
diff --git a/flowsint-core/tests/repositories/test_key_repository.py b/flowsint-core/tests/repositories/test_key_repository.py
index 3a1d4f89..be534dab 100644
--- a/flowsint-core/tests/repositories/test_key_repository.py
+++ b/flowsint-core/tests/repositories/test_key_repository.py
@@ -1,9 +1,9 @@
"""Tests for KeyRepository."""
-from uuid import uuid4
-from tests.factories import KeyFactory, ProfileFactory
from flowsint_core.core.repositories import KeyRepository
+from tests.factories import KeyFactory, ProfileFactory
+
class TestKeyRepository:
def _setup(self, db_session):
diff --git a/flowsint-core/tests/repositories/test_log_repository.py b/flowsint-core/tests/repositories/test_log_repository.py
index 10f5764f..b3877521 100644
--- a/flowsint-core/tests/repositories/test_log_repository.py
+++ b/flowsint-core/tests/repositories/test_log_repository.py
@@ -1,14 +1,16 @@
"""Tests for LogRepository."""
-from datetime import datetime, timedelta, timezone
+
+from datetime import datetime
from uuid import uuid4
+from flowsint_core.core.repositories import LogRepository
+
from tests.factories import (
InvestigationFactory,
LogFactory,
ProfileFactory,
SketchFactory,
)
-from flowsint_core.core.repositories import LogRepository
class TestLogRepository:
diff --git a/flowsint-core/tests/repositories/test_profile_repository.py b/flowsint-core/tests/repositories/test_profile_repository.py
index f7d2d75c..75f1dc68 100644
--- a/flowsint-core/tests/repositories/test_profile_repository.py
+++ b/flowsint-core/tests/repositories/test_profile_repository.py
@@ -1,9 +1,11 @@
"""Tests for ProfileRepository."""
+
from uuid import uuid4
-from tests.factories import ProfileFactory
from flowsint_core.core.repositories import ProfileRepository
+from tests.factories import ProfileFactory
+
class TestProfileRepository:
def test_get_by_id(self, db_session):
@@ -41,9 +43,7 @@ class TestProfileRepository:
from flowsint_core.core.models import Profile
repo = ProfileRepository(db_session)
- profile = Profile(
- id=uuid4(), email="[email protected]", hashed_password="pw"
- )
+ profile = Profile(id=uuid4(), email="[email protected]", hashed_password="pw")
repo.add(profile)
db_session.commit()
diff --git a/flowsint-core/tests/repositories/test_scan_repository.py b/flowsint-core/tests/repositories/test_scan_repository.py
index 6d418941..0d0f865d 100644
--- a/flowsint-core/tests/repositories/test_scan_repository.py
+++ b/flowsint-core/tests/repositories/test_scan_repository.py
@@ -1,4 +1,8 @@
"""Tests for ScanRepository."""
+
+from flowsint_core.core.repositories import ScanRepository
+from flowsint_core.core.types import Role
+
from tests.factories import (
InvestigationFactory,
InvestigationUserRoleFactory,
@@ -6,8 +10,6 @@ from tests.factories import (
ScanFactory,
SketchFactory,
)
-from flowsint_core.core.repositories import ScanRepository
-from flowsint_core.core.types import Role
class TestScanRepository:
@@ -31,9 +33,7 @@ class TestScanRepository:
self._setup(db_session)
user = ProfileFactory()
inv = InvestigationFactory(owner=user)
- InvestigationUserRoleFactory(
- user=user, investigation=inv, roles=[Role.OWNER]
- )
+ InvestigationUserRoleFactory(user=user, investigation=inv, roles=[Role.OWNER])
sketch = SketchFactory(investigation=inv, owner_id=user.id)
ScanFactory(sketch=sketch)
diff --git a/flowsint-core/tests/repositories/test_sketch_repository.py b/flowsint-core/tests/repositories/test_sketch_repository.py
index f012c2b9..bb31af7a 100644
--- a/flowsint-core/tests/repositories/test_sketch_repository.py
+++ b/flowsint-core/tests/repositories/test_sketch_repository.py
@@ -1,9 +1,9 @@
"""Tests for SketchRepository."""
-from uuid import uuid4
-from tests.factories import InvestigationFactory, ProfileFactory, SketchFactory
from flowsint_core.core.repositories import SketchRepository
+from tests.factories import InvestigationFactory, ProfileFactory, SketchFactory
+
class TestSketchRepository:
def _setup(self, db_session):
diff --git a/flowsint-core/tests/services/test_enricher_template_service.py b/flowsint-core/tests/services/test_enricher_template_service.py
index 04444873..e7cf2060 100644
--- a/flowsint-core/tests/services/test_enricher_template_service.py
+++ b/flowsint-core/tests/services/test_enricher_template_service.py
@@ -1,13 +1,14 @@
"""Tests for EnricherTemplateService."""
-import pytest
-from tests.factories import EnricherTemplateFactory, ProfileFactory
+import pytest
from flowsint_core.core.repositories import EnricherTemplateRepository
from flowsint_core.core.services.enricher_template_service import (
EnricherTemplateService,
)
from flowsint_core.core.services.exceptions import ConflictError, NotFoundError
+from tests.factories import EnricherTemplateFactory, ProfileFactory
+
class TestEnricherTemplateService:
def _setup(self, db_session):
@@ -173,7 +174,12 @@ class TestEnricherTemplateService:
t = EnricherTemplateFactory(owner=user, name="T1", version=1.0)
service = self._make_service(db_session)
- new_content = {"name": "T2", "category": "domain", "version": 2.0, "description": "new desc"}
+ new_content = {
+ "name": "T2",
+ "category": "domain",
+ "version": 2.0,
+ "description": "new desc",
+ }
updated = service.update_template(t.id, user.id, {"content": new_content})
assert updated.name == "T2"
diff --git a/flowsint-core/tests/services/test_template_generator_service.py b/flowsint-core/tests/services/test_template_generator_service.py
index ae92e0d7..e0b7fad0 100644
--- a/flowsint-core/tests/services/test_template_generator_service.py
+++ b/flowsint-core/tests/services/test_template_generator_service.py
@@ -4,14 +4,12 @@ from unittest.mock import AsyncMock, MagicMock, patch
from uuid import uuid4
import pytest
-
from flowsint_core.core.services.exceptions import ValidationError
from flowsint_core.core.services.template_generator_service import (
TemplateGeneratorService,
_extract_yaml,
)
-
VALID_YAML = """\
name: ip-api-lookup
category: Ip
diff --git a/flowsint-core/tests/templates/enricher.py b/flowsint-core/tests/templates/enricher.py
index 999dd48c..74e1883b 100644
--- a/flowsint-core/tests/templates/enricher.py
+++ b/flowsint-core/tests/templates/enricher.py
@@ -6,8 +6,6 @@
from pathlib import Path
-import pytest
-
from flowsint_core.core.template_enricher import TemplateEnricher
from flowsint_core.templates.loader.yaml_loader import YamlLoader
from flowsint_core.templates.types import Template
diff --git a/flowsint-core/tests/templates/test_loader.py b/flowsint-core/tests/templates/test_loader.py
index 306ac232..1b928942 100644
--- a/flowsint-core/tests/templates/test_loader.py
+++ b/flowsint-core/tests/templates/test_loader.py
@@ -1,12 +1,9 @@
"""Tests for YamlLoader and related utilities."""
-import json
from pathlib import Path
import pytest
-
from flowsint_core.templates.loader.yaml_loader import (
- BLOCKED_HOSTNAMES,
SSRFError,
TemplateRenderError,
YamlLoader,
@@ -28,7 +25,9 @@ class TestYamlLoader:
assert isinstance(file, Template)
assert file.name == "ip-api-lookup"
assert file.category == "Ip"
- assert file.request.params == {"fields": "query,status,country,city,lat,lon,isp"}
+ assert file.request.params == {
+ "fields": "query,status,country,city,lat,lon,isp"
+ }
assert file.request.method == "GET"
def test_yaml_loader_invalid_method(self):
diff --git a/flowsint-core/tests/templates/test_template_enricher.py b/flowsint-core/tests/templates/test_template_enricher.py
index 55ed5d8b..69bd6159 100644
--- a/flowsint-core/tests/templates/test_template_enricher.py
+++ b/flowsint-core/tests/templates/test_template_enricher.py
@@ -1,18 +1,14 @@
"""Tests for TemplateEnricher."""
-import json
from pathlib import Path
from typing import Optional
-from unittest.mock import MagicMock
import httpx
import pytest
-
from flowsint_core.core.template_enricher import (
TemplateEnricher,
- TemplateEnricherError,
)
-from flowsint_core.templates.loader.yaml_loader import SSRFError, YamlLoader
+from flowsint_core.templates.loader.yaml_loader import YamlLoader
from flowsint_core.templates.types import (
Template,
TemplateHttpRequest,
@@ -50,7 +46,9 @@ def create_test_template(
category="Test",
version=1.0,
input=TemplateInput(type=input_type, key=input_key),
- output=TemplateOutput(type=output_type, is_array=is_array, array_path=array_path),
+ output=TemplateOutput(
+ type=output_type, is_array=is_array, array_path=array_path
+ ),
request=TemplateHttpRequest(
method=method,
url=url,
@@ -350,7 +348,7 @@ class TestTemplateEnricherResponseParsing:
inputs = [Ip(address="8.8.8.8")]
# Text response won't map well, but shouldn't crash
- results = await enricher.scan(inputs)
+ await enricher.scan(inputs)
# May return empty due to mapping failure, that's OK
@@ -611,7 +609,9 @@ class TestTemplateEnricherFromYaml:
def test_load_post_template(self):
"""Should load POST template from YAML."""
- template = YamlLoader.get_template_from_file(str(TEST_DIR / "example-post.yaml"))
+ template = YamlLoader.get_template_from_file(
+ str(TEST_DIR / "example-post.yaml")
+ )
enricher = TemplateEnricher(template=template, sketch_id="test")
assert enricher.request.method == "POST"
diff --git a/flowsint-core/tests/vault/test_enricher_vault_integration.py b/flowsint-core/tests/vault/test_enricher_vault_integration.py
index 8af50a54..7099039e 100644
--- a/flowsint-core/tests/vault/test_enricher_vault_integration.py
+++ b/flowsint-core/tests/vault/test_enricher_vault_integration.py
@@ -1,8 +1,10 @@
"""Tests for Enricher base class vault integration."""
-import pytest
+
import uuid
-from typing import List, Dict, Any
-from unittest.mock import Mock, MagicMock, patch
+from typing import Any, Dict, List
+from unittest.mock import Mock
+
+import pytest
from flowsint_core.core.enricher_base import Enricher
@@ -123,7 +125,9 @@ class TestResolveParams:
await enricher.async_init()
# Should have called get_secret with the vault ID
- assert any(call[0][0] == vault_id for call in mock_vault.get_secret.call_args_list)
+ assert any(
+ call[0][0] == vault_id for call in mock_vault.get_secret.call_args_list
+ )
assert "TEST_API_KEY" in enricher.params
assert enricher.params["TEST_API_KEY"] == secret_value
@@ -176,7 +180,9 @@ class TestResolveParams:
assert "Vault settings" in str(exc_info.value)
@pytest.mark.asyncio
- async def test_resolve_vault_secret_optional_with_default(self, enricher, mock_vault):
+ async def test_resolve_vault_secret_optional_with_default(
+ self, enricher, mock_vault
+ ):
"""Test that optional vault secret uses default if not found."""
# Only return secret for TEST_API_KEY, not OPTIONAL_KEY
mock_vault.get_secret.side_effect = lambda key: (
@@ -367,7 +373,7 @@ class TestEnricherExecuteWithVault:
mock_vault.get_secret.return_value = secret_value
# Execute the enricher
- result = await enricher.execute(["test"])
+ await enricher.execute(["test"])
# Vault secret should be resolved
assert mock_vault.get_secret.called
diff --git a/flowsint-core/tests/vault/test_vault.py b/flowsint-core/tests/vault/test_vault.py
index ef12466c..4c3c3ff4 100644
--- a/flowsint-core/tests/vault/test_vault.py
+++ b/flowsint-core/tests/vault/test_vault.py
@@ -1,10 +1,11 @@
"""Tests for the Vault class and secret management."""
-import pytest
+
import uuid
-import os
-from unittest.mock import Mock, MagicMock
-from flowsint_core.core.vault import Vault
+from unittest.mock import MagicMock, Mock
+
+import pytest
from flowsint_core.core.models import Key
+from flowsint_core.core.vault import Vault
@pytest.fixture
@@ -221,11 +222,14 @@ class TestVaultEncryptionDecryption:
def test_invalid_master_key_length(self, vault, monkeypatch):
"""Test that invalid master key length raises error."""
import base64
+
# Set a valid base64 but wrong length (16 bytes instead of 32)
short_key = base64.b64encode(b"0" * 16).decode()
monkeypatch.setenv("MASTER_VAULT_KEY_V1", f"base64:{short_key}")
- with pytest.raises(ValueError, match="Master key must be 32 bytes \\(256 bits\\)"):
+ with pytest.raises(
+ ValueError, match="Master key must be 32 bytes \\(256 bits\\)"
+ ):
vault._get_master_key()
def test_missing_master_key(self, vault, monkeypatch):
diff --git a/flowsint-enrichers/src/flowsint_enrichers/__init__.py b/flowsint-enrichers/src/flowsint_enrichers/__init__.py
index 2e365ba6..13d1f590 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/__init__.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/__init__.py
@@ -12,4 +12,4 @@ __all__ = [
"ENRICHER_REGISTRY",
"flowsint_enricher",
"load_all_enrichers",
-]
\ No newline at end of file
+]
diff --git a/flowsint-enrichers/src/flowsint_enrichers/asn/to_cidrs.py b/flowsint-enrichers/src/flowsint_enrichers/asn/to_cidrs.py
index 2dc8d487..0d922f32 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/asn/to_cidrs.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/asn/to_cidrs.py
@@ -1,12 +1,11 @@
-import json
import os
-from typing import List, Dict, Any, Union, Optional
+from typing import Any, Dict, List, Optional
+
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
-from flowsint_types.cidr import CIDR
from flowsint_types.asn import ASN
-from flowsint_core.utils import is_valid_asn, parse_asn
-from flowsint_core.core.logger import Logger
+from flowsint_types.cidr import CIDR
from tools.network.asnmap import AsnmapTool
@@ -75,7 +74,9 @@ class AsnToCidrsEnricher(Enricher):
asn_cidrs = []
# Use asnmap tool to get CIDR info, passing the API key
# asnmap expects ASN with "AS" prefix
- cidr_data = asnmap.launch(f"AS{asn.number}", type="asn", api_key=api_key)
+ cidr_data = asnmap.launch(
+ f"AS{asn.number}", type="asn", api_key=api_key
+ )
if cidr_data and "as_range" in cidr_data and cidr_data["as_range"]:
# Add all CIDRs for this ASN
@@ -87,7 +88,9 @@ class AsnToCidrsEnricher(Enricher):
except Exception as e:
Logger.error(
self.sketch_id,
- {"message": f"Failed to parse CIDR {cidr_str}: {str(e)}"},
+ {
+ "message": f"Failed to parse CIDR {cidr_str}: {str(e)}"
+ },
)
Logger.info(
@@ -114,7 +117,9 @@ class AsnToCidrsEnricher(Enricher):
return cidrs
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
# Create Neo4j relationships between ASNs and their corresponding CIDRs
# Use the mapping from scan if available, else fallback to zip
asn_to_cidrs = getattr(self, "_asn_to_cidrs_map", None)
diff --git a/flowsint-enrichers/src/flowsint_enrichers/cidr/to_ips.py b/flowsint-enrichers/src/flowsint_enrichers/cidr/to_ips.py
index e522e5ce..4a6f1146 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/cidr/to_ips.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/cidr/to_ips.py
@@ -1,10 +1,11 @@
import os
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional
+
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.cidr import CIDR
from flowsint_types.ip import Ip
-from flowsint_core.core.logger import Logger
from tools.network.mapcidr import MapcidrTool
@@ -110,7 +111,9 @@ class CidrToIpsEnricher(Enricher):
return ips
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
# Create Neo4j relationships between CIDRs and their corresponding IPs
# Use the mapping from scan if available, else fallback to zip
cidr_to_ips = getattr(self, "_cidr_to_ips_map", None)
diff --git a/flowsint-enrichers/src/flowsint_enrichers/crypto/to_nfts.py b/flowsint-enrichers/src/flowsint_enrichers/crypto/to_nfts.py
index 3d1676ec..9dc24fb8 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/crypto/to_nfts.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/crypto/to_nfts.py
@@ -1,11 +1,11 @@
import os
-from typing import List, Dict, Any, Optional, Union
+from typing import Any, Dict, List, Optional
+
import requests
+from dotenv import load_dotenv
from flowsint_core.core.enricher_base import Enricher
from flowsint_enrichers.registry import flowsint_enricher
-from flowsint_types.wallet import CryptoWallet, CryptoNFT
-from flowsint_core.core.logger import Logger
-from dotenv import load_dotenv
+from flowsint_types.wallet import CryptoNFT, CryptoWallet
load_dotenv()
@@ -117,7 +117,9 @@ class CryptoWalletAddressToNFTs(Enricher):
)
return nfts
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
if not self._graph_service:
return results
@@ -126,7 +128,7 @@ class CryptoWalletAddressToNFTs(Enricher):
# Create or update wallet node
self.create_node(nft.wallet)
# Create or update NFT node
- nft_key = f"{nft.contract_address}_{nft.token_id}"
+ f"{nft.contract_address}_{nft.token_id}"
self.create_node(nft)
# Create relationship from wallet to NFT
wallet_obj = CryptoWallet(address=nft.wallet.address)
diff --git a/flowsint-enrichers/src/flowsint_enrichers/crypto/to_transactions.py b/flowsint-enrichers/src/flowsint_enrichers/crypto/to_transactions.py
index 8365d6d3..7ca276ad 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/crypto/to_transactions.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/crypto/to_transactions.py
@@ -1,13 +1,14 @@
import os
-from typing import List, Dict, Any, Optional, Union
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
import requests
import requests.exceptions
-from datetime import datetime
from dotenv import load_dotenv
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.wallet import CryptoWallet, CryptoWalletTransaction
-from flowsint_core.core.logger import Logger
load_dotenv()
@@ -81,7 +82,9 @@ class CryptoWalletAddressToTransactions(Enricher):
async def scan(self, data: List[InputType]) -> List[OutputType]:
results: List[OutputType] = []
api_key = self.get_secret("ETHERSCAN_API_KEY", os.getenv("ETHERSCAN_API_KEY"))
- api_url = self.get_params().get("ETHERSCAN_API_URL", "https://api.etherscan.io/v2/api")
+ api_url = self.get_params().get(
+ "ETHERSCAN_API_URL", "https://api.etherscan.io/v2/api"
+ )
for d in data:
try:
transactions = await self._get_transactions(d.address, api_key, api_url)
@@ -172,7 +175,9 @@ class CryptoWalletAddressToTransactions(Enricher):
)
return transactions
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
if not self._graph_service:
return results
diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_asn.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_asn.py
index fe5dc992..22a4a33f 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_asn.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_asn.py
@@ -1,13 +1,11 @@
-import json
import os
-import socket
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional
+
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
-from flowsint_types.domain import Domain
from flowsint_types.asn import ASN
-from flowsint_core.utils import is_valid_domain
-from flowsint_core.core.logger import Logger
+from flowsint_types.domain import Domain
from tools.network.asnmap import AsnmapTool
diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_dns.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_dns.py
index 3709b9f6..e146e3da 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_dns.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_dns.py
@@ -1,6 +1,7 @@
from typing import Any, Dict, List, Optional
-from flowsint_core.core.logger import Logger
+
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.domain import Domain
from flowsint_types.ip import Ip
diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_history.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_history.py
index 3dfe7466..c482d791 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_history.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_history.py
@@ -6,14 +6,13 @@ from typing import Any, Dict, List, Optional, Set
from dotenv import load_dotenv
from flowsint_core.core.enricher_base import Enricher
from flowsint_core.core.logger import Logger
+from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.address import Location
from flowsint_types.domain import Domain
from flowsint_types.email import Email
from flowsint_types.individual import Individual
from flowsint_types.organization import Organization
from flowsint_types.phone import Phone
-
-from flowsint_enrichers.registry import flowsint_enricher
from tools.network.whoxy import WhoxyTool
load_dotenv()
@@ -148,7 +147,6 @@ class DomainToHistoryEnricher(Enricher):
}
self._extracted_organizations.append(organization_info)
-
def __get_infos_from_whoxy(self, domain: str, api_key: str) -> Dict[str, Any]:
"""Get WHOIS history information from Whoxy API or test data."""
infos: Dict[str, Any] = {}
diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_ip.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_ip.py
index 4157e2ab..9970f333 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_ip.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_ip.py
@@ -1,7 +1,8 @@
import socket
from typing import List
-from flowsint_core.core.logger import Logger
+
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.domain import Domain
from flowsint_types.ip import Ip
@@ -52,7 +53,9 @@ class ResolveEnricher(Enricher):
continue
return results
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
for domain_obj, ip_obj in self.domain_ip_mapping:
if not self._graph_service:
continue
diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_root_domain.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_root_domain.py
index eeb236fd..e4f461a8 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_root_domain.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_root_domain.py
@@ -1,9 +1,10 @@
-from typing import List, Union
-from flowsint_enrichers.utils import is_valid_domain, get_root_domain
+from typing import List
+
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
+from flowsint_enrichers.utils import get_root_domain
from flowsint_types.domain import Domain
-from flowsint_core.core.logger import Logger
@flowsint_enricher
@@ -53,7 +54,9 @@ class DomainToRootDomain(Enricher):
return results
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
# Use the mapping we created during scan to create relationships
for original_domain, root_domain in self.domain_root_mapping:
if not self._graph_service:
diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_ssl.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_ssl.py
index d52d7e56..9980e995 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_ssl.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_ssl.py
@@ -1,15 +1,10 @@
-import json
-import os
-import requests
+from typing import List
-from typing import Any, Dict, List, Optional
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.domain import Domain
from flowsint_types.website import Website
-from flowsint_core.core.logger import Logger
-from flowsint_types.address import Location
-
from tools.network.httpx import HttpxTool
@@ -44,34 +39,43 @@ class DomainToTLS(Enricher):
for domain in data:
try:
tool = HttpxTool()
- httpx_results = tool.launch(
- target=domain.domain,
- args=["-tls-probe"]
- )
-
- if not httpx_results :
- Logger.error(None, {"message": f"(DomainToTLS) No results for the domain '{domain.domain}'"})
+ httpx_results = tool.launch(target=domain.domain, args=["-tls-probe"])
+
+ if not httpx_results:
+ Logger.error(
+ None,
+ {
+ "message": f"(DomainToTLS) No results for the domain '{domain.domain}'"
+ },
+ )
continue
else:
- for r_line in httpx_results :
- web_url = r_line.get("url")
- web_title = r_line.get("title")
- web_content_type = r_line.get("content_type")
- web_statuscode = r_line.get("status_code") # status_code is int
-
- results.append(
- Website(
- url=web_url if web_url else None,
- domain=domain,
- active=True,
- title=web_title if web_title else None,
- content_type=web_content_type if web_content_type else None,
- status_code=web_statuscode if web_statuscode else None
- )
- )
+ for r_line in httpx_results:
+ web_url = r_line.get("url")
+ web_title = r_line.get("title")
+ web_content_type = r_line.get("content_type")
+ web_statuscode = r_line.get("status_code") # status_code is int
+
+ results.append(
+ Website(
+ url=web_url if web_url else None,
+ domain=domain,
+ active=True,
+ title=web_title if web_title else None,
+ content_type=(
+ web_content_type if web_content_type else None
+ ),
+ status_code=web_statuscode if web_statuscode else None,
+ )
+ )
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"(DomainToTLS) Exception while querying the domain '{domain.domain}': {e}"})
-
+ Logger.error(
+ self.sketch_id,
+ {
+ "message": f"(DomainToTLS) Exception while querying the domain '{domain.domain}': {e}"
+ },
+ )
+
return results
def postprocess(
@@ -97,4 +101,4 @@ class DomainToTLS(Enricher):
# Make types available at module level for easy access
InputType = DomainToTLS.InputType
-OutputType = DomainToTLS.OutputType
\ No newline at end of file
+OutputType = DomainToTLS.OutputType
diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_subdomains.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_subdomains.py
index d65d0178..249afd34 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_subdomains.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_subdomains.py
@@ -1,10 +1,11 @@
+from typing import List
+
import requests
-from typing import List, Union
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
+from flowsint_core.utils import is_valid_domain
from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.domain import Domain
-from flowsint_core.utils import is_valid_domain
-from flowsint_core.core.logger import Logger
from tools.network.subfinder import SubfinderTool
@@ -89,7 +90,9 @@ class SubdomainEnricher(Enricher):
)
return subdomains
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
output: List[OutputType] = []
for domain_obj in results:
if not self._graph_service:
@@ -107,7 +110,9 @@ class SubdomainEnricher(Enricher):
self.create_node(subdomain_obj)
# Create relationship from parent domain to subdomain
- self.create_relationship(parent_domain_obj, subdomain_obj, "HAS_SUBDOMAIN")
+ self.create_relationship(
+ parent_domain_obj, subdomain_obj, "HAS_SUBDOMAIN"
+ )
self.log_graph_message(
f"{domain_obj['domain']} -> {len(domain_obj['subdomains'])} subdomain(s) found."
diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_website.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_website.py
index 8a247164..edfc3a00 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_website.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_website.py
@@ -1,12 +1,12 @@
-from typing import Dict, List, Union
+from typing import Dict, List
+
import requests
from bs4 import BeautifulSoup
-from flowsint_core.utils import is_valid_domain
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.domain import Domain
from flowsint_types.website import Website
-from flowsint_core.core.logger import Logger
@flowsint_enricher
@@ -31,24 +31,24 @@ class DomainToWebsiteEnricher(Enricher):
def _extract_page_info(self, html_content: str) -> Dict[str, any]:
"""Extract title, description, content and technologies from HTML."""
- soup = BeautifulSoup(html_content, 'html.parser')
+ soup = BeautifulSoup(html_content, "html.parser")
# Extract title
title = None
- title_tag = soup.find('title')
+ title_tag = soup.find("title")
if title_tag:
title = title_tag.get_text().strip()
# Extract meta description
description = None
- meta_desc = soup.find('meta', attrs={'name': 'description'})
- if meta_desc and meta_desc.get('content'):
- description = meta_desc.get('content').strip()
+ meta_desc = soup.find("meta", attrs={"name": "description"})
+ if meta_desc and meta_desc.get("content"):
+ description = meta_desc.get("content").strip()
# Extract text content (remove scripts and styles)
- for script in soup(['script', 'style']):
+ for script in soup(["script", "style"]):
script.decompose()
- content = soup.get_text(separator=' ', strip=True)
+ content = soup.get_text(separator=" ", strip=True)
# Limit content to first 5000 characters
if len(content) > 5000:
content = content[:5000] + "..."
@@ -57,32 +57,35 @@ class DomainToWebsiteEnricher(Enricher):
technologies = []
# Check for common frameworks and libraries
- if soup.find('meta', attrs={'name': 'generator'}):
- generator = soup.find('meta', attrs={'name': 'generator'}).get('content')
+ if soup.find("meta", attrs={"name": "generator"}):
+ generator = soup.find("meta", attrs={"name": "generator"}).get("content")
if generator:
technologies.append(generator)
# Check for React
- if soup.find('div', id='root') or soup.find('div', id='react-root'):
- technologies.append('React')
+ if soup.find("div", id="root") or soup.find("div", id="react-root"):
+ technologies.append("React")
# Check for Vue.js
- if soup.find(attrs={'data-v-'}):
- technologies.append('Vue.js')
+ if soup.find(attrs={"data-v-"}):
+ technologies.append("Vue.js")
# Check for Angular
- if soup.find(attrs={'ng-app'}) or soup.find(attrs={'ng-version'}):
- technologies.append('Angular')
+ if soup.find(attrs={"ng-app"}) or soup.find(attrs={"ng-version"}):
+ technologies.append("Angular")
# Check for WordPress
- if soup.find('meta', attrs={'name': 'generator', 'content': lambda x: x and 'WordPress' in x}):
- technologies.append('WordPress')
+ if soup.find(
+ "meta",
+ attrs={"name": "generator", "content": lambda x: x and "WordPress" in x},
+ ):
+ technologies.append("WordPress")
return {
- 'title': title,
- 'description': description,
- 'content': content,
- 'technologies': technologies
+ "title": title,
+ "description": description,
+ "content": content,
+ "technologies": technologies,
}
async def scan(self, data: List[InputType]) -> List[OutputType]:
@@ -90,31 +93,31 @@ class DomainToWebsiteEnricher(Enricher):
for domain in data:
try:
website_data = {
- 'url': f"https://{domain.domain}",
- 'domain': domain,
- 'active': False
+ "url": f"https://{domain.domain}",
+ "domain": domain,
+ "active": False,
}
# Try HTTPS first
try:
https_url = f"https://{domain.domain}"
- response = requests.get(
- https_url, timeout=10, allow_redirects=True
- )
+ response = requests.get(https_url, timeout=10, allow_redirects=True)
if response.status_code < 400:
- website_data['url'] = https_url
- website_data['active'] = True
- website_data['status_code'] = response.status_code
+ website_data["url"] = https_url
+ website_data["active"] = True
+ website_data["status_code"] = response.status_code
# Extract relevant headers
- website_data['headers'] = {
- 'content-type': response.headers.get('content-type'),
- 'server': response.headers.get('server'),
- 'x-powered-by': response.headers.get('x-powered-by')
+ website_data["headers"] = {
+ "content-type": response.headers.get("content-type"),
+ "server": response.headers.get("server"),
+ "x-powered-by": response.headers.get("x-powered-by"),
}
# Remove None values
- website_data['headers'] = {k: v for k, v in website_data['headers'].items() if v}
+ website_data["headers"] = {
+ k: v for k, v in website_data["headers"].items() if v
+ }
# Extract page information
page_info = self._extract_page_info(response.text)
@@ -131,18 +134,20 @@ class DomainToWebsiteEnricher(Enricher):
response = requests.get(http_url, timeout=10, allow_redirects=True)
if response.status_code < 400:
- website_data['url'] = http_url
- website_data['active'] = True
- website_data['status_code'] = response.status_code
+ website_data["url"] = http_url
+ website_data["active"] = True
+ website_data["status_code"] = response.status_code
# Extract relevant headers
- website_data['headers'] = {
- 'content-type': response.headers.get('content-type'),
- 'server': response.headers.get('server'),
- 'x-powered-by': response.headers.get('x-powered-by')
+ website_data["headers"] = {
+ "content-type": response.headers.get("content-type"),
+ "server": response.headers.get("server"),
+ "x-powered-by": response.headers.get("x-powered-by"),
}
# Remove None values
- website_data['headers'] = {k: v for k, v in website_data['headers'].items() if v}
+ website_data["headers"] = {
+ k: v for k, v in website_data["headers"].items() if v
+ }
# Extract page information
page_info = self._extract_page_info(response.text)
@@ -164,11 +169,15 @@ class DomainToWebsiteEnricher(Enricher):
},
)
# Add HTTPS URL as fallback
- results.append(Website(url=f"https://{domain.domain}", domain=domain, active=False))
+ results.append(
+ Website(url=f"https://{domain.domain}", domain=domain, active=False)
+ )
return results
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
for website in results:
# Log each redirect step
if website.redirects:
diff --git a/flowsint-enrichers/src/flowsint_enrichers/domain/to_whois_history.py b/flowsint-enrichers/src/flowsint_enrichers/domain/to_whois_history.py
index 4f77b7c2..c9bf77b6 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/domain/to_whois_history.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/domain/to_whois_history.py
@@ -5,14 +5,13 @@ from typing import Any, Dict, List, Optional, Set
from dotenv import load_dotenv
from flowsint_core.core.enricher_base import Enricher
from flowsint_core.core.logger import Logger
+from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.address import Location
from flowsint_types.domain import Domain
from flowsint_types.email import Email
from flowsint_types.individual import Individual
from flowsint_types.organization import Organization
from flowsint_types.whois import Whois
-
-from flowsint_enrichers.registry import flowsint_enricher
from tools.network.whoisxml import WhoisXmlTool
load_dotenv()
@@ -80,14 +79,18 @@ class DomainToWhoisHistoryEnricher(Enricher):
if not api_data or "records" not in api_data:
Logger.info(
self.sketch_id,
- {"message": f"[WHOISXML] No WHOIS history found for {domain.domain}."},
+ {
+ "message": f"[WHOISXML] No WHOIS history found for {domain.domain}."
+ },
)
continue
records = api_data["records"]
Logger.info(
self.sketch_id,
- {"message": f"[WHOISXML] Found {len(records)} WHOIS history records for {domain.domain}"},
+ {
+ "message": f"[WHOISXML] Found {len(records)} WHOIS history records for {domain.domain}"
+ },
)
for record in records:
@@ -106,7 +109,11 @@ class DomainToWhoisHistoryEnricher(Enricher):
# Extract registrant email
email_str = registrant.get("email")
- email = Email(email=email_str) if email_str and self.__is_valid_email(email_str) else None
+ email = (
+ Email(email=email_str)
+ if email_str and self.__is_valid_email(email_str)
+ else None
+ )
domain_obj = Domain(domain=domain_name)
@@ -123,22 +130,27 @@ class DomainToWhoisHistoryEnricher(Enricher):
results.append(whois_obj)
# Store extracted data for postprocess
- self._extracted_data.append({
- "whois": whois_obj,
- "original_domain": domain,
- "record": record,
- "contacts": {
- "registrant": registrant,
- "administrative": record.get("administrativeContact") or {},
- "technical": record.get("technicalContact") or {},
- "billing": record.get("billingContact") or {},
- },
- })
+ self._extracted_data.append(
+ {
+ "whois": whois_obj,
+ "original_domain": domain,
+ "record": record,
+ "contacts": {
+ "registrant": registrant,
+ "administrative": record.get("administrativeContact")
+ or {},
+ "technical": record.get("technicalContact") or {},
+ "billing": record.get("billingContact") or {},
+ },
+ }
+ )
except Exception as e:
Logger.error(
self.sketch_id,
- {"message": f"[WHOISXML] Error fetching WHOIS history for {domain.domain}: {e}"},
+ {
+ "message": f"[WHOISXML] Error fetching WHOIS history for {domain.domain}: {e}"
+ },
)
continue
@@ -148,10 +160,12 @@ class DomainToWhoisHistoryEnricher(Enricher):
"""Fetch WHOIS history from WhoisXML API."""
tool = WhoisXmlTool()
try:
- return tool.launch(params={
- "apiKey": api_key,
- "domainName": domain,
- })
+ return tool.launch(
+ params={
+ "apiKey": api_key,
+ "domainName": domain,
+ }
+ )
except Exception as e:
Logger.error(
self.sketch_id,
@@ -207,7 +221,7 @@ class DomainToWhoisHistoryEnricher(Enricher):
"""Extract a Location from a contact block."""
street = contact.get("street", "")
city = contact.get("city")
- state = contact.get("state", "")
+ contact.get("state", "")
postal_code = contact.get("postalCode", "")
country = contact.get("country", "")
@@ -257,7 +271,10 @@ class DomainToWhoisHistoryEnricher(Enricher):
self.create_relationship(whois_obj.domain, whois_obj, "HAS_WHOIS")
# Create organization node if available
- if whois_obj.organization and whois_obj.organization.name not in processed_organizations:
+ if (
+ whois_obj.organization
+ and whois_obj.organization.name not in processed_organizations
+ ):
processed_organizations.add(whois_obj.organization.name)
self.create_node(whois_obj.organization)
self.create_relationship(
@@ -283,16 +300,23 @@ class DomainToWhoisHistoryEnricher(Enricher):
processed_individuals.add(ind_id)
self.create_node(individual)
self.create_relationship(
- individual, whois_obj.domain, f"IS_{contact_type.upper()}_CONTACT"
+ individual,
+ whois_obj.domain,
+ f"IS_{contact_type.upper()}_CONTACT",
)
# Individual emails
if individual.email_addresses:
for email_obj in individual.email_addresses:
- if email_obj.email and email_obj.email not in processed_emails:
+ if (
+ email_obj.email
+ and email_obj.email not in processed_emails
+ ):
processed_emails.add(email_obj.email)
self.create_node(email_obj)
- self.create_relationship(individual, email_obj, "HAS_EMAIL")
+ self.create_relationship(
+ individual, email_obj, "HAS_EMAIL"
+ )
# Extract location
location = self.__extract_location(contact)
@@ -301,7 +325,9 @@ class DomainToWhoisHistoryEnricher(Enricher):
if loc_id not in processed_locations:
processed_locations.add(loc_id)
self.create_node(location)
- self.create_relationship(whois_obj.domain, location, "REGISTERED_IN")
+ self.create_relationship(
+ whois_obj.domain, location, "REGISTERED_IN"
+ )
self.log_graph_message(
f"WHOIS history for {domain_name} -> registrar: {whois_obj.registrar} "
diff --git a/flowsint-enrichers/src/flowsint_enrichers/email/to_domain.py b/flowsint-enrichers/src/flowsint_enrichers/email/to_domain.py
index d90984ed..21e15c4c 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/email/to_domain.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/email/to_domain.py
@@ -1,11 +1,10 @@
from typing import List
from flowsint_core.core.enricher_base import Enricher
+from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.domain import Domain
from flowsint_types.email import Email
-from flowsint_enrichers.registry import flowsint_enricher
-
@flowsint_enricher
class EmailToDomainEnricher(Enricher):
diff --git a/flowsint-enrichers/src/flowsint_enrichers/email/to_domains.py b/flowsint-enrichers/src/flowsint_enrichers/email/to_domains.py
index 29f257fb..a67bfc1e 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/email/to_domains.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/email/to_domains.py
@@ -5,13 +5,12 @@ from typing import Any, Dict, List, Optional, Set
from dotenv import load_dotenv
from flowsint_core.core.enricher_base import Enricher
from flowsint_core.core.logger import Logger
+from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.address import Location
from flowsint_types.domain import Domain
from flowsint_types.email import Email
from flowsint_types.individual import Individual
from flowsint_types.phone import Phone
-
-from flowsint_enrichers.registry import flowsint_enricher
from tools.network.whoxy import WhoxyTool
load_dotenv()
@@ -231,7 +230,7 @@ class EmailToDomainsEnricher(Enricher):
for extracted_info in self._extracted_data:
email = extracted_info["email"]
domain = extracted_info["domain"]
- domain_data = extracted_info["domain_data"]
+ extracted_info["domain_data"]
contacts = extracted_info["contacts"]
domain_name = domain.domain
diff --git a/flowsint-enrichers/src/flowsint_enrichers/email/to_gravatar.py b/flowsint-enrichers/src/flowsint_enrichers/email/to_gravatar.py
index bb0589cf..0b75405b 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/email/to_gravatar.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/email/to_gravatar.py
@@ -1,9 +1,10 @@
import hashlib
-from typing import List, Union
+from typing import List
+
import requests
from flowsint_core.core.enricher_base import Enricher
-from flowsint_enrichers.registry import flowsint_enricher
from flowsint_core.core.logger import Logger
+from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.email import Email
from flowsint_types.gravatar import Gravatar
@@ -90,7 +91,9 @@ class EmailToGravatarEnricher(Enricher):
return results
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
for email_obj, gravatar_obj in self.email_gravatar_mapping:
if not self._graph_service:
continue
diff --git a/flowsint-enrichers/src/flowsint_enrichers/email/to_hudsonrock.py b/flowsint-enrichers/src/flowsint_enrichers/email/to_hudsonrock.py
index 758d9664..d7d74fc7 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/email/to_hudsonrock.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/email/to_hudsonrock.py
@@ -1,11 +1,10 @@
-import requests
from typing import List
+import requests
from flowsint_core.core.enricher_base import Enricher
from flowsint_core.core.logger import Logger
-from flowsint_types import Email, Device
-
from flowsint_enrichers.registry import flowsint_enricher
+from flowsint_types import Device, Email
@flowsint_enricher
@@ -34,7 +33,10 @@ class HudsonRockToEmail(Enricher):
for email_obj in data:
email_value = email_obj.email
try:
- api_request = requests.get(f'https://cavalier.hudsonrock.com/api/json/v2/osint-tools/search-by-email?email={email_value}', timeout=30)
+ api_request = requests.get(
+ f"https://cavalier.hudsonrock.com/api/json/v2/osint-tools/search-by-email?email={email_value}",
+ timeout=30,
+ )
if api_request.status_code != 200:
Logger.error(
@@ -47,10 +49,15 @@ class HudsonRockToEmail(Enricher):
try:
response_json = api_request.json()
- except Exception as e:
- Logger.error(None, {"message": f"(HudsonRockToEmail) Failed to parse JSON for {email_value}: {api_request.text}"})
+ except Exception:
+ Logger.error(
+ None,
+ {
+ "message": f"(HudsonRockToEmail) Failed to parse JSON for {email_value}: {api_request.text}"
+ },
+ )
continue
-
+
if response_json["total_user_services"] == 0:
Logger.error(
self.sketch_id,
@@ -83,12 +90,22 @@ class HudsonRockToEmail(Enricher):
results.append(
Device(
- device_id=device_id, type=device_type, os=os, last_seen=last_seen, is_desktop=is_desktop,
- ip_addresses=ip_addresses, source=source
+ device_id=device_id,
+ type=device_type,
+ os=os,
+ last_seen=last_seen,
+ is_desktop=is_desktop,
+ ip_addresses=ip_addresses,
+ source=source,
)
)
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"(HudsonRockToEmail) Exception while querying {email_value}: {e}"})
+ Logger.error(
+ self.sketch_id,
+ {
+ "message": f"(HudsonRockToEmail) Exception while querying {email_value}: {e}"
+ },
+ )
return results
@@ -102,12 +119,15 @@ class HudsonRockToEmail(Enricher):
for device in results:
try:
self.create_node(device)
-
+
for email_obj in original_input:
self.create_relationship(
- email_obj, device, "ASSOCIATED_WITH_DEVICE")
- self.log_graph_message(f"(HudsonRockToEmail) {email_obj.email} -> found device '{device.device_id}'")
-
+ email_obj, device, "ASSOCIATED_WITH_DEVICE"
+ )
+ self.log_graph_message(
+ f"(HudsonRockToEmail) {email_obj.email} -> found device '{device.device_id}'"
+ )
+
except Exception as e:
Logger.error(
self.sketch_id,
@@ -122,4 +142,4 @@ class HudsonRockToEmail(Enricher):
# Make types available at module level for easy access
InputType = HudsonRockToEmail.InputType
-OutputType = HudsonRockToEmail.OutputType
\ No newline at end of file
+OutputType = HudsonRockToEmail.OutputType
diff --git a/flowsint-enrichers/src/flowsint_enrichers/email/to_leaks.py b/flowsint-enrichers/src/flowsint_enrichers/email/to_leaks.py
index 802c6a8b..7b5bc510 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/email/to_leaks.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/email/to_leaks.py
@@ -1,13 +1,14 @@
import os
-from typing import Any, Dict, List, Optional, Union
-import requests
+from typing import Any, Dict, List, Optional
from urllib.parse import urljoin
+
+import requests
+from dotenv import load_dotenv
from flowsint_core.core.enricher_base import Enricher
-from flowsint_enrichers.registry import flowsint_enricher
from flowsint_core.core.logger import Logger
-from flowsint_types.email import Email
+from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.breach import Breach
-from dotenv import load_dotenv
+from flowsint_types.email import Email
# Load environment variables
load_dotenv()
@@ -75,7 +76,9 @@ class EmailToBreachesEnricher(Enricher):
async def scan(self, data: List[InputType]) -> List[OutputType]:
results: List[OutputType] = []
api_key = self.get_secret("HIBP_API_KEY", os.getenv("HIBP_API_KEY"))
- api_url = self.get_params().get("HIBP_API_URL", "https://haveibeenpwned.com/api/v3/breachedaccount/")
+ api_url = self.get_params().get(
+ "HIBP_API_URL", "https://haveibeenpwned.com/api/v3/breachedaccount/"
+ )
headers = {"hibp-api-key": api_key, "User-Agent": "FlowsInt-Enricher"}
for email in data:
@@ -130,7 +133,9 @@ class EmailToBreachesEnricher(Enricher):
return results
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
# Create email nodes first
for email_obj in original_input:
if not self._graph_service:
@@ -143,7 +148,7 @@ class EmailToBreachesEnricher(Enricher):
continue
# Create breach node
- breach_key = f"{breach_obj.name}_{self.sketch_id}"
+ f"{breach_obj.name}_{self.sketch_id}"
self.create_node(breach_obj)
# Create relationship between the specific email and this breach
diff --git a/flowsint-enrichers/src/flowsint_enrichers/email/to_username.py b/flowsint-enrichers/src/flowsint_enrichers/email/to_username.py
index 43965dc0..797d3682 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/email/to_username.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/email/to_username.py
@@ -1,11 +1,10 @@
from typing import List
from flowsint_core.core.enricher_base import Enricher
+from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.email import Email
from flowsint_types.username import Username
-from flowsint_enrichers.registry import flowsint_enricher
-
@flowsint_enricher
class EmailToUsernameEnricher(Enricher):
diff --git a/flowsint-enrichers/src/flowsint_enrichers/individual/to_domains.py b/flowsint-enrichers/src/flowsint_enrichers/individual/to_domains.py
index 2480ad3a..3e0431c5 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/individual/to_domains.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/individual/to_domains.py
@@ -5,9 +5,8 @@ from typing import Any, Dict, List, Optional, Set, Union
from dotenv import load_dotenv
from flowsint_core.core.enricher_base import Enricher
from flowsint_core.core.logger import Logger
-from flowsint_types import Domain, Email, Individual, Location, Phone
-
from flowsint_enrichers.registry import flowsint_enricher
+from flowsint_types import Domain, Email, Individual, Location, Phone
from tools.network.whoxy import WhoxyTool
load_dotenv()
@@ -257,7 +256,7 @@ class IndividualToDomainsEnricher(Enricher):
for extracted_info in self._extracted_data:
individual = extracted_info["individual"]
domain = extracted_info["domain"]
- domain_data = extracted_info["domain_data"]
+ extracted_info["domain_data"]
contacts = extracted_info["contacts"]
domain_name = domain.domain
diff --git a/flowsint-enrichers/src/flowsint_enrichers/individual/to_org.py b/flowsint-enrichers/src/flowsint_enrichers/individual/to_org.py
index 42df7104..8b667331 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/individual/to_org.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/individual/to_org.py
@@ -1,10 +1,10 @@
-import requests
-from typing import List, Dict, Union, Set
+from typing import Dict, List, Set, Union
+
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
-from flowsint_types.organization import Organization
from flowsint_types.individual import Individual
-from flowsint_core.core.logger import Logger
+from flowsint_types.organization import Organization
from tools.organizations.sirene import SireneTool
@@ -28,7 +28,9 @@ class IndividualToOrgEnricher(Enricher):
def key(cls) -> str:
return "fullname"
- def preprocess(self, data: Union[List[str], List[dict], List[InputType]]) -> List[InputType]:
+ def preprocess(
+ self, data: Union[List[str], List[dict], List[InputType]]
+ ) -> List[InputType]:
if not isinstance(data, list):
raise ValueError(f"Expected list input, got {type(data).__name__}")
cleaned: List[InputType] = []
@@ -280,7 +282,9 @@ class IndividualToOrgEnricher(Enricher):
)
return None
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
if not self._graph_service:
return results
@@ -312,7 +316,7 @@ class IndividualToOrgEnricher(Enricher):
individual_id = individual.full_name
for org in results:
org_key = f"{org.name}_FR"
-
+
# Create relationship between individual and organization
self.create_relationship(individual, org, "WORKS_FOR")
diff --git a/flowsint-enrichers/src/flowsint_enrichers/ip/to_asn.py b/flowsint-enrichers/src/flowsint_enrichers/ip/to_asn.py
index 0acb0c2e..b6568e10 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/ip/to_asn.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/ip/to_asn.py
@@ -1,12 +1,11 @@
-import json
import os
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional
+
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
-from flowsint_types.ip import Ip
from flowsint_types.asn import ASN
-from flowsint_core.utils import is_valid_ip
-from flowsint_core.core.logger import Logger
+from flowsint_types.ip import Ip
from tools.network.asnmap import AsnmapTool
diff --git a/flowsint-enrichers/src/flowsint_enrichers/ip/to_domain.py b/flowsint-enrichers/src/flowsint_enrichers/ip/to_domain.py
index dd2d63c7..1cb0ee81 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/ip/to_domain.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/ip/to_domain.py
@@ -1,8 +1,9 @@
import socket
-import requests
from typing import List
-from flowsint_core.core.logger import Logger
+
+import requests
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.domain import Domain
from flowsint_types.ip import Ip
@@ -64,7 +65,9 @@ class ReverseResolveEnricher(Enricher):
return results
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
for ip_obj in original_input:
self.create_node(ip_obj)
for domain_obj in results:
diff --git a/flowsint-enrichers/src/flowsint_enrichers/ip/to_dummy_domains.py b/flowsint-enrichers/src/flowsint_enrichers/ip/to_dummy_domains.py
index f8d17768..0c271702 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/ip/to_dummy_domains.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/ip/to_dummy_domains.py
@@ -1,14 +1,12 @@
-import asyncio
import time
from typing import List
from flowsint_core.core.enricher_base import Enricher
from flowsint_core.core.logger import Logger
+from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.domain import Domain
from flowsint_types.ip import Ip
-from flowsint_enrichers.registry import flowsint_enricher
-
@flowsint_enricher
class IpToDummyDomainsEnricher(Enricher):
diff --git a/flowsint-enrichers/src/flowsint_enrichers/ip/to_fraudscore.py b/flowsint-enrichers/src/flowsint_enrichers/ip/to_fraudscore.py
index 3ceb7014..fa79884d 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/ip/to_fraudscore.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/ip/to_fraudscore.py
@@ -1,15 +1,14 @@
-import json
-import os
-import requests
import datetime
+import os
+from typing import Any, Dict, List, Optional
-from typing import Any, Dict, List, Optional, Union
+import requests
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.ip import Ip
from flowsint_types.risk_profile import RiskProfile
-from flowsint_core.utils import is_valid_ip
-from flowsint_core.core.logger import Logger
+
@flowsint_enricher
class IpToFraudScore(Enricher):
@@ -73,7 +72,9 @@ class IpToFraudScore(Enricher):
results: List[OutputType] = []
self.ip_risk_mapping = []
- api_username = self.get_secret("SCAMLYTICS_USERNAME", os.getenv("SCAMLYTICS_USERNAME"))
+ api_username = self.get_secret(
+ "SCAMLYTICS_USERNAME", os.getenv("SCAMLYTICS_USERNAME")
+ )
api_key = self.get_secret("SCAMLYTICS_API_KEY", os.getenv("SCAMLYTICS_API_KEY"))
# Scamalytics assigns each account a numbered API node (api11, api12,
# ...); a hardcoded host 404s for every account that is not on it.
@@ -83,8 +84,11 @@ class IpToFraudScore(Enricher):
for ip in data:
try:
- api_request = requests.get(f'https://{api_host}.scamalytics.com/v3/{api_username}/?key={api_key}&ip={ip.address}', timeout=30)
-
+ api_request = requests.get(
+ f"https://{api_host}.scamalytics.com/v3/{api_username}/?key={api_key}&ip={ip.address}",
+ timeout=30,
+ )
+
if api_request.status_code != 200:
Logger.error(
self.sketch_id,
@@ -96,8 +100,13 @@ class IpToFraudScore(Enricher):
try:
response_json = api_request.json()
- except Exception as e:
- Logger.error(None, {"message": f"(IpToFraudScore) Failed to parse JSON for {ip.address}: {api_request.text}"})
+ except Exception:
+ Logger.error(
+ None,
+ {
+ "message": f"(IpToFraudScore) Failed to parse JSON for {ip.address}: {api_request.text}"
+ },
+ )
continue
scamalytics_info = response_json.get("scamalytics", {})
@@ -119,7 +128,7 @@ class IpToFraudScore(Enricher):
"is_vpn": "VPN",
"is_apple_icloud_private_relay": "iCloud Private Relay",
"is_amazon_aws": "Amazon AWS",
- "is_google": "Google"
+ "is_google": "Google",
}
proxy_info = scamalytics_info.get("scamalytics_proxy", {})
@@ -133,7 +142,9 @@ class IpToFraudScore(Enricher):
entity_type="IP address",
overall_risk_score=fraud_score,
risk_level=fraud_risk,
- last_updated=datetime.datetime.now(datetime.timezone.utc).isoformat(),
+ last_updated=datetime.datetime.now(
+ datetime.timezone.utc
+ ).isoformat(),
risk_factors=proxy_flags,
source="Scamalytics",
)
@@ -141,8 +152,13 @@ class IpToFraudScore(Enricher):
self.ip_risk_mapping.append((ip, risk_profile))
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"(IpToFraudScore) Exception while querying {ip.address}: {e}"})
-
+ Logger.error(
+ self.sketch_id,
+ {
+ "message": f"(IpToFraudScore) Exception while querying {ip.address}: {e}"
+ },
+ )
+
return results
def postprocess(
diff --git a/flowsint-enrichers/src/flowsint_enrichers/n8n/connector.py b/flowsint-enrichers/src/flowsint_enrichers/n8n/connector.py
index e922d8ee..54378f40 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/n8n/connector.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/n8n/connector.py
@@ -1,9 +1,10 @@
import json
+from typing import Any, Dict, List, Optional
+
import aiohttp
-from typing import List, Dict, Any, Optional
from flowsint_core.core.enricher_base import Enricher
-from flowsint_enrichers.registry import flowsint_enricher
from flowsint_core.core.logger import Logger
+from flowsint_enrichers.registry import flowsint_enricher
@flowsint_enricher
@@ -344,7 +345,9 @@ Check Flowsint logs for detailed debugging information.
# Re-raise the exception so the caller knows something went wrong
raise
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
Logger.success(
self.sketch_id, {"message": f"n8n connector results: {json.dumps(results)}"}
)
diff --git a/flowsint-enrichers/src/flowsint_enrichers/organization/to_asn.py b/flowsint-enrichers/src/flowsint_enrichers/organization/to_asn.py
index ea26784e..245a1b97 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/organization/to_asn.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/organization/to_asn.py
@@ -1,10 +1,11 @@
import os
-from typing import List, Dict, Any, Union, Optional
+from typing import Any, Dict, List, Optional
+
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
-from flowsint_types.organization import Organization
from flowsint_types.asn import ASN
-from flowsint_core.core.logger import Logger
+from flowsint_types.organization import Organization
from tools.network.asnmap import AsnmapTool
@@ -105,7 +106,9 @@ class OrgToAsnEnricher(Enricher):
return results
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
# Create Neo4j relationships between organizations and their corresponding ASNs
for input_org, result_asn in zip(original_input, results):
# Skip if no valid ASN was found
diff --git a/flowsint-enrichers/src/flowsint_enrichers/organization/to_domains.py b/flowsint-enrichers/src/flowsint_enrichers/organization/to_domains.py
index 58ce5d29..b5b4ccaf 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/organization/to_domains.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/organization/to_domains.py
@@ -1,17 +1,17 @@
import os
import re
-from typing import Any, List, Dict, Set, Optional
+from typing import Any, Dict, List, Optional, Set
+
+from dotenv import load_dotenv
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types import Email, Phone
+from flowsint_types.address import Location
from flowsint_types.domain import Domain
-from flowsint_types.organization import Organization
from flowsint_types.individual import Individual
-from flowsint_types.address import Location
-from flowsint_core.core.logger import Logger
+from flowsint_types.organization import Organization
from tools.network.whoxy import WhoxyTool
-from flowsint_core.utils import is_valid_domain, is_root_domain
-from dotenv import load_dotenv
load_dotenv()
@@ -95,15 +95,17 @@ class OrgToDomainsEnricher(Enricher):
# Store extracted data for postprocess
extracted_info = {
- 'org': org,
- 'domain': domain,
- 'domain_data': result,
- 'contacts': {
- 'registrant': result.get("registrant_contact", {}),
- 'administrative': result.get("administrative_contact", {}),
- 'technical': result.get("technical_contact", {}),
- 'billing': result.get("billing_contact", {})
- }
+ "org": org,
+ "domain": domain,
+ "domain_data": result,
+ "contacts": {
+ "registrant": result.get("registrant_contact", {}),
+ "administrative": result.get(
+ "administrative_contact", {}
+ ),
+ "technical": result.get("technical_contact", {}),
+ "billing": result.get("billing_contact", {}),
+ },
}
self._extracted_data.append(extracted_info)
@@ -183,7 +185,9 @@ class OrgToDomainsEnricher(Enricher):
)
# Extract other non-redacted information (country, email, etc.)
- self.__extract_additional_info_from_contact(contact, contact_type, domain_name, org_name)
+ self.__extract_additional_info_from_contact(
+ contact, contact_type, domain_name, org_name
+ )
else:
Logger.info(
self.sketch_id,
@@ -234,7 +238,9 @@ class OrgToDomainsEnricher(Enricher):
if self.__is_redacted(full_name) or not full_name:
Logger.info(
self.sketch_id,
- {"message": f"[WHOXY] Skipping contact with redacted/empty name: {full_name}"},
+ {
+ "message": f"[WHOXY] Skipping contact with redacted/empty name: {full_name}"
+ },
)
return None
@@ -318,9 +324,7 @@ class OrgToDomainsEnricher(Enricher):
if not all([address, city, zip_code, country]):
return None
- return Location(
- address=address, city=city, zip=zip_code, country=country
- )
+ return Location(address=address, city=city, zip=zip_code, country=country)
def __extract_organization_from_contact(
self, contact: Dict[str, Any], contact_type: str
@@ -345,7 +349,11 @@ class OrgToDomainsEnricher(Enricher):
return organization
def __extract_additional_info_from_contact(
- self, contact: Dict[str, Any], contact_type: str, domain_name: str, org_name: str
+ self,
+ contact: Dict[str, Any],
+ contact_type: str,
+ domain_name: str,
+ org_name: str,
):
"""Extract additional non-redacted information from contact data."""
# Extract country information
@@ -385,7 +393,9 @@ class OrgToDomainsEnricher(Enricher):
},
)
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
"""Create Neo4j nodes and relationships from extracted data."""
if not self._graph_service:
Logger.info(
@@ -449,7 +459,9 @@ class OrgToDomainsEnricher(Enricher):
# Create relationship between organization and domain
org_obj_domain = Organization(name=org_name)
domain_obj_org = Domain(domain=domain_name)
- self.create_relationship(org_obj_domain, domain_obj_org, "HAS_REGISTERED_DOMAIN")
+ self.create_relationship(
+ org_obj_domain, domain_obj_org, "HAS_REGISTERED_DOMAIN"
+ )
# Create individual node if not already processed
individual_id = (
@@ -467,7 +479,9 @@ class OrgToDomainsEnricher(Enricher):
# Create relationship between individual and domain
domain_obj_ind = Domain(domain=domain_name)
- self.create_relationship(individual, domain_obj_ind, f"IS_{contact_type.upper()}_CONTACT")
+ self.create_relationship(
+ individual, domain_obj_ind, f"IS_{contact_type.upper()}_CONTACT"
+ )
# Create relationship between individual and organization
org_obj_ind = Organization(name=org_name)
@@ -560,7 +574,9 @@ class OrgToDomainsEnricher(Enricher):
# Create relationship between input organization and domain
org_obj_domain2 = Organization(name=org_name)
domain_obj_org2 = Domain(domain=domain_name)
- self.create_relationship(org_obj_domain2, domain_obj_org2, "HAS_REGISTERED_DOMAIN")
+ self.create_relationship(
+ org_obj_domain2, domain_obj_org2, "HAS_REGISTERED_DOMAIN"
+ )
# Create extracted organization node if not already processed
if organization.name not in processed_organizations:
@@ -575,7 +591,11 @@ class OrgToDomainsEnricher(Enricher):
# Create relationship between extracted organization and domain
domain_obj_extracted = Domain(domain=domain_name)
- self.create_relationship(organization, domain_obj_extracted, f"IS_{contact_type.upper()}_CONTACT")
+ self.create_relationship(
+ organization,
+ domain_obj_extracted,
+ f"IS_{contact_type.upper()}_CONTACT",
+ )
self.log_graph_message(
f"Processed organization {organization.name} ({contact_type}) for domain {domain_name}"
diff --git a/flowsint-enrichers/src/flowsint_enrichers/organization/to_infos.py b/flowsint-enrichers/src/flowsint_enrichers/organization/to_infos.py
index da8232fd..ef205b2f 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/organization/to_infos.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/organization/to_infos.py
@@ -1,10 +1,11 @@
-from typing import List, Dict, Any, Union
+from typing import Dict, List
+
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
-from flowsint_types.organization import Organization
from flowsint_types.address import Location
from flowsint_types.individual import Individual
-from flowsint_core.core.logger import Logger
+from flowsint_types.organization import Organization
from tools.organizations.sirene import SireneTool
@@ -246,7 +247,9 @@ class OrgToInfosEnricher(Enricher):
)
return None
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
if not self._graph_service:
return results
@@ -265,7 +268,9 @@ class OrgToInfosEnricher(Enricher):
for dirigeant in org.dirigeants:
self.create_node(dirigeant)
self.create_relationship(org, dirigeant, "HAS_LEADER")
- self.log_graph_message(f"{org.name}: HAS_LEADER -> {dirigeant.full_name}")
+ self.log_graph_message(
+ f"{org.name}: HAS_LEADER -> {dirigeant.full_name}"
+ )
# Add siege address as Location node if available
if org.siege_geo_adresse:
@@ -292,11 +297,15 @@ class OrgToInfosEnricher(Enricher):
# Log activity codes
if org.activite_principale:
- self.log_graph_message(f"{org.name}: HAS_ACTIVITY -> {org.activite_principale}")
+ self.log_graph_message(
+ f"{org.name}: HAS_ACTIVITY -> {org.activite_principale}"
+ )
# Log legal nature
if org.nature_juridique:
- self.log_graph_message(f"{org.name}: HAS_LEGAL_NATURE -> {org.nature_juridique}")
+ self.log_graph_message(
+ f"{org.name}: HAS_LEGAL_NATURE -> {org.nature_juridique}"
+ )
return results
diff --git a/flowsint-enrichers/src/flowsint_enrichers/phone/to_carrier.py b/flowsint-enrichers/src/flowsint_enrichers/phone/to_carrier.py
index 133c5c4b..4a1f837e 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/phone/to_carrier.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/phone/to_carrier.py
@@ -1,10 +1,10 @@
-import requests
import os
+from typing import Any, Dict, List, Optional
-from typing import Dict, Any, List, Optional
+import requests
from flowsint_core.core.enricher_base import Enricher
-from flowsint_enrichers.registry import flowsint_enricher
from flowsint_core.core.logger import Logger
+from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.phone import Phone
@@ -13,8 +13,8 @@ class PhoneToCarrier(Enricher):
"""[veriphone] Looks up phone number/s carrier using veriphone API."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = Phone
- OutputType = Phone
+ InputType = Phone
+ OutputType = Phone
def __init__(
self,
@@ -54,18 +54,21 @@ class PhoneToCarrier(Enricher):
@classmethod
def key(cls) -> str:
return "number"
-
+
async def scan(self, data: List[InputType]) -> List[OutputType]:
results: List[OutputType] = []
api_key = self.get_secret("VERIPHONE_API_KEY", os.getenv("VERIPHONE_API_KEY"))
Logger.debug(self.sketch_id, {"message": f"API key present: {bool(api_key)}"})
-
+
for phone_obj in data:
phonenum_value = phone_obj.number
try:
- api_request = requests.get(f'https://api.veriphone.io/v2/verify?key={api_key}&phone={phonenum_value}', timeout=30)
-
+ api_request = requests.get(
+ f"https://api.veriphone.io/v2/verify?key={api_key}&phone={phonenum_value}",
+ timeout=30,
+ )
+
if api_request.status_code != 200:
Logger.error(
self.sketch_id,
@@ -77,23 +80,32 @@ class PhoneToCarrier(Enricher):
try:
response_json = api_request.json()
- except Exception as e:
- Logger.error(None, {"message": f"(PhoneToCarrier) Failed to parse JSON for phone number '{phonenum_value}'': {api_request.text}"})
+ except Exception:
+ Logger.error(
+ None,
+ {
+ "message": f"(PhoneToCarrier) Failed to parse JSON for phone number '{phonenum_value}'': {api_request.text}"
+ },
+ )
continue
carrier = response_json.get("carrier")
if carrier:
- results.append({
- "number": phonenum_value,
- "carrier": carrier
- })
+ results.append({"number": phonenum_value, "carrier": carrier})
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"(PhoneToCarrier) Exception while querying phone number '{phonenum_value}'': {e}"})
-
+ Logger.error(
+ self.sketch_id,
+ {
+ "message": f"(PhoneToCarrier) Exception while querying phone number '{phonenum_value}'': {e}"
+ },
+ )
+
return results
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
if not self._graph_service:
return results
@@ -106,7 +118,7 @@ class PhoneToCarrier(Enricher):
phone_obj.carrier = carrier
self.create_node(phone_obj)
-
+
self.log_graph_message(
f"(PhoneToCarrier) Found carrier ({carrier}) for the phone number '{number}'."
)
@@ -117,4 +129,4 @@ class PhoneToCarrier(Enricher):
# Make types available at module level for easy access
InputType = PhoneToCarrier.InputType
-OutputType = PhoneToCarrier.OutputType
\ No newline at end of file
+OutputType = PhoneToCarrier.OutputType
diff --git a/flowsint-enrichers/src/flowsint_enrichers/phone/to_hudsonrock.py b/flowsint-enrichers/src/flowsint_enrichers/phone/to_hudsonrock.py
index 05e3f11d..2c9e35a3 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/phone/to_hudsonrock.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/phone/to_hudsonrock.py
@@ -1,11 +1,10 @@
-import requests
from typing import List
+import requests
from flowsint_core.core.enricher_base import Enricher
from flowsint_core.core.logger import Logger
-from flowsint_types import Phone, Device
-
from flowsint_enrichers.registry import flowsint_enricher
+from flowsint_types import Device, Phone
@flowsint_enricher
@@ -34,7 +33,10 @@ class HudsonRockToPhone(Enricher):
for phone_obj in data:
phonenum_value = phone_obj.number
try:
- api_request = requests.get(f'https://cavalier.hudsonrock.com/api/json/v2/osint-tools/search-by-username?username={phonenum_value}', timeout=30)
+ api_request = requests.get(
+ f"https://cavalier.hudsonrock.com/api/json/v2/osint-tools/search-by-username?username={phonenum_value}",
+ timeout=30,
+ )
if api_request.status_code != 200:
Logger.error(
@@ -47,10 +49,15 @@ class HudsonRockToPhone(Enricher):
try:
response_json = api_request.json()
- except Exception as e:
- Logger.error(None, {"message": f"(HudsonRockToPhone) Failed to parse JSON for {phonenum_value}: {api_request.text}"})
+ except Exception:
+ Logger.error(
+ None,
+ {
+ "message": f"(HudsonRockToPhone) Failed to parse JSON for {phonenum_value}: {api_request.text}"
+ },
+ )
continue
-
+
if response_json["total_user_services"] == 0:
Logger.error(
self.sketch_id,
@@ -83,12 +90,22 @@ class HudsonRockToPhone(Enricher):
results.append(
Device(
- device_id=device_id, type=device_type, os=os, last_seen=last_seen, is_desktop=is_desktop,
- ip_addresses=ip_addresses, source=source
+ device_id=device_id,
+ type=device_type,
+ os=os,
+ last_seen=last_seen,
+ is_desktop=is_desktop,
+ ip_addresses=ip_addresses,
+ source=source,
)
)
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"(HudsonRockToPhone) Exception while querying {phonenum_value}: {e}"})
+ Logger.error(
+ self.sketch_id,
+ {
+ "message": f"(HudsonRockToPhone) Exception while querying {phonenum_value}: {e}"
+ },
+ )
return results
@@ -102,12 +119,15 @@ class HudsonRockToPhone(Enricher):
for device in results:
try:
self.create_node(device)
-
+
for phone_obj in original_input:
self.create_relationship(
- phone_obj, device, "ASSOCIATED_WITH_DEVICE")
- self.log_graph_message(f"(HudsonRockToPhone) {phone_obj.email} -> found device '{device.device_id}'")
-
+ phone_obj, device, "ASSOCIATED_WITH_DEVICE"
+ )
+ self.log_graph_message(
+ f"(HudsonRockToPhone) {phone_obj.email} -> found device '{device.device_id}'"
+ )
+
except Exception as e:
Logger.error(
self.sketch_id,
@@ -122,4 +142,4 @@ class HudsonRockToPhone(Enricher):
# Make types available at module level for easy access
InputType = HudsonRockToPhone.InputType
-OutputType = HudsonRockToPhone.OutputType
\ No newline at end of file
+OutputType = HudsonRockToPhone.OutputType
diff --git a/flowsint-enrichers/src/flowsint_enrichers/phone/to_infos.py b/flowsint-enrichers/src/flowsint_enrichers/phone/to_infos.py
index d904a290..73b41d04 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/phone/to_infos.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/phone/to_infos.py
@@ -1,11 +1,12 @@
import asyncio
-from typing import Dict, Any, List, Union
+from typing import Any, Dict, List
+
+import httpx
from flowsint_core.core.enricher_base import Enricher
-from flowsint_enrichers.registry import flowsint_enricher
-from flowsint_core.utils import is_valid_number
from flowsint_core.core.logger import Logger
+from flowsint_core.utils import is_valid_number
+from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.phone import Phone
-import httpx
@flowsint_enricher
@@ -38,7 +39,9 @@ class IgnorantEnricher(Enricher):
result = await self._perform_ignorant_research(phone_obj.number)
results.append(result)
else:
- results.append({"number": phone_obj.number, "error": "Invalid phone number"})
+ results.append(
+ {"number": phone_obj.number, "error": "Invalid phone number"}
+ )
except Exception as e:
results.append(
{
@@ -78,7 +81,9 @@ class IgnorantEnricher(Enricher):
except Exception as e:
return {"number": phone, "error": f"Error in Ignorant research: {str(e)}"}
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
"""
Create Neo4j relationships for found phone accounts.
"""
diff --git a/flowsint-enrichers/src/flowsint_enrichers/registry.py b/flowsint-enrichers/src/flowsint_enrichers/registry.py
index f21fc95a..19f1e479 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/registry.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/registry.py
@@ -7,13 +7,13 @@ Auto-discovery is performed by calling load_all_enrichers() which imports all mo
in the flowsint_enrichers package, triggering the @flowsint_enricher decorators.
"""
-import inspect
import importlib
+import inspect
import os
import sys
-from typing import Dict, Optional, Type, List, Any, TypeVar
-from flowsint_core.core.enricher_base import Enricher
+from typing import Any, Dict, List, Optional, Type, TypeVar
+from flowsint_core.core.enricher_base import Enricher
E = TypeVar("E", bound=Enricher)
diff --git a/flowsint-enrichers/src/flowsint_enrichers/social/to_hudsonrock.py b/flowsint-enrichers/src/flowsint_enrichers/social/to_hudsonrock.py
index 459a7821..5dea67d0 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/social/to_hudsonrock.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/social/to_hudsonrock.py
@@ -1,15 +1,10 @@
-import subprocess
-from pathlib import Path
from typing import List
import requests
-import json
-
from flowsint_core.core.enricher_base import Enricher
from flowsint_core.core.logger import Logger
-from flowsint_types import Device, Username
-
from flowsint_enrichers.registry import flowsint_enricher
+from flowsint_types import Device, Username
@flowsint_enricher
@@ -37,7 +32,10 @@ class HudsonRockToUsername(Enricher):
for username in data:
try:
- api_request = requests.get(f'https://cavalier.hudsonrock.com/api/json/v2/osint-tools/search-by-username?username={username.value}', timeout=30)
+ api_request = requests.get(
+ f"https://cavalier.hudsonrock.com/api/json/v2/osint-tools/search-by-username?username={username.value}",
+ timeout=30,
+ )
if api_request.status_code != 200:
Logger.error(
@@ -81,8 +79,13 @@ class HudsonRockToUsername(Enricher):
results.append(
Device(
- device_id=device_id, type=device_type, os=os, last_seen=last_seen, is_desktop=is_desktop,
- ip_addresses=ip_addresses, source=source
+ device_id=device_id,
+ type=device_type,
+ os=os,
+ last_seen=last_seen,
+ is_desktop=is_desktop,
+ ip_addresses=ip_addresses,
+ source=source,
)
)
except Exception as e:
@@ -105,11 +108,9 @@ class HudsonRockToUsername(Enricher):
for device in results:
try:
self.create_node(device)
-
+
for username in original_input:
- self.create_relationship(
- username, device, "ASSOCIATED_WITH_DEVICE"
- )
+ self.create_relationship(username, device, "ASSOCIATED_WITH_DEVICE")
self.log_graph_message(
f"{username.value} -> found device '{device.device_id}'"
)
@@ -127,4 +128,4 @@ class HudsonRockToUsername(Enricher):
# Make types available at module level for easy access
InputType = HudsonRockToUsername.InputType
-OutputType = HudsonRockToUsername.OutputType
\ No newline at end of file
+OutputType = HudsonRockToUsername.OutputType
diff --git a/flowsint-enrichers/src/flowsint_enrichers/social/to_maigret.py b/flowsint-enrichers/src/flowsint_enrichers/social/to_maigret.py
index 97f0759a..ce92de73 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/social/to_maigret.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/social/to_maigret.py
@@ -2,11 +2,12 @@ import json
import subprocess
from pathlib import Path
from typing import List
+
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types import Username
from flowsint_types.social_account import SocialAccount
-from flowsint_core.core.logger import Logger
false_positives = ["LeagueOfLegends"]
@@ -47,7 +48,9 @@ class MaigretEnricher(Enricher):
)
return output_file
- def parse_maigret_output(self, username_obj: Username, output_file: Path) -> List[SocialAccount]:
+ def parse_maigret_output(
+ self, username_obj: Username, output_file: Path
+ ) -> List[SocialAccount]:
results: List[SocialAccount] = []
if not output_file.exists():
return results
@@ -58,7 +61,9 @@ class MaigretEnricher(Enricher):
except Exception as e:
Logger.error(
self.sketch_id,
- {"message": f"Failed to load output file for {username_obj.value}: {e}"},
+ {
+ "message": f"Failed to load output file for {username_obj.value}: {e}"
+ },
)
return results
@@ -111,7 +116,9 @@ class MaigretEnricher(Enricher):
except Exception as e:
Logger.error(
self.sketch_id,
- {"message": f"Failed to create SocialAccount for {username_obj.value} on {platform}: {e}"},
+ {
+ "message": f"Failed to create SocialAccount for {username_obj.value} on {platform}: {e}"
+ },
)
continue
@@ -134,7 +141,9 @@ class MaigretEnricher(Enricher):
continue
return results
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
if not self._graph_service:
return results
@@ -145,14 +154,18 @@ class MaigretEnricher(Enricher):
# Create social profile node
self.create_node(profile)
# Create relationship
- self.create_relationship(profile.username, profile, "HAS_SOCIAL_ACCOUNT")
+ self.create_relationship(
+ profile.username, profile, "HAS_SOCIAL_ACCOUNT"
+ )
self.log_graph_message(
f"{profile.username.value} -> account found on {profile.platform}"
)
except Exception as e:
Logger.error(
self.sketch_id,
- {"message": f"Failed to create graph nodes for {profile.username.value} on {profile.platform}: {e}"},
+ {
+ "message": f"Failed to create graph nodes for {profile.username.value} on {profile.platform}: {e}"
+ },
)
continue
return results
diff --git a/flowsint-enrichers/src/flowsint_enrichers/social/to_sherlock.py b/flowsint-enrichers/src/flowsint_enrichers/social/to_sherlock.py
index 5e2a26b6..1dcc6f63 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/social/to_sherlock.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/social/to_sherlock.py
@@ -4,9 +4,8 @@ from typing import List
from flowsint_core.core.enricher_base import Enricher
from flowsint_core.core.logger import Logger
-from flowsint_types import SocialAccount, Username
-
from flowsint_enrichers.registry import flowsint_enricher
+from flowsint_types import SocialAccount, Username
@flowsint_enricher
diff --git a/flowsint-enrichers/src/flowsint_enrichers/utils.py b/flowsint-enrichers/src/flowsint_enrichers/utils.py
index a7d01c82..c8f2d670 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/utils.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/utils.py
@@ -1,16 +1,13 @@
-from urllib.parse import urlparse
-import phonenumbers
+import inspect
import ipaddress
-from phonenumbers import NumberParseException
-from pydantic import TypeAdapter, BaseModel
-from urllib.parse import urlparse
import re
-import ssl
import socket
-from typing import Dict, Any, List, Type
-from pydantic import BaseModel
-import inspect
-from typing import Any, Dict, Type
+import ssl
+from typing import Any, Dict, List, Type
+from urllib.parse import urlparse
+
+import phonenumbers
+from phonenumbers import NumberParseException
from pydantic import BaseModel, TypeAdapter
@@ -49,7 +46,7 @@ def is_valid_domain(url_or_domain: str) -> str:
return False
return True
- except Exception as e:
+ except Exception:
return False
@@ -148,7 +145,7 @@ def get_root_domain(domain: str) -> str:
# For regular TLDs, take the last 2 parts (e.g., example.com)
if len(parts) >= 2:
return ".".join(parts[-2:])
-
+
return domain
except Exception:
# If we can't parse it, return the original domain
@@ -343,6 +340,7 @@ def get_inline_relationships(nodes: List[Any], edges: List[Any]) -> List[str]:
def to_json_serializable(obj):
"""Convert any object to a JSON-serializable format."""
import json
+
from pydantic import BaseModel
try:
diff --git a/flowsint-enrichers/src/flowsint_enrichers/website/to_crawler.py b/flowsint-enrichers/src/flowsint_enrichers/website/to_crawler.py
index e5a2c320..f8805b4b 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/website/to_crawler.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/website/to_crawler.py
@@ -1,13 +1,14 @@
-from typing import List, Optional, Union
+from typing import List, Optional
from urllib.parse import urlparse
+
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
-from flowsint_types.website import Website
-from flowsint_types.phone import Phone
from flowsint_types.email import Email
-from flowsint_core.core.logger import Logger
-from tools.network.reconcrawl import ReconCrawlTool
+from flowsint_types.phone import Phone
+from flowsint_types.website import Website
from pydantic import BaseModel
+from tools.network.reconcrawl import ReconCrawlTool
class ReturnType(BaseModel):
@@ -89,7 +90,9 @@ class WebsiteToCrawler(Enricher):
except Exception as e:
Logger.warn(
self.sketch_id,
- {"message": f"Skipping invalid email '{item.value}': {e}"},
+ {
+ "message": f"Skipping invalid email '{item.value}': {e}"
+ },
)
if item.type == "phone":
try:
@@ -97,7 +100,9 @@ class WebsiteToCrawler(Enricher):
except Exception as e:
Logger.warn(
self.sketch_id,
- {"message": f"Skipping invalid phone '{item.value}': {e}"},
+ {
+ "message": f"Skipping invalid phone '{item.value}': {e}"
+ },
)
# Log results
@@ -148,7 +153,9 @@ class WebsiteToCrawler(Enricher):
return results
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
# Create Neo4j relationships between websites and their corresponding emails and phones
for input_website, result in zip(original_input, results):
website_url = str(input_website.url)
diff --git a/flowsint-enrichers/src/flowsint_enrichers/website/to_domain.py b/flowsint-enrichers/src/flowsint_enrichers/website/to_domain.py
index bab0d615..73f126db 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/website/to_domain.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/website/to_domain.py
@@ -1,10 +1,11 @@
-from typing import List, Union
+from typing import List
from urllib.parse import urlparse
+
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
-from flowsint_types.website import Website
from flowsint_types.domain import Domain
-from flowsint_core.core.logger import Logger
+from flowsint_types.website import Website
@flowsint_enricher
diff --git a/flowsint-enrichers/src/flowsint_enrichers/website/to_links.py b/flowsint-enrichers/src/flowsint_enrichers/website/to_links.py
index 585b596f..9f74f310 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/website/to_links.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/website/to_links.py
@@ -1,10 +1,11 @@
-from typing import List, Union
+from typing import List
from urllib.parse import urlparse
+
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
-from flowsint_types.website import Website
from flowsint_types.domain import Domain
-from flowsint_core.core.logger import Logger
+from flowsint_types.website import Website
from reconspread import Crawler
@@ -56,7 +57,9 @@ class WebsiteToLinks(Enricher):
if main_domain:
domain_obj = Domain(domain=main_domain)
self.create_node(domain_obj)
- self.create_relationship(website, domain_obj, "BELONGS_TO_DOMAIN")
+ self.create_relationship(
+ website, domain_obj, "BELONGS_TO_DOMAIN"
+ )
self.log_graph_message(
f"Website {str(website.url)} belongs to domain {main_domain}"
)
@@ -86,9 +89,13 @@ class WebsiteToLinks(Enricher):
if domain != main_domain:
domain_obj_ext = Domain(domain=domain)
self.create_node(domain_obj_ext)
- self.create_relationship(url_obj, domain_obj_ext, "BELONGS_TO_DOMAIN")
+ self.create_relationship(
+ url_obj, domain_obj_ext, "BELONGS_TO_DOMAIN"
+ )
domain_obj_main = Domain(domain=main_domain)
- self.create_relationship(domain_obj_main, domain_obj_ext, "LINKS_TO")
+ self.create_relationship(
+ domain_obj_main, domain_obj_ext, "LINKS_TO"
+ )
self.log_graph_message(
f"External website {url} belongs to domain {domain}"
)
@@ -107,7 +114,9 @@ class WebsiteToLinks(Enricher):
): # Don't create duplicate of main website
internal_website = Website(url=url)
self.create_node(internal_website)
- self.create_relationship(website, internal_website, "LINKS_TO")
+ self.create_relationship(
+ website, internal_website, "LINKS_TO"
+ )
self.log_graph_message(
f"Website {str(website.url)} links to internal website {url}"
)
@@ -115,7 +124,11 @@ class WebsiteToLinks(Enricher):
# Also link internal websites to main domain
if main_domain:
domain_obj_int = Domain(domain=main_domain)
- self.create_relationship(internal_website, domain_obj_int, "BELONGS_TO_DOMAIN")
+ self.create_relationship(
+ internal_website,
+ domain_obj_int,
+ "BELONGS_TO_DOMAIN",
+ )
Logger.info(
self.sketch_id, {"message": f"[INTERNAL] Found: {url}"}
)
@@ -185,7 +198,9 @@ class WebsiteToLinks(Enricher):
if main_domain:
domain_obj_err = Domain(domain=main_domain)
self.create_node(domain_obj_err)
- self.create_relationship(website, domain_obj_err, "BELONGS_TO_DOMAIN")
+ self.create_relationship(
+ website, domain_obj_err, "BELONGS_TO_DOMAIN"
+ )
self.log_graph_message(
f"Website {str(website.url)} belongs to domain {main_domain}"
)
@@ -204,7 +219,9 @@ class WebsiteToLinks(Enricher):
return results
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
# Neo4j nodes and relationships are created in real-time during the callback
# No additional processing needed here
return results
diff --git a/flowsint-enrichers/src/flowsint_enrichers/website/to_subdomain.py b/flowsint-enrichers/src/flowsint_enrichers/website/to_subdomain.py
index 79ad423d..d88881fd 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/website/to_subdomain.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/website/to_subdomain.py
@@ -1,10 +1,12 @@
-from typing import List, Union, Optional, Dict, Any
+import os
+from typing import Any, Dict, List, Optional
+
+import requests
from flowsint_core.core.enricher_base import Enricher
-from flowsint_enrichers.registry import flowsint_enricher
from flowsint_core.core.logger import Logger
+from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.website import Website
-import requests
-import os
+
@flowsint_enricher
class WebsiteToSubdomains(Enricher):
@@ -58,7 +60,10 @@ class WebsiteToSubdomains(Enricher):
for website in data:
try:
- api_request = requests.get(f'https://api.c99.nl/subdomainfinder?key={api_key}&domain={website.url}', timeout=30)
+ api_request = requests.get(
+ f"https://api.c99.nl/subdomainfinder?key={api_key}&domain={website.url}",
+ timeout=30,
+ )
if api_request.status_code != 200:
Logger.error(
@@ -73,7 +78,12 @@ class WebsiteToSubdomains(Enricher):
for subdomain in response_json.get("subdomains", []):
url = subdomain.get("subdomain")
- if website.url not in [f'http://{url}', f'https://{url}', f'http://www.{url}', f'https://www.{url}']:
+ if website.url not in [
+ f"http://{url}",
+ f"https://{url}",
+ f"http://www.{url}",
+ f"https://www.{url}",
+ ]:
ip = subdomain.get("ip")
cloudflare = []
@@ -84,7 +94,9 @@ class WebsiteToSubdomains(Enricher):
results.append(
Website(
- url=f"http://{url}", description=f"IP address: {ip}", technologies=cloudflare
+ url=f"http://{url}",
+ description=f"IP address: {ip}",
+ technologies=cloudflare,
)
)
@@ -98,7 +110,9 @@ class WebsiteToSubdomains(Enricher):
return results
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
if not self._graph_service:
return results
@@ -116,12 +130,11 @@ class WebsiteToSubdomains(Enricher):
except Exception as e:
Logger.error(
self.sketch_id,
- {
- "message": f"(WebsiteToSubdomains) relationship error: {e}"
- },
+ {"message": f"(WebsiteToSubdomains) relationship error: {e}"},
)
return results
+
InputType = WebsiteToSubdomains.InputType
-OutputType = WebsiteToSubdomains.OutputType
\ No newline at end of file
+OutputType = WebsiteToSubdomains.OutputType
diff --git a/flowsint-enrichers/src/flowsint_enrichers/website/to_technologies.py b/flowsint-enrichers/src/flowsint_enrichers/website/to_technologies.py
index a4e4b0cc..1e96c1aa 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/website/to_technologies.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/website/to_technologies.py
@@ -1,9 +1,10 @@
from typing import Any, Dict, List, Optional
-from flowsint_core.core.logger import Logger
+
from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
from flowsint_enrichers.registry import flowsint_enricher
-from flowsint_types.website import Website
from flowsint_types.technology import Technology
+from flowsint_types.website import Website
from tools.network.httpx import HttpxTool
diff --git a/flowsint-enrichers/src/flowsint_enrichers/website/to_text.py b/flowsint-enrichers/src/flowsint_enrichers/website/to_text.py
index 7e53d40c..8656cbc1 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/website/to_text.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/website/to_text.py
@@ -1,10 +1,11 @@
-from typing import List, Union
+from typing import List
+
+import requests
+from bs4 import BeautifulSoup
from flowsint_core.core.enricher_base import Enricher
from flowsint_enrichers.registry import flowsint_enricher
from flowsint_types.phrase import Phrase
from flowsint_types.website import Website
-import requests
-from bs4 import BeautifulSoup
@flowsint_enricher
@@ -49,7 +50,9 @@ class WebsiteToText(Enricher):
print(f"An error occurred: {e}")
return None
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
# Create Neo4j relationships between websites and their corresponding phrases
for input_website, result in zip(original_input, results):
website_url = str(input_website.url)
diff --git a/flowsint-enrichers/src/flowsint_enrichers/website/to_webtrackers.py b/flowsint-enrichers/src/flowsint_enrichers/website/to_webtrackers.py
index 418616de..671ff1c9 100644
--- a/flowsint-enrichers/src/flowsint_enrichers/website/to_webtrackers.py
+++ b/flowsint-enrichers/src/flowsint_enrichers/website/to_webtrackers.py
@@ -1,10 +1,11 @@
-from typing import List, Dict, Any, Union, Optional
+from typing import Any, Dict, List, Optional
+
from flowsint_core.core.enricher_base import Enricher
-from flowsint_enrichers.registry import flowsint_enricher
-from flowsint_types.website import Website
-from flowsint_types.web_tracker import WebTracker
from flowsint_core.core.logger import Logger
from flowsint_core.core.vault import VaultProtocol
+from flowsint_enrichers.registry import flowsint_enricher
+from flowsint_types.web_tracker import WebTracker
+from flowsint_types.website import Website
from recontrack import TrackingCodeExtractor
@@ -70,7 +71,9 @@ class WebsiteToWebtrackersEnricher(Enricher):
return results
- def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
+ def postprocess(
+ self, results: List[OutputType], original_input: List[InputType]
+ ) -> List[OutputType]:
# Create Neo4j relationships between websites and their corresponding trackers
if self._graph_service:
# Group trackers by website using the mapping we created during scan
diff --git a/flowsint-enrichers/src/tools/__init__.py b/flowsint-enrichers/src/tools/__init__.py
index 734926b1..b01e0885 100644
--- a/flowsint-enrichers/src/tools/__init__.py
+++ b/flowsint-enrichers/src/tools/__init__.py
@@ -8,4 +8,4 @@ from .dockertool import DockerTool
__version__ = "0.1.0"
__author__ = "dextmorgn <[email protected]>"
-__all__ = ["Tool", "DockerTool"]
\ No newline at end of file
+__all__ = ["Tool", "DockerTool"]
diff --git a/flowsint-enrichers/src/tools/network/asnmap.py b/flowsint-enrichers/src/tools/network/asnmap.py
index df581273..0a612b64 100644
--- a/flowsint-enrichers/src/tools/network/asnmap.py
+++ b/flowsint-enrichers/src/tools/network/asnmap.py
@@ -1,5 +1,6 @@
import json
from typing import Any, Literal
+
from ..dockertool import DockerTool
@@ -51,7 +52,10 @@ class AsnmapTool(DockerTool):
return super().is_installed()
def launch(
- self, item: str, type: Literal["domain", "organization", "ip", "asn"] = "domain", api_key: str = None
+ self,
+ item: str,
+ type: Literal["domain", "organization", "ip", "asn"] = "domain",
+ api_key: str = None,
) -> Any:
flags = {"domain": "-d", "org": "-org", "ip": "-i", "asn": "-a"}
if type not in flags:
@@ -70,7 +74,7 @@ class AsnmapTool(DockerTool):
result = super().launch(f"{flag} {item} -silent -json", environment=env)
if result and result != "":
# asnmap returns newline-delimited JSON (one JSON object per line)
- lines = result.strip().split('\n')
+ lines = result.strip().split("\n")
if len(lines) == 1:
# Single JSON object
@@ -93,14 +97,21 @@ class AsnmapTool(DockerTool):
combined_data["as_range"].extend(data["as_range"])
if data.get("as_name") and not combined_data["as_name"]:
combined_data["as_name"] = data["as_name"]
- if data.get("as_country") and not combined_data["as_country"]:
+ if (
+ data.get("as_country")
+ and not combined_data["as_country"]
+ ):
combined_data["as_country"] = data["as_country"]
if data.get("as_number") and not combined_data["as_number"]:
combined_data["as_number"] = data["as_number"]
except json.JSONDecodeError:
continue
- return combined_data if combined_data["as_range"] or combined_data["as_number"] else {}
+ return (
+ combined_data
+ if combined_data["as_range"] or combined_data["as_number"]
+ else {}
+ )
else:
return {}
except json.JSONDecodeError as e:
diff --git a/flowsint-enrichers/src/tools/network/dnsx.py b/flowsint-enrichers/src/tools/network/dnsx.py
index cfa140ce..f5e14f06 100644
--- a/flowsint-enrichers/src/tools/network/dnsx.py
+++ b/flowsint-enrichers/src/tools/network/dnsx.py
@@ -1,6 +1,7 @@
import json
import shlex
from typing import List
+
from ..dockertool import DockerTool
diff --git a/flowsint-enrichers/src/tools/network/httpx.py b/flowsint-enrichers/src/tools/network/httpx.py
index ed13c88a..d77bd0dc 100644
--- a/flowsint-enrichers/src/tools/network/httpx.py
+++ b/flowsint-enrichers/src/tools/network/httpx.py
@@ -1,5 +1,6 @@
import json
from typing import Any, List
+
from ..dockertool import DockerTool
diff --git a/flowsint-enrichers/src/tools/network/mapcidr.py b/flowsint-enrichers/src/tools/network/mapcidr.py
index e019edce..70a5e862 100644
--- a/flowsint-enrichers/src/tools/network/mapcidr.py
+++ b/flowsint-enrichers/src/tools/network/mapcidr.py
@@ -1,4 +1,5 @@
from typing import List
+
from ..dockertool import DockerTool
diff --git a/flowsint-enrichers/src/tools/network/naabu.py b/flowsint-enrichers/src/tools/network/naabu.py
index c9bdad53..30bcf06d 100644
--- a/flowsint-enrichers/src/tools/network/naabu.py
+++ b/flowsint-enrichers/src/tools/network/naabu.py
@@ -1,5 +1,6 @@
import json
-from typing import Any, List, Optional, Literal
+from typing import List, Literal, Optional
+
from ..dockertool import DockerTool
@@ -16,7 +17,9 @@ class NaabuTool(DockerTool):
@classmethod
def description(cls) -> str:
- return "Fast port scanner written in Go with focus on reliability and simplicity."
+ return (
+ "Fast port scanner written in Go with focus on reliability and simplicity."
+ )
@classmethod
def category(cls) -> str:
diff --git a/flowsint-enrichers/src/tools/network/reconcrawl.py b/flowsint-enrichers/src/tools/network/reconcrawl.py
index 8183b4a5..22109351 100644
--- a/flowsint-enrichers/src/tools/network/reconcrawl.py
+++ b/flowsint-enrichers/src/tools/network/reconcrawl.py
@@ -28,7 +28,7 @@ class ReconCrawlTool(Tool):
def is_installed(self) -> bool:
try:
- import reconcrawl
+ pass
return True
except ImportError:
diff --git a/flowsint-enrichers/src/tools/network/subfinder.py b/flowsint-enrichers/src/tools/network/subfinder.py
index 2c8c5e81..17381b8e 100644
--- a/flowsint-enrichers/src/tools/network/subfinder.py
+++ b/flowsint-enrichers/src/tools/network/subfinder.py
@@ -1,7 +1,9 @@
from typing import Any, List
-from ..dockertool import DockerTool
+
from flowsint_core.utils import is_valid_domain
+from ..dockertool import DockerTool
+
class SubfinderTool(DockerTool):
image = "projectdiscovery/subfinder"
diff --git a/flowsint-enrichers/src/tools/network/whoisxml.py b/flowsint-enrichers/src/tools/network/whoisxml.py
index 9f603c84..17c5c4bf 100644
--- a/flowsint-enrichers/src/tools/network/whoisxml.py
+++ b/flowsint-enrichers/src/tools/network/whoisxml.py
@@ -1,6 +1,7 @@
from typing import Dict
import requests
+
from ..base import Tool
diff --git a/flowsint-enrichers/src/tools/network/whoxy.py b/flowsint-enrichers/src/tools/network/whoxy.py
index 2e31a77a..d21d4912 100644
--- a/flowsint-enrichers/src/tools/network/whoxy.py
+++ b/flowsint-enrichers/src/tools/network/whoxy.py
@@ -40,7 +40,7 @@ class WhoxyTool(Tool):
)
if resp.status_code == 429:
- delay = self.INITIAL_DELAY * (2 ** attempt)
+ delay = self.INITIAL_DELAY * (2**attempt)
time.sleep(delay)
continue
@@ -56,7 +56,7 @@ class WhoxyTool(Tool):
except requests.exceptions.HTTPError as e:
if resp.status_code in (429, 502, 503, 504):
last_exception = e
- delay = self.INITIAL_DELAY * (2 ** attempt)
+ delay = self.INITIAL_DELAY * (2**attempt)
time.sleep(delay)
continue
raise RuntimeError(f"{str(e)}")
@@ -64,8 +64,10 @@ class WhoxyTool(Tool):
raise
except Exception as e:
last_exception = e
- delay = self.INITIAL_DELAY * (2 ** attempt)
+ delay = self.INITIAL_DELAY * (2**attempt)
time.sleep(delay)
continue
- raise RuntimeError(f"Whoxy API failed after {self.MAX_RETRIES} retries: {str(last_exception)}")
+ raise RuntimeError(
+ f"Whoxy API failed after {self.MAX_RETRIES} retries: {str(last_exception)}"
+ )
diff --git a/flowsint-enrichers/src/tools/organizations/sirene.py b/flowsint-enrichers/src/tools/organizations/sirene.py
index a19ad8bb..5ce1c7de 100644
--- a/flowsint-enrichers/src/tools/organizations/sirene.py
+++ b/flowsint-enrichers/src/tools/organizations/sirene.py
@@ -1,6 +1,7 @@
from typing import Dict
import requests
+
from ..base import Tool
diff --git a/flowsint-enrichers/tests/conftest.py b/flowsint-enrichers/tests/conftest.py
index 161ca750..c12d21c7 100644
--- a/flowsint-enrichers/tests/conftest.py
+++ b/flowsint-enrichers/tests/conftest.py
@@ -1,4 +1,5 @@
import pytest
+
from tests.logger import TestLogger
diff --git a/flowsint-enrichers/tests/enrichers/test_domain_to_dns.py b/flowsint-enrichers/tests/enrichers/test_domain_to_dns.py
index a7c89525..a8ce0d79 100644
--- a/flowsint-enrichers/tests/enrichers/test_domain_to_dns.py
+++ b/flowsint-enrichers/tests/enrichers/test_domain_to_dns.py
@@ -1,5 +1,4 @@
import pytest
-
from flowsint_enrichers import ENRICHER_REGISTRY
from flowsint_enrichers.domain.to_dns import DomainToDnsEnricher
from flowsint_types.ip import Ip
@@ -81,10 +80,10 @@ class _FakeDnsx:
@pytest.mark.asyncio
async def test_scan_returns_ip_objects_tagged_with_source_domain(monkeypatch):
- fake = _FakeDnsx({"example.com": ["93.184.216.34", "2606:2800:220:1:248:1893:25c8:1946"]})
- monkeypatch.setattr(
- "flowsint_enrichers.domain.to_dns.DnsxTool", lambda: fake
+ fake = _FakeDnsx(
+ {"example.com": ["93.184.216.34", "2606:2800:220:1:248:1893:25c8:1946"]}
)
+ monkeypatch.setattr("flowsint_enrichers.domain.to_dns.DnsxTool", lambda: fake)
enricher = DomainToDnsEnricher(sketch_id="s", scan_id="t", graph_service=None)
from flowsint_types.domain import Domain
@@ -102,9 +101,7 @@ async def test_scan_returns_ip_objects_tagged_with_source_domain(monkeypatch):
@pytest.mark.asyncio
async def test_scan_ipv6_param_disables_aaaa(monkeypatch):
fake = _FakeDnsx({"example.com": ["93.184.216.34"]})
- monkeypatch.setattr(
- "flowsint_enrichers.domain.to_dns.DnsxTool", lambda: fake
- )
+ monkeypatch.setattr("flowsint_enrichers.domain.to_dns.DnsxTool", lambda: fake)
enricher = DomainToDnsEnricher(
sketch_id="s", scan_id="t", graph_service=None, params={"ipv6": "false"}
@@ -120,9 +117,7 @@ async def test_scan_ipv6_param_disables_aaaa(monkeypatch):
@pytest.mark.asyncio
async def test_scan_skips_unresolvable_domain(monkeypatch):
fake = _FakeDnsx({}) # no records for anything
- monkeypatch.setattr(
- "flowsint_enrichers.domain.to_dns.DnsxTool", lambda: fake
- )
+ monkeypatch.setattr("flowsint_enrichers.domain.to_dns.DnsxTool", lambda: fake)
enricher = DomainToDnsEnricher(sketch_id="s", scan_id="t", graph_service=None)
from flowsint_types.domain import Domain
diff --git a/flowsint-enrichers/tests/enrichers/test_tech_detect.py b/flowsint-enrichers/tests/enrichers/test_tech_detect.py
index 6541d671..0fca0d49 100644
--- a/flowsint-enrichers/tests/enrichers/test_tech_detect.py
+++ b/flowsint-enrichers/tests/enrichers/test_tech_detect.py
@@ -1,5 +1,4 @@
import pytest
-
from flowsint_enrichers import ENRICHER_REGISTRY
from flowsint_enrichers.website.to_technologies import TechDetectEnricher
from flowsint_types.technology import Technology
diff --git a/flowsint-enrichers/tests/logger.py b/flowsint-enrichers/tests/logger.py
index b00a2f02..a33b405e 100644
--- a/flowsint-enrichers/tests/logger.py
+++ b/flowsint-enrichers/tests/logger.py
@@ -1,4 +1,4 @@
-from typing import Literal, Union, Any
+from typing import Any, Literal, Union
from uuid import UUID
EventLevel = Literal["info", "warn", "error", "success", "debug"]
diff --git a/flowsint-enrichers/tests/test_vault_integration.py b/flowsint-enrichers/tests/test_vault_integration.py
index dc1997de..11c3ad65 100644
--- a/flowsint-enrichers/tests/test_vault_integration.py
+++ b/flowsint-enrichers/tests/test_vault_integration.py
@@ -1,13 +1,15 @@
"""Integration tests for enrichers using vault secrets."""
-import pytest
+
import uuid
from unittest.mock import Mock
-from flowsint_enrichers.domain.to_history import DomainToHistoryEnricher
-from flowsint_enrichers.individual.to_domains import IndividualToDomainsEnricher
-from flowsint_enrichers.email.to_domains import EmailToDomainsEnricher
+
+import pytest
from flowsint_enrichers.crypto.to_nfts import CryptoWalletAddressToNFTs
from flowsint_enrichers.crypto.to_transactions import CryptoWalletAddressToTransactions
+from flowsint_enrichers.domain.to_history import DomainToHistoryEnricher
+from flowsint_enrichers.email.to_domains import EmailToDomainsEnricher
from flowsint_enrichers.email.to_leaks import EmailToBreachesEnricher
+from flowsint_enrichers.individual.to_domains import IndividualToDomainsEnricher
@pytest.fixture
@@ -227,7 +229,9 @@ class TestMissingRequiredVaultSecret:
"""Tests for error handling when required vault secrets are missing."""
@pytest.mark.asyncio
- async def test_missing_required_secret_raises_exception(self, mock_vault, sketch_id):
+ async def test_missing_required_secret_raises_exception(
+ self, mock_vault, sketch_id
+ ):
"""Test that missing required secret raises an exception."""
# Vault returns None (secret not found)
mock_vault.get_secret.return_value = None
diff --git a/flowsint-enrichers/tests/tools/organizations/sirene.py b/flowsint-enrichers/tests/tools/organizations/sirene.py
index 762b3b98..333e43fc 100644
--- a/flowsint-enrichers/tests/tools/organizations/sirene.py
+++ b/flowsint-enrichers/tests/tools/organizations/sirene.py
@@ -1,4 +1,5 @@
from typing import Dict
+
from tools.organizations.sirene import SireneTool
tool = SireneTool()
diff --git a/flowsint-types/src/flowsint_types/address.py b/flowsint-types/src/flowsint_types/address.py
index a04ce1cd..e7c408dd 100644
--- a/flowsint-types/src/flowsint_types/address.py
+++ b/flowsint-types/src/flowsint_types/address.py
@@ -1,5 +1,7 @@
-from pydantic import Field, model_validator
from typing import Optional, Self
+
+from pydantic import Field, model_validator
+
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -8,7 +10,12 @@ from .registry import flowsint_type
class Location(FlowsintType):
"""Represents a physical address with geographical coordinates."""
- address: str = Field(..., description="Street address", title="Street Address", json_schema_extra={"primary": True})
+ address: str = Field(
+ ...,
+ description="Street address",
+ title="Street Address",
+ json_schema_extra={"primary": True},
+ )
city: str = Field(..., description="City name", title="City")
country: str = Field(..., description="Country name", title="Country")
zip: str = Field(..., description="ZIP or postal code", title="ZIP/Postal Code")
diff --git a/flowsint-types/src/flowsint_types/affiliation.py b/flowsint-types/src/flowsint_types/affiliation.py
index ea2cd80d..1f5fc051 100644
--- a/flowsint-types/src/flowsint_types/affiliation.py
+++ b/flowsint-types/src/flowsint_types/affiliation.py
@@ -1,5 +1,6 @@
+from typing import List, Optional, Self
+
from pydantic import Field, model_validator
-from typing import Optional, List, Self
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -10,7 +11,10 @@ class Affiliation(FlowsintType):
"""Represents an organizational affiliation or employment relationship."""
organization: str = Field(
- ..., description="Organization or group name", title="Organization", json_schema_extra={"primary": True}
+ ...,
+ description="Organization or group name",
+ title="Organization",
+ json_schema_extra={"primary": True},
)
role: Optional[str] = Field(
None, description="Role or position within organization", title="Role"
@@ -58,7 +62,7 @@ class Affiliation(FlowsintType):
title="Hierarchy Level",
)
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
if self.role:
self.nodeLabel = f"{self.role} at {self.organization}"
diff --git a/flowsint-types/src/flowsint_types/alias.py b/flowsint-types/src/flowsint_types/alias.py
index a3ddf674..2757e63c 100644
--- a/flowsint-types/src/flowsint_types/alias.py
+++ b/flowsint-types/src/flowsint_types/alias.py
@@ -1,5 +1,6 @@
+from typing import List, Optional, Self
+
from pydantic import Field, model_validator
-from typing import Optional, List, Self
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -9,7 +10,12 @@ from .registry import flowsint_type
class Alias(FlowsintType):
"""Represents an alias or alternative name used by an entity."""
- alias: str = Field(..., description="Alias or alternative name", title="Alias", json_schema_extra={"primary": True})
+ alias: str = Field(
+ ...,
+ description="Alias or alternative name",
+ title="Alias",
+ json_schema_extra={"primary": True},
+ )
type: Optional[str] = Field(
None,
description="Type of alias (nickname, code name, etc.)",
@@ -48,7 +54,7 @@ class Alias(FlowsintType):
None, description="Geographic region where alias is used", title="Region"
)
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
self.nodeLabel = self.alias
return self
diff --git a/flowsint-types/src/flowsint_types/bank_account.py b/flowsint-types/src/flowsint_types/bank_account.py
index 82560217..0361b041 100644
--- a/flowsint-types/src/flowsint_types/bank_account.py
+++ b/flowsint-types/src/flowsint_types/bank_account.py
@@ -1,5 +1,6 @@
+from typing import List, Optional, Self
+
from pydantic import Field, model_validator
-from typing import Optional, List, Self
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -10,7 +11,10 @@ class BankAccount(FlowsintType):
"""Represents a bank account with financial and security information."""
account_number: str = Field(
- ..., description="Bank account number", title="Account Number", json_schema_extra={"primary": True}
+ ...,
+ description="Bank account number",
+ title="Account Number",
+ json_schema_extra={"primary": True},
)
bank_name: Optional[str] = Field(None, description="Bank name", title="Bank Name")
account_type: Optional[str] = Field(
@@ -66,12 +70,16 @@ class BankAccount(FlowsintType):
None, description="Source of breach if compromised", title="Breach Source"
)
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
parts = []
if self.bank_name:
parts.append(self.bank_name)
- parts.append(f"****{self.account_number[-4:]}" if len(self.account_number) > 4 else self.account_number)
+ parts.append(
+ f"****{self.account_number[-4:]}"
+ if len(self.account_number) > 4
+ else self.account_number
+ )
self.nodeLabel = " - ".join(parts)
return self
diff --git a/flowsint-types/src/flowsint_types/breach.py b/flowsint-types/src/flowsint_types/breach.py
index 1e41335c..15a4f699 100644
--- a/flowsint-types/src/flowsint_types/breach.py
+++ b/flowsint-types/src/flowsint_types/breach.py
@@ -1,5 +1,7 @@
-from typing import List, Optional, Dict, Self
+from typing import Dict, List, Optional, Self
+
from pydantic import Field, model_validator
+
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -9,7 +11,10 @@ class Breach(FlowsintType):
"""Represents a data breach incident with affected accounts and details."""
name: str = Field(
- ..., description="The name of the breach or service", title="Breach Name", json_schema_extra={"primary": True}
+ ...,
+ description="The name of the breach or service",
+ title="Breach Name",
+ json_schema_extra={"primary": True},
)
title: Optional[str] = Field(
None, description="Title of the breach", title="Breach Title"
@@ -70,7 +75,7 @@ class Breach(FlowsintType):
title="Full Breach Data",
)
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
# Use title if available, otherwise name
if self.title:
diff --git a/flowsint-types/src/flowsint_types/cidr.py b/flowsint-types/src/flowsint_types/cidr.py
index 1a9a2ebe..9faff663 100644
--- a/flowsint-types/src/flowsint_types/cidr.py
+++ b/flowsint-types/src/flowsint_types/cidr.py
@@ -1,6 +1,8 @@
-from pydantic import IPvAnyNetwork, Field, model_validator
-from typing import Self
import ipaddress
+from typing import Self
+
+from pydantic import Field, IPvAnyNetwork, model_validator
+
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -10,10 +12,13 @@ class CIDR(FlowsintType):
"""Represents a CIDR (Classless Inter-Domain Routing) network block."""
network: IPvAnyNetwork = Field(
- ..., description="CIDR block (e.g., 8.8.8.0/24)", title="Network Block", json_schema_extra={"primary": True}
+ ...,
+ description="CIDR block (e.g., 8.8.8.0/24)",
+ title="Network Block",
+ json_schema_extra={"primary": True},
)
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
self.nodeLabel = str(self.network)
return self
@@ -27,7 +32,7 @@ class CIDR(FlowsintType):
def detect(cls, line: str) -> bool:
"""Detect if a line of text contains a CIDR block."""
line = line.strip()
- if not line or '/' not in line:
+ if not line or "/" not in line:
return False
try:
diff --git a/flowsint-types/src/flowsint_types/credential.py b/flowsint-types/src/flowsint_types/credential.py
index 865b0ced..bfb6216a 100644
--- a/flowsint-types/src/flowsint_types/credential.py
+++ b/flowsint-types/src/flowsint_types/credential.py
@@ -1,5 +1,6 @@
+from typing import List, Optional, Self
+
from pydantic import Field, model_validator
-from typing import Optional, List, Self
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -9,7 +10,12 @@ from .registry import flowsint_type
class Credential(FlowsintType):
"""Represents user credentials with compromise and usage information."""
- username: str = Field(..., description="Username or identifier", title="Username", json_schema_extra={"primary": True})
+ username: str = Field(
+ ...,
+ description="Username or identifier",
+ title="Username",
+ json_schema_extra={"primary": True},
+ )
service: Optional[str] = Field(
None,
description="Service or platform where credential is used",
@@ -67,7 +73,7 @@ class Credential(FlowsintType):
None, description="Source of credential information", title="Source"
)
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
if self.service:
self.nodeLabel = f"{self.username}@{self.service}"
diff --git a/flowsint-types/src/flowsint_types/credit_card.py b/flowsint-types/src/flowsint_types/credit_card.py
index 33d1eafd..bcacb4cb 100644
--- a/flowsint-types/src/flowsint_types/credit_card.py
+++ b/flowsint-types/src/flowsint_types/credit_card.py
@@ -1,5 +1,6 @@
+from typing import List, Optional, Self
+
from pydantic import Field, model_validator
-from typing import Optional, List, Self
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -9,7 +10,12 @@ from .registry import flowsint_type
class CreditCard(FlowsintType):
"""Represents a credit card with financial details and security status."""
- card_number: str = Field(..., description="Credit card number", title="Card Number", json_schema_extra={"primary": True})
+ card_number: str = Field(
+ ...,
+ description="Credit card number",
+ title="Card Number",
+ json_schema_extra={"primary": True},
+ )
card_type: Optional[str] = Field(
None, description="Type of card (Visa, Mastercard, etc.)", title="Card Type"
)
@@ -61,18 +67,23 @@ class CreditCard(FlowsintType):
None, description="Last time card was used", title="Last Used"
)
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
parts = []
if self.card_type:
parts.append(self.card_type)
- parts.append(f"****{self.card_number[-4:]}" if len(self.card_number) > 4 else self.card_number)
+ parts.append(
+ f"****{self.card_number[-4:]}"
+ if len(self.card_number) > 4
+ else self.card_number
+ )
self.nodeLabel = " ".join(parts)
return self
@classmethod
def _luhn_check(cls, card_number: str) -> bool:
"""Validate a credit card number using the Luhn algorithm."""
+
def digits_of(n):
return [int(d) for d in str(n)]
@@ -92,7 +103,7 @@ class CreditCard(FlowsintType):
@classmethod
def detect(cls, line: str) -> bool:
"""Detect if a line of text contains a credit card number."""
- line = line.strip().replace(' ', '').replace('-', '')
+ line = line.strip().replace(" ", "").replace("-", "")
if not line or not line.isdigit():
return False
diff --git a/flowsint-types/src/flowsint_types/device.py b/flowsint-types/src/flowsint_types/device.py
index 87ca72f2..8c8109a2 100644
--- a/flowsint-types/src/flowsint_types/device.py
+++ b/flowsint-types/src/flowsint_types/device.py
@@ -1,5 +1,6 @@
+from typing import List, Optional, Self
+
from pydantic import Field, model_validator
-from typing import Optional, List, Self
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -10,7 +11,10 @@ class Device(FlowsintType):
"""Represents a device with hardware, software, and network information."""
device_id: str = Field(
- ..., description="Unique device identifier", title="Device ID", json_schema_extra={"primary": True}
+ ...,
+ description="Unique device identifier",
+ title="Device ID",
+ json_schema_extra={"primary": True},
)
type: Optional[str] = Field(
None,
@@ -68,7 +72,7 @@ class Device(FlowsintType):
None, description="Source of device information", title="Source"
)
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
parts = []
if self.manufacturer:
diff --git a/flowsint-types/src/flowsint_types/dns_record.py b/flowsint-types/src/flowsint_types/dns_record.py
index 5c0b5bc1..d18125f4 100644
--- a/flowsint-types/src/flowsint_types/dns_record.py
+++ b/flowsint-types/src/flowsint_types/dns_record.py
@@ -1,5 +1,6 @@
+from typing import List, Optional, Self
+
from pydantic import Field, model_validator
-from typing import Optional, List, Self
from .flowsint_base import FlowsintType
from .registry import flowsint_type
diff --git a/flowsint-types/src/flowsint_types/document.py b/flowsint-types/src/flowsint_types/document.py
index fb4c2b34..3b405e9d 100644
--- a/flowsint-types/src/flowsint_types/document.py
+++ b/flowsint-types/src/flowsint_types/document.py
@@ -1,5 +1,6 @@
+from typing import List, Optional, Self
+
from pydantic import Field, model_validator
-from typing import Optional, List, Self
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -9,7 +10,12 @@ from .registry import flowsint_type
class Document(FlowsintType):
"""Represents a document with metadata, security, and content information."""
- title: str = Field(..., description="Document title", title="Title", json_schema_extra={"primary": True})
+ title: str = Field(
+ ...,
+ description="Document title",
+ title="Title",
+ json_schema_extra={"primary": True},
+ )
doc_type: Optional[str] = Field(
None, description="Type of document (PDF, DOC, etc.)", title="Document Type"
)
@@ -61,7 +67,7 @@ class Document(FlowsintType):
None, description="Source of document information", title="Source"
)
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
self.nodeLabel = self.title
return self
diff --git a/flowsint-types/src/flowsint_types/domain.py b/flowsint-types/src/flowsint_types/domain.py
index d04643ef..6071873f 100644
--- a/flowsint-types/src/flowsint_types/domain.py
+++ b/flowsint-types/src/flowsint_types/domain.py
@@ -1,7 +1,9 @@
+import re
from typing import Optional, Self
-from pydantic import Field, field_validator, model_validator
from urllib.parse import urlparse
-import re
+
+from pydantic import Field, field_validator, model_validator
+
from .flowsint_base import FlowsintType
from .registry import flowsint_type
diff --git a/flowsint-types/src/flowsint_types/email.py b/flowsint-types/src/flowsint_types/email.py
index 77ac22ac..eaec30e2 100644
--- a/flowsint-types/src/flowsint_types/email.py
+++ b/flowsint-types/src/flowsint_types/email.py
@@ -1,6 +1,8 @@
-from pydantic import Field, EmailStr, model_validator
-from typing import Any, Self
import re
+from typing import Self
+
+from pydantic import EmailStr, Field, model_validator
+
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -9,9 +11,14 @@ from .registry import flowsint_type
class Email(FlowsintType):
"""Represents an email address."""
- email: EmailStr = Field(..., description="Email address", title="Email Address", json_schema_extra={"primary": True})
+ email: EmailStr = Field(
+ ...,
+ description="Email address",
+ title="Email Address",
+ json_schema_extra={"primary": True},
+ )
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
self.nodeLabel = self.email
return self
@@ -29,5 +36,5 @@ class Email(FlowsintType):
return False
# Email regex pattern (RFC 5322 simplified)
- email_pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
+ email_pattern = r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$"
return bool(re.match(email_pattern, line))
diff --git a/flowsint-types/src/flowsint_types/file.py b/flowsint-types/src/flowsint_types/file.py
index 39e5cd75..ff7a74ef 100644
--- a/flowsint-types/src/flowsint_types/file.py
+++ b/flowsint-types/src/flowsint_types/file.py
@@ -1,7 +1,7 @@
import re
+from typing import Optional, Self
from pydantic import Field, model_validator
-from typing import Optional, Self
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -27,7 +27,12 @@ def _hash_field_for(value: str) -> Optional[str]:
class File(FlowsintType):
"""Represents a file with metadata, type information, and security assessment."""
- filename: str = Field(..., description="File name", title="Filename", json_schema_extra={"primary": True})
+ filename: str = Field(
+ ...,
+ description="File name",
+ title="Filename",
+ json_schema_extra={"primary": True},
+ )
file_type: Optional[str] = Field(
None, description="File type or extension", title="File Type"
)
@@ -84,7 +89,7 @@ class File(FlowsintType):
None, description="Threat level assessment", title="Threat Level"
)
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
self.nodeLabel = self.filename
return self
diff --git a/flowsint-types/src/flowsint_types/gravatar.py b/flowsint-types/src/flowsint_types/gravatar.py
index 85860e8a..b54bedaf 100644
--- a/flowsint-types/src/flowsint_types/gravatar.py
+++ b/flowsint-types/src/flowsint_types/gravatar.py
@@ -1,5 +1,7 @@
+from typing import Optional, Self
+
from pydantic import Field, HttpUrl, model_validator
-from typing import Optional, List, Self
+
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -9,7 +11,12 @@ class Gravatar(FlowsintType):
"""Represents a Gravatar profile with image and user information."""
src: HttpUrl = Field(..., description="Gravatar image URL", title="Image URL")
- hash: str = Field(..., description="Gravatar hash", title="Hash", json_schema_extra={"primary": True})
+ hash: str = Field(
+ ...,
+ description="Gravatar hash",
+ title="Hash",
+ json_schema_extra={"primary": True},
+ )
size: Optional[int] = Field(
None, description="Image size in pixels", title="Image Size"
)
@@ -46,7 +53,7 @@ class Gravatar(FlowsintType):
None, description="Larger version of the image", title="Large Image URL"
)
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
# Use display name if available, otherwise hash
if self.display_name:
diff --git a/flowsint-types/src/flowsint_types/ip.py b/flowsint-types/src/flowsint_types/ip.py
index ef59928f..da05f5f6 100644
--- a/flowsint-types/src/flowsint_types/ip.py
+++ b/flowsint-types/src/flowsint_types/ip.py
@@ -1,6 +1,8 @@
-from pydantic import Field, field_validator, model_validator
-from typing import Optional, Any, Self
import ipaddress
+from typing import Optional, Self
+
+from pydantic import Field, field_validator, model_validator
+
from .flowsint_base import FlowsintType
from .registry import flowsint_type
diff --git a/flowsint-types/src/flowsint_types/leak.py b/flowsint-types/src/flowsint_types/leak.py
index 64583aca..48ace720 100644
--- a/flowsint-types/src/flowsint_types/leak.py
+++ b/flowsint-types/src/flowsint_types/leak.py
@@ -1,4 +1,5 @@
-from typing import List, Dict, Optional, Self
+from typing import Dict, List, Optional, Self
+
from pydantic import Field, model_validator
from .flowsint_base import FlowsintType
@@ -10,13 +11,16 @@ class Leak(FlowsintType):
"""Represents a data leak or breach with associated data."""
name: str = Field(
- ..., description="The name of the leak or service brea", title="Leak Name", json_schema_extra={"primary": True}
+ ...,
+ description="The name of the leak or service brea",
+ title="Leak Name",
+ json_schema_extra={"primary": True},
)
leak: Optional[List[Dict]] = Field(
None, description="List of data leaks found", title="Leak Data"
)
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
self.nodeLabel = self.name
return self
diff --git a/flowsint-types/src/flowsint_types/malware.py b/flowsint-types/src/flowsint_types/malware.py
index 2c6cf79e..b94bec6c 100644
--- a/flowsint-types/src/flowsint_types/malware.py
+++ b/flowsint-types/src/flowsint_types/malware.py
@@ -1,5 +1,6 @@
+from typing import List, Optional, Self
+
from pydantic import Field, model_validator
-from typing import Optional, List, Self
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -9,7 +10,12 @@ from .registry import flowsint_type
class Malware(FlowsintType):
"""Represents malware with family, capabilities, and threat intelligence."""
- name: str = Field(..., description="Malware name or identifier", title="Name", json_schema_extra={"primary": True})
+ name: str = Field(
+ ...,
+ description="Malware name or identifier",
+ title="Name",
+ json_schema_extra={"primary": True},
+ )
family: Optional[str] = Field(None, description="Malware family", title="Family")
type: Optional[str] = Field(
None, description="Type of malware (trojan, ransomware, etc.)", title="Type"
@@ -63,7 +69,7 @@ class Malware(FlowsintType):
None, description="Sample file hashes", title="Sample Hashes"
)
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
if self.family:
self.nodeLabel = f"{self.name} ({self.family})"
diff --git a/flowsint-types/src/flowsint_types/message.py b/flowsint-types/src/flowsint_types/message.py
index 084e0c7e..fc278ba2 100644
--- a/flowsint-types/src/flowsint_types/message.py
+++ b/flowsint-types/src/flowsint_types/message.py
@@ -1,5 +1,6 @@
+from typing import List, Optional, Self
+
from pydantic import Field, model_validator
-from typing import Optional, List, Self
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -10,7 +11,10 @@ class Message(FlowsintType):
"""Represents a message with content, metadata, and security analysis."""
message_id: str = Field(
- ..., description="Unique message identifier", title="Message ID", json_schema_extra={"primary": True}
+ ...,
+ description="Unique message identifier",
+ title="Message ID",
+ json_schema_extra={"primary": True},
)
content: str = Field(..., description="Message content", title="Content")
sender: Optional[str] = Field(None, description="Message sender", title="Sender")
@@ -65,13 +69,15 @@ class Message(FlowsintType):
None, description="Threat level assessment", title="Threat Level"
)
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
if self.subject:
self.nodeLabel = self.subject
else:
# Truncate content to first 50 characters
- content_preview = self.content[:50] + "..." if len(self.content) > 50 else self.content
+ content_preview = (
+ self.content[:50] + "..." if len(self.content) > 50 else self.content
+ )
self.nodeLabel = content_preview
return self
diff --git a/flowsint-types/src/flowsint_types/organization.py b/flowsint-types/src/flowsint_types/organization.py
index 1005c105..4480cdb6 100644
--- a/flowsint-types/src/flowsint_types/organization.py
+++ b/flowsint-types/src/flowsint_types/organization.py
@@ -1,8 +1,10 @@
+from typing import Any, List, Optional, Self
+
from pydantic import Field, model_validator
-from typing import Any, Optional, List, Self
-from .individual import Individual
+
from .address import Location
from .flowsint_base import FlowsintType
+from .individual import Individual
from .registry import flowsint_type
@@ -11,15 +13,21 @@ class Organization(FlowsintType):
"""Represents an organization with detailed business and administrative information."""
# Basic information
- name: Any = Field(..., description="Organization name", title="Organization Name", json_schema_extra={"primary": True})
+ name: Any = Field(
+ ...,
+ description="Organization name",
+ title="Organization Name",
+ json_schema_extra={"primary": True},
+ )
- @model_validator(mode='before')
+ @model_validator(mode="before")
@classmethod
def convert_string_to_dict(cls, data: Any) -> Any:
"""Allow creating Organization from a string directly."""
if isinstance(data, str):
- return {'name': data}
+ return {"name": data}
return data
+
siren: Optional[Any] = Field(None, description="SIREN number", title="SIREN Number")
nom_complet: Optional[Any] = Field(
None, description="Complete name", title="Complete Name"
@@ -373,7 +381,7 @@ class Organization(FlowsintType):
None, description="Complements SIAE type", title="SIAE Type"
)
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
# Use the full name if available, otherwise use name
if self.nom_complet:
diff --git a/flowsint-types/src/flowsint_types/phone.py b/flowsint-types/src/flowsint_types/phone.py
index 6790d9b3..3feb28e1 100644
--- a/flowsint-types/src/flowsint_types/phone.py
+++ b/flowsint-types/src/flowsint_types/phone.py
@@ -1,17 +1,24 @@
-from pydantic import Field, field_validator, model_validator
-from typing import Optional, Any, Self
+import ipaddress
+from typing import Any, Optional, Self
+
import phonenumbers
from phonenumbers import NumberParseException
+from pydantic import Field, field_validator, model_validator
+
from .flowsint_base import FlowsintType
from .registry import flowsint_type
-import ipaddress
@flowsint_type
class Phone(FlowsintType):
"""Represents a phone number with country and carrier information."""
- number: str = Field(..., description="Phone number", title="Phone Number", json_schema_extra={"primary": True})
+ number: str = Field(
+ ...,
+ description="Phone number",
+ title="Phone Number",
+ json_schema_extra={"primary": True},
+ )
country: Optional[str] = Field(
None, description="Country code (ISO 3166-1 alpha-2)", title="Country Code"
)
@@ -19,15 +26,15 @@ class Phone(FlowsintType):
None, description="Mobile carrier or service provider", title="Carrier"
)
- @model_validator(mode='before')
+ @model_validator(mode="before")
@classmethod
def convert_string_to_dict(cls, data: Any) -> Any:
"""Allow creating Phone from a string directly."""
if isinstance(data, str):
- return {'number': data}
+ return {"number": data}
return data
- @field_validator('number')
+ @field_validator("number")
@classmethod
def validate_phone_number(cls, v: str) -> str:
"""Validate phone number format using phonenumbers library.
@@ -37,29 +44,35 @@ class Phone(FlowsintType):
2. With common region codes (US, GB, FR, etc.)
"""
# If it looks like an international number (starts with +), try parsing without region
- if v.startswith('+'):
+ if v.startswith("+"):
try:
parsed = phonenumbers.parse(v, None)
if phonenumbers.is_valid_number(parsed):
- return phonenumbers.format_number(parsed, phonenumbers.PhoneNumberFormat.E164)
+ return phonenumbers.format_number(
+ parsed, phonenumbers.PhoneNumberFormat.E164
+ )
except NumberParseException:
pass
# Try parsing with common regions
- common_regions = ['US', 'GB', 'FR', 'DE', 'ES', 'IT', 'CA', 'AU', 'JP', 'CN']
+ common_regions = ["US", "GB", "FR", "DE", "ES", "IT", "CA", "AU", "JP", "CN"]
for region in common_regions:
try:
parsed = phonenumbers.parse(v, region)
if phonenumbers.is_valid_number(parsed):
# Return the number in international format
- return phonenumbers.format_number(parsed, phonenumbers.PhoneNumberFormat.E164)
+ return phonenumbers.format_number(
+ parsed, phonenumbers.PhoneNumberFormat.E164
+ )
except NumberParseException:
continue
# If all attempts fail, raise an error
- raise ValueError(f"Invalid phone number: {v}. Must be in international format (+...) or a valid format for common regions.")
+ raise ValueError(
+ f"Invalid phone number: {v}. Must be in international format (+...) or a valid format for common regions."
+ )
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
self.nodeLabel = self.number
return self
@@ -82,7 +95,7 @@ class Phone(FlowsintType):
except ValueError:
pass
# Try international format first (starts with +)
- if line.startswith('+'):
+ if line.startswith("+"):
try:
parsed = phonenumbers.parse(line, None)
return phonenumbers.is_valid_number(parsed)
@@ -90,7 +103,7 @@ class Phone(FlowsintType):
pass
# Try parsing with common regions
- common_regions = ['US', 'GB', 'FR', 'DE', 'ES', 'IT', 'CA', 'AU', 'JP', 'CN']
+ common_regions = ["US", "GB", "FR", "DE", "ES", "IT", "CA", "AU", "JP", "CN"]
for region in common_regions:
try:
parsed = phonenumbers.parse(line, region)
diff --git a/flowsint-types/src/flowsint_types/phrase.py b/flowsint-types/src/flowsint_types/phrase.py
index 4934f551..720478c6 100644
--- a/flowsint-types/src/flowsint_types/phrase.py
+++ b/flowsint-types/src/flowsint_types/phrase.py
@@ -1,6 +1,7 @@
-from pydantic import Field, model_validator
from typing import Any, Self
+from pydantic import Field, model_validator
+
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -10,10 +11,13 @@ class Phrase(FlowsintType):
"""Represents a phrase or text content."""
text: Any = Field(
- ..., description="The content of the phrase.", title="Phrase text value.", json_schema_extra={"primary": True}
+ ...,
+ description="The content of the phrase.",
+ title="Phrase text value.",
+ json_schema_extra={"primary": True},
)
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
text_str = str(self.text)
# Truncate to 100 characters for display
diff --git a/flowsint-types/src/flowsint_types/port.py b/flowsint-types/src/flowsint_types/port.py
index c79b2dcd..08b9366e 100644
--- a/flowsint-types/src/flowsint_types/port.py
+++ b/flowsint-types/src/flowsint_types/port.py
@@ -1,5 +1,7 @@
-from pydantic import Field, field_validator, model_validator
from typing import Optional, Self
+
+from pydantic import Field, field_validator, model_validator
+
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -8,7 +10,12 @@ from .registry import flowsint_type
class Port(FlowsintType):
"""Represents an open network port related to an IP address."""
- number: int = Field(..., description="Port number", title="Port Number", json_schema_extra={"primary": True})
+ number: int = Field(
+ ...,
+ description="Port number",
+ title="Port Number",
+ json_schema_extra={"primary": True},
+ )
protocol: Optional[str] = Field(
None, description="Protocol (TCP, UDP, etc.)", title="Protocol"
)
@@ -22,7 +29,7 @@ class Port(FlowsintType):
None, description="Service banner information", title="Banner"
)
- @field_validator('number')
+ @field_validator("number")
@classmethod
def validate_port_number(cls, v: int) -> int:
"""Validate that port number is in valid range (0-65535)."""
@@ -30,7 +37,7 @@ class Port(FlowsintType):
raise ValueError(f"Port number must be between 0 and 65535, got {v}")
return v
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
# Include service and protocol if available
parts = [str(self.number)]
diff --git a/flowsint-types/src/flowsint_types/reputation_score.py b/flowsint-types/src/flowsint_types/reputation_score.py
index e1aa4e51..b7ea744f 100644
--- a/flowsint-types/src/flowsint_types/reputation_score.py
+++ b/flowsint-types/src/flowsint_types/reputation_score.py
@@ -1,5 +1,6 @@
+from typing import List, Optional, Self
+
from pydantic import Field, model_validator
-from typing import Optional, List, Self
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -9,7 +10,12 @@ from .registry import flowsint_type
class ReputationScore(FlowsintType):
"""Represents a reputation score for an entity with historical data and trends."""
- entity_id: str = Field(..., description="Entity identifier", title="Entity ID", json_schema_extra={"primary": True})
+ entity_id: str = Field(
+ ...,
+ description="Entity identifier",
+ title="Entity ID",
+ json_schema_extra={"primary": True},
+ )
entity_type: Optional[str] = Field(
None,
description="Type of entity (domain, IP, email, etc.)",
@@ -59,7 +65,7 @@ class ReputationScore(FlowsintType):
None, description="Recommendations based on score", title="Recommendations"
)
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
parts = [self.entity_id]
if self.score is not None:
diff --git a/flowsint-types/src/flowsint_types/risk_profile.py b/flowsint-types/src/flowsint_types/risk_profile.py
index 0ebc0408..55d3739a 100644
--- a/flowsint-types/src/flowsint_types/risk_profile.py
+++ b/flowsint-types/src/flowsint_types/risk_profile.py
@@ -1,5 +1,6 @@
+from typing import List, Optional, Self
+
from pydantic import Field, model_validator
-from typing import Optional, List, Self
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -9,7 +10,12 @@ from .registry import flowsint_type
class RiskProfile(FlowsintType):
"""Represents a comprehensive risk assessment profile for an entity."""
- entity_id: str = Field(..., description="Entity identifier", title="Entity ID", json_schema_extra={"primary": True})
+ entity_id: str = Field(
+ ...,
+ description="Entity identifier",
+ title="Entity ID",
+ json_schema_extra={"primary": True},
+ )
entity_type: Optional[str] = Field(
None, description="Type of entity", title="Entity Type"
)
@@ -76,7 +82,7 @@ class RiskProfile(FlowsintType):
None, description="Next review date", title="Next Review Date"
)
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
parts = [self.entity_id]
if self.risk_level:
diff --git a/flowsint-types/src/flowsint_types/script.py b/flowsint-types/src/flowsint_types/script.py
index db131e05..1351f47b 100644
--- a/flowsint-types/src/flowsint_types/script.py
+++ b/flowsint-types/src/flowsint_types/script.py
@@ -1,5 +1,6 @@
+from typing import List, Optional, Self
+
from pydantic import Field, model_validator
-from typing import Optional, List, Self
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -10,7 +11,10 @@ class Script(FlowsintType):
"""Represents a script or code file with analysis and security information."""
script_id: str = Field(
- ..., description="Unique script identifier", title="Script ID", json_schema_extra={"primary": True}
+ ...,
+ description="Unique script identifier",
+ title="Script ID",
+ json_schema_extra={"primary": True},
)
name: Optional[str] = Field(None, description="Script name", title="Name")
language: Optional[str] = Field(
@@ -68,7 +72,7 @@ class Script(FlowsintType):
None, description="Whether script is minified", title="Minified"
)
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
if self.name:
self.nodeLabel = self.name
diff --git a/flowsint-types/src/flowsint_types/session.py b/flowsint-types/src/flowsint_types/session.py
index c8577a15..80a7a243 100644
--- a/flowsint-types/src/flowsint_types/session.py
+++ b/flowsint-types/src/flowsint_types/session.py
@@ -1,5 +1,6 @@
+from typing import List, Optional, Self
+
from pydantic import Field, model_validator
-from typing import Optional, List, Self
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -10,7 +11,10 @@ class Session(FlowsintType):
"""Represents a user session with device and activity information."""
session_id: str = Field(
- ..., description="Unique session identifier", title="Session ID", json_schema_extra={"primary": True}
+ ...,
+ description="Unique session identifier",
+ title="Session ID",
+ json_schema_extra={"primary": True},
)
user_id: Optional[str] = Field(None, description="User identifier", title="User ID")
service: Optional[str] = Field(
@@ -54,7 +58,7 @@ class Session(FlowsintType):
None, description="Source of session information", title="Source"
)
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
parts = []
if self.user_id:
diff --git a/flowsint-types/src/flowsint_types/ssl_certificate.py b/flowsint-types/src/flowsint_types/ssl_certificate.py
index 1aca7f80..181244f9 100644
--- a/flowsint-types/src/flowsint_types/ssl_certificate.py
+++ b/flowsint-types/src/flowsint_types/ssl_certificate.py
@@ -1,5 +1,6 @@
+from typing import List, Optional, Self
+
from pydantic import Field, model_validator
-from typing import Optional, List, Self
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -10,7 +11,10 @@ class SSLCertificate(FlowsintType):
"""Represents an SSL/TLS certificate with validation and security details."""
subject: str = Field(
- ..., description="Certificate subject (domain name)", title="Subject", json_schema_extra={"primary": True}
+ ...,
+ description="Certificate subject (domain name)",
+ title="Subject",
+ json_schema_extra={"primary": True},
)
issuer: Optional[str] = Field(
None,
@@ -74,7 +78,7 @@ class SSLCertificate(FlowsintType):
None, description="SHA256 fingerprint", title="SHA256 Fingerprint"
)
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
self.nodeLabel = self.subject
return self
diff --git a/flowsint-types/src/flowsint_types/weapon.py b/flowsint-types/src/flowsint_types/weapon.py
index 8dda830c..3cfa14a0 100644
--- a/flowsint-types/src/flowsint_types/weapon.py
+++ b/flowsint-types/src/flowsint_types/weapon.py
@@ -1,5 +1,6 @@
+from typing import List, Literal, Optional, Self
+
from pydantic import Field, model_validator
-from typing import Literal, Optional, List, Self
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -9,7 +10,12 @@ from .registry import flowsint_type
class Weapon(FlowsintType):
"""Represents a weapon with detailed specifications and forensic information."""
- name: str = Field(..., description="Weapon name or identifier", title="Name", json_schema_extra={"primary": True})
+ name: str = Field(
+ ...,
+ description="Weapon name or identifier",
+ title="Name",
+ json_schema_extra={"primary": True},
+ )
type: Optional[
Literal[
"firearm",
@@ -117,7 +123,7 @@ class Weapon(FlowsintType):
None, description="Additional notes or observations", title="Notes"
)
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
parts = [self.name]
if self.type:
diff --git a/flowsint-types/src/flowsint_types/website.py b/flowsint-types/src/flowsint_types/website.py
index 3b1a82c6..ef2ac2a5 100644
--- a/flowsint-types/src/flowsint_types/website.py
+++ b/flowsint-types/src/flowsint_types/website.py
@@ -1,6 +1,8 @@
+import re
from typing import Dict, List, Optional, Self
+
from pydantic import Field, HttpUrl, model_validator
-import re
+
from .domain import Domain
from .flowsint_base import FlowsintType
from .registry import flowsint_type
@@ -35,7 +37,11 @@ class Website(FlowsintType):
None, description="Text content of the page", title="Content"
)
status_code: Optional[int] = Field(
- None, description="HTTP status code", title="Status Code", ge=100, le=599 # Enforce status code
+ None,
+ description="HTTP status code",
+ title="Status Code",
+ ge=100,
+ le=599, # Enforce status code
)
headers: Optional[Dict[str, str]] = Field(
None, description="Relevant HTTP headers", title="Headers"
diff --git a/flowsint-types/src/flowsint_types/whois.py b/flowsint-types/src/flowsint_types/whois.py
index 6b50d47d..e31bbd69 100644
--- a/flowsint-types/src/flowsint_types/whois.py
+++ b/flowsint-types/src/flowsint_types/whois.py
@@ -1,9 +1,11 @@
-from typing import Optional, Union, Self
+from typing import Optional, Self, Union
+
from pydantic import Field, field_validator, model_validator
-from .email import Email
+
from .domain import Domain
-from .organization import Organization
+from .email import Email
from .flowsint_base import FlowsintType
+from .organization import Organization
from .registry import flowsint_type
@@ -11,9 +13,16 @@ from .registry import flowsint_type
class Whois(FlowsintType):
"""Represents WHOIS domain registration information."""
- domain: Domain = Field(..., description="Domain information", title="Domain", json_schema_extra={"primary": True})
+ domain: Domain = Field(
+ ...,
+ description="Domain information",
+ title="Domain",
+ json_schema_extra={"primary": True},
+ )
registry_domain_id: Optional[str] = Field(
- None, description="Registry Domain ID (unique identifier)", title="Registry Domain ID"
+ None,
+ description="Registry Domain ID (unique identifier)",
+ title="Registry Domain ID",
)
registrar: Optional[str] = Field(
None, description="Domain registrar name", title="Registrar"
@@ -39,7 +48,7 @@ class Whois(FlowsintType):
None, description="Date when the domain expires", title="Expiration Date"
)
- @field_validator('domain', mode='before')
+ @field_validator("domain", mode="before")
@classmethod
def convert_domain(cls, v: Union[str, dict, Domain]) -> Domain:
"""Convert string or dict to Domain object if needed."""
@@ -51,9 +60,11 @@ class Whois(FlowsintType):
return Domain(**v)
return v
- @field_validator('organization', mode='before')
+ @field_validator("organization", mode="before")
@classmethod
- def convert_organization(cls, v: Union[str, dict, Organization, None]) -> Optional[Organization]:
+ def convert_organization(
+ cls, v: Union[str, dict, Organization, None]
+ ) -> Optional[Organization]:
"""Convert string or dict to Organization object if needed."""
if v is None:
return None
@@ -65,7 +76,7 @@ class Whois(FlowsintType):
return Organization(**v)
return v
- @model_validator(mode='after')
+ @model_validator(mode="after")
def compute_label(self) -> Self:
# Use domain and organization if available
if self.organization:
diff --git a/flowsint-types/tests/test_domain.py b/flowsint-types/tests/test_domain.py
index ecbaf8de..da640eed 100644
--- a/flowsint-types/tests/test_domain.py
+++ b/flowsint-types/tests/test_domain.py
@@ -1,5 +1,4 @@
import pytest
-
from flowsint_types.domain import Domain
diff --git a/flowsint-types/tests/test_status_code.py b/flowsint-types/tests/test_status_code.py
index 3f39c387..20318627 100644
--- a/flowsint-types/tests/test_status_code.py
+++ b/flowsint-types/tests/test_status_code.py
@@ -1,5 +1,5 @@
-from flowsint_types.website import Website
import pytest
+from flowsint_types.website import Website
def test_website_valid_status_code():