Skip to content

A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.

feat: python formatter finally working

dextmorgn7620a6d145ac
diff --git a/flowsint-api/alembic/env.py b/flowsint-api/alembic/env.py
index 2280ed23..a9734b5b 100644
--- a/flowsint-api/alembic/env.py
+++ b/flowsint-api/alembic/env.py
@@ -10,10 +10,9 @@ from dotenv import load_dotenv
load_dotenv()
# Permet les imports comme from app.models.base import Base
-sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
# Importer Base et toutes les tables pour qu'Alembic les détecte
-from app.models.base import Base
from app.models.models import * # noqa
# Configuration Alembic
@@ -30,6 +29,7 @@ config.set_main_option("sqlalchemy.url", database_url)
# Définir la métadonnée cible pour autogenerate
target_metadata = Base.metadata
+
def run_migrations_offline() -> None:
"""Run migrations in 'offline' mode."""
url = config.get_main_option("sqlalchemy.url")
@@ -42,6 +42,7 @@ def run_migrations_offline() -> None:
with context.begin_transaction():
context.run_migrations()
+
def run_migrations_online() -> None:
"""Run migrations in 'online' mode."""
connectable = engine_from_config(
@@ -55,6 +56,7 @@ def run_migrations_online() -> None:
with context.begin_transaction():
context.run_migrations()
+
if context.is_offline_mode():
run_migrations_offline()
else:
diff --git a/flowsint-api/app/api/deps.py b/flowsint-api/app/api/deps.py
index d33ac39c..bf4d8fa6 100644
--- a/flowsint-api/app/api/deps.py
+++ b/flowsint-api/app/api/deps.py
@@ -8,6 +8,7 @@ from app.models.models import Profile
import os
from dotenv import load_dotenv
+
# Remplace avec ton URL de BDD
AUTH_SECRET = os.getenv("AUTH_SECRET")
@@ -15,7 +16,10 @@ load_dotenv()
oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")
-def get_current_user(token: str = Depends(oauth2_scheme), db: Session = Depends(get_db)) -> Profile:
+
+def get_current_user(
+ token: str = Depends(oauth2_scheme), db: Session = Depends(get_db)
+) -> Profile:
credentials_exception = HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Could not validate credentials",
diff --git a/flowsint-api/app/api/routes/analysis.py b/flowsint-api/app/api/routes/analysis.py
index bebc31da..643632fa 100644
--- a/flowsint-api/app/api/routes/analysis.py
+++ b/flowsint-api/app/api/routes/analysis.py
@@ -10,15 +10,25 @@ from app.api.schemas.analysis import AnalysisRead, AnalysisCreate, AnalysisUpdat
router = APIRouter()
+
# Get the list of all analyses for the current user
@router.get("", response_model=List[AnalysisRead])
-def get_analyses(db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
+def get_analyses(
+ db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)
+):
analyses = db.query(Analysis).filter(Analysis.owner_id == current_user.id).all()
return analyses
+
# Create a new analysis
[email protected]("/create", response_model=AnalysisRead, status_code=status.HTTP_201_CREATED)
-def create_analysis(payload: AnalysisCreate, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
+ "/create", response_model=AnalysisRead, status_code=status.HTTP_201_CREATED
+)
+def create_analysis(
+ payload: AnalysisCreate,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
new_analysis = Analysis(
id=uuid4(),
title=payload.title,
@@ -34,31 +44,55 @@ def create_analysis(payload: AnalysisCreate, db: Session = Depends(get_db), curr
db.refresh(new_analysis)
return new_analysis
+
# Get an analysis by ID
@router.get("/{analysis_id}", response_model=AnalysisRead)
-def get_analysis_by_id(analysis_id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
- analysis = db.query(Analysis).filter(Analysis.id == analysis_id, Analysis.owner_id == current_user.id).first()
+def get_analysis_by_id(
+ analysis_id: UUID,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
+ analysis = (
+ db.query(Analysis)
+ .filter(Analysis.id == analysis_id, Analysis.owner_id == current_user.id)
+ .first()
+ )
if not analysis:
raise HTTPException(status_code=404, detail="Analysis not found")
return analysis
+
# Get analyses by investigation ID
@router.get("/investigation/{investigation_id}", response_model=List[AnalysisRead])
def get_analyses_by_investigation(
investigation_id: UUID,
db: Session = Depends(get_db),
- current_user: Profile = Depends(get_current_user)
+ current_user: Profile = Depends(get_current_user),
):
- analyses = db.query(Analysis).filter(
- Analysis.investigation_id == investigation_id,
- Analysis.owner_id == current_user.id
- ).all()
+ analyses = (
+ db.query(Analysis)
+ .filter(
+ Analysis.investigation_id == investigation_id,
+ Analysis.owner_id == current_user.id,
+ )
+ .all()
+ )
return analyses
+
# Update an analysis by ID
@router.put("/{analysis_id}", response_model=AnalysisRead)
-def update_analysis(analysis_id: UUID, payload: AnalysisUpdate, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
- analysis = db.query(Analysis).filter(Analysis.id == analysis_id, Analysis.owner_id == current_user.id).first()
+def update_analysis(
+ analysis_id: UUID,
+ payload: AnalysisUpdate,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
+ analysis = (
+ db.query(Analysis)
+ .filter(Analysis.id == analysis_id, Analysis.owner_id == current_user.id)
+ .first()
+ )
if not analysis:
raise HTTPException(status_code=404, detail="Analysis not found")
if payload.title is not None:
@@ -74,12 +108,21 @@ def update_analysis(analysis_id: UUID, payload: AnalysisUpdate, db: Session = De
db.refresh(analysis)
return analysis
+
# Delete an analysis by ID
@router.delete("/{analysis_id}", status_code=status.HTTP_204_NO_CONTENT)
-def delete_analysis(analysis_id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
- analysis = db.query(Analysis).filter(Analysis.id == analysis_id, Analysis.owner_id == current_user.id).first()
+def delete_analysis(
+ analysis_id: UUID,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
+ analysis = (
+ db.query(Analysis)
+ .filter(Analysis.id == analysis_id, Analysis.owner_id == current_user.id)
+ .first()
+ )
if not analysis:
raise HTTPException(status_code=404, detail="Analysis not found")
db.delete(analysis)
db.commit()
- return None
+ return None
diff --git a/flowsint-api/app/api/routes/auth.py b/flowsint-api/app/api/routes/auth.py
index 6a87ff76..3e8c7164 100644
--- a/flowsint-api/app/api/routes/auth.py
+++ b/flowsint-api/app/api/routes/auth.py
@@ -10,14 +10,18 @@ from flowsint_core.core.postgre_db import get_db
router = APIRouter()
+
@router.post("/token")
-def login_for_access_token(form_data: OAuth2PasswordRequestForm = Depends(), db: Session = Depends(get_db)):
+def login_for_access_token(
+ form_data: OAuth2PasswordRequestForm = Depends(), db: Session = Depends(get_db)
+):
user = db.query(Profile).filter(Profile.email == form_data.username).first()
if not user or not verify_password(form_data.password, user.hashed_password):
raise HTTPException(status_code=400, detail="Incorrect email or password")
access_token = create_access_token(data={"sub": user.email})
return {"access_token": access_token, "user_id": user.id, "token_type": "bearer"}
+
@router.post("/register", status_code=201)
def register(user: ProfileCreate, db: Session = Depends(get_db)):
print(user)
@@ -30,6 +34,5 @@ def register(user: ProfileCreate, db: Session = Depends(get_db)):
db.add(new_user)
db.commit()
db.refresh(new_user)
-
return {"message": "User registered successfully", "email": new_user.email}
diff --git a/flowsint-api/app/api/routes/chat.py b/flowsint-api/app/api/routes/chat.py
index 0b93d9f2..b7fd1cb8 100644
--- a/flowsint-api/app/api/routes/chat.py
+++ b/flowsint-api/app/api/routes/chat.py
@@ -14,6 +14,7 @@ from flowsint_core.core.postgre_db import get_db
from app.models.models import Chat, ChatMessage, Profile
from app.api.deps import get_current_user
from app.api.schemas.chat import ChatCreate, ChatRead
+
router = APIRouter()
@@ -25,22 +26,23 @@ def clean_context(context: List[Dict]) -> List[Dict]:
if isinstance(item, dict):
# Create a copy and remove unwanted keys
cleaned_item = item["data"].copy()
-
+
# Remove top-level keys
- cleaned_item.pop('id', None)
- cleaned_item.pop('sketch_id', None)
-
+ cleaned_item.pop("id", None)
+ cleaned_item.pop("sketch_id", None)
+
# Remove from data if it exists
- if 'data' in cleaned_item and isinstance(cleaned_item['data'], dict):
- cleaned_item['data'].pop('sketch_id', None)
-
+ if "data" in cleaned_item and isinstance(cleaned_item["data"], dict):
+ cleaned_item["data"].pop("sketch_id", None)
+
# Remove measured/dimensions
- cleaned_item.pop('measured', None)
-
+ cleaned_item.pop("measured", None)
+
cleaned.append(cleaned_item)
print(cleaned)
return cleaned
+
class ChatRequest(BaseModel):
prompt: str
context: Optional[List[Dict]] = None
@@ -49,17 +51,14 @@ class ChatRequest(BaseModel):
# Get all chats
@router.get("/", response_model=List[ChatRead])
def get_chats(
- db: Session = Depends(get_db),
- current_user: Profile = Depends(get_current_user)
+ db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)
):
- chats = db.query(Chat).filter(
- Chat.owner_id == current_user.id
- ).all()
-
+ chats = db.query(Chat).filter(Chat.owner_id == current_user.id).all()
+
# Sort messages for each chat by created_at in ascending order
for chat in chats:
chat.messages.sort(key=lambda x: x.created_at)
-
+
return chats
@@ -69,7 +68,7 @@ def get_chats(
# for chat in chats:
# db.delete(chat)
# db.commit()
-# return {"result": "done"}
+# return {"result": "done"}
# Get analyses by investigation ID
@@ -77,26 +76,39 @@ def get_chats(
def get_chats_by_investigation(
investigation_id: UUID,
db: Session = Depends(get_db),
- current_user: Profile = Depends(get_current_user)
+ current_user: Profile = Depends(get_current_user),
):
- chats = db.query(Chat).filter(
- Chat.investigation_id == investigation_id,
- Chat.owner_id == current_user.id
- ).all()
-
+ chats = (
+ db.query(Chat)
+ .filter(
+ Chat.investigation_id == investigation_id, Chat.owner_id == current_user.id
+ )
+ .all()
+ )
+
# Sort messages for each chat by created_at in ascending order
for chat in chats:
chat.messages.sort(key=lambda x: x.created_at)
-
+
return chats
+
@router.post("/stream/{chat_id}")
-async def stream_chat(chat_id: UUID, payload: ChatRequest, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
+async def stream_chat(
+ chat_id: UUID,
+ payload: ChatRequest,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
# Check if Chat exists
- chat = db.query(Chat).filter(Chat.id == chat_id, Chat.owner_id == current_user.id).first()
+ chat = (
+ db.query(Chat)
+ .filter(Chat.id == chat_id, Chat.owner_id == current_user.id)
+ .first()
+ )
if not chat:
raise HTTPException(status_code=404, detail="Chat not found")
-
+
# Update chat's last_updated_at
chat.last_updated_at = datetime.utcnow()
db.commit()
@@ -112,19 +124,25 @@ async def stream_chat(chat_id: UUID, payload: ChatRequest, db: Session = Depends
db.add(user_message)
db.commit()
db.refresh(user_message)
-
+
try:
api_key = os.environ.get("MISTRAL_API_KEY")
if not api_key:
- raise HTTPException(status_code=500, detail="Mistral API key not configured")
-
+ raise HTTPException(
+ status_code=500, detail="Mistral API key not configured"
+ )
+
client = Mistral(api_key=api_key)
model = "mistral-small-latest"
accumulated_content = []
context_message = None
# Convert database messages to Mistral format
- messages = [SystemMessage(content="You are a CTI/OSINT investigator and you are trying to investigate on a variety of real life cases. Use your knowledge and analytics capabilities to analyse the context and answer the question the best you can. If you need to reference some items (an IP, a domain or something particular) please use the code brackets, like : `12.23.34.54` to reference it.")]
-
+ messages = [
+ SystemMessage(
+ content="You are a CTI/OSINT investigator and you are trying to investigate on a variety of real life cases. Use your knowledge and analytics capabilities to analyse the context and answer the question the best you can. If you need to reference some items (an IP, a domain or something particular) please use the code brackets, like : `12.23.34.54` to reference it."
+ )
+ ]
+
# Add context as a single system message if provided
if payload.context:
try:
@@ -139,16 +157,22 @@ async def stream_chat(chat_id: UUID, payload: ChatRequest, db: Session = Depends
except Exception as e:
# If context processing fails, skip it
print(f"Context processing error: {e}")
-
+
# Sort messages by created_at in ascending order and get recent messages
sorted_messages = sorted(chat.messages, key=lambda x: x.created_at)
- recent_messages = sorted_messages[-5:] if len(sorted_messages) > 5 else sorted_messages
+ recent_messages = (
+ sorted_messages[-5:] if len(sorted_messages) > 5 else sorted_messages
+ )
for message in recent_messages:
if message.is_bot:
- messages.append(AssistantMessage(content=json.dumps(message.content, default=str)))
+ messages.append(
+ AssistantMessage(content=json.dumps(message.content, default=str))
+ )
else:
- messages.append(UserMessage(content=json.dumps(message.content, default=str)))
-
+ messages.append(
+ UserMessage(content=json.dumps(message.content, default=str))
+ )
+
# Add the current context
if context_message:
messages.append(SystemMessage(content=context_message))
@@ -156,17 +180,14 @@ async def stream_chat(chat_id: UUID, payload: ChatRequest, db: Session = Depends
messages.append(UserMessage(content=payload.prompt))
async def generate():
- response = await client.chat.stream_async(
- model=model,
- messages=messages
- )
-
+ response = await client.chat.stream_async(model=model, messages=messages)
+
async for chunk in response:
if chunk.data.choices[0].delta.content is not None:
content_chunk = chunk.data.choices[0].delta.content
accumulated_content.append(content_chunk)
yield f"data: {json.dumps({'content': content_chunk})}\n\n"
-
+
# Create the bot message after all chunks have been processed
chat_message = ChatMessage(
id=uuid4(),
@@ -175,24 +196,25 @@ async def stream_chat(chat_id: UUID, payload: ChatRequest, db: Session = Depends
is_bot=True,
created_at=datetime.utcnow(),
)
-
+
db.add(chat_message)
db.commit()
db.refresh(chat_message)
-
+
yield "data: [DONE]\n\n"
- return StreamingResponse(
- generate(),
- media_type="text/event-stream"
- )
+ return StreamingResponse(generate(), media_type="text/event-stream")
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
-
-
+
+
# Create a new chat
@router.post("/create", response_model=ChatRead, status_code=status.HTTP_201_CREATED)
-def create_chat(payload: ChatCreate, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
+def create_chat(
+ payload: ChatCreate,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
new_chat = Chat(
id=uuid4(),
title=payload.title,
@@ -207,29 +229,42 @@ def create_chat(payload: ChatCreate, db: Session = Depends(get_db), current_user
db.refresh(new_chat)
return new_chat
+
# Get a chat by ID
@router.get("/{chat_id}", response_model=ChatRead)
-def get_chat_by_id(chat_id: UUID, db: Session = Depends(get_db),
- current_user: Profile = Depends(get_current_user)
- ):
- chat = db.query(Chat).filter(Chat.id == chat_id,
- Chat.owner_id == current_user.id
- ).first()
+def get_chat_by_id(
+ chat_id: UUID,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
+ chat = (
+ db.query(Chat)
+ .filter(Chat.id == chat_id, Chat.owner_id == current_user.id)
+ .first()
+ )
if not chat:
raise HTTPException(status_code=404, detail="Chat not found")
-
+
# Sort messages by created_at in ascending order
chat.messages.sort(key=lambda x: x.created_at)
-
+
return chat
# Delete an chat by ID
@router.delete("/{chat_id}", status_code=status.HTTP_204_NO_CONTENT)
-def delete_chat(chat_id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
- chat = db.query(Chat).filter(Chat.id == chat_id, Chat.owner_id == current_user.id).first()
+def delete_chat(
+ chat_id: UUID,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
+ chat = (
+ db.query(Chat)
+ .filter(Chat.id == chat_id, Chat.owner_id == current_user.id)
+ .first()
+ )
if not chat:
raise HTTPException(status_code=404, detail="Chat not found")
db.delete(chat)
db.commit()
- return None
+ return None
diff --git a/flowsint-api/app/api/routes/events.py b/flowsint-api/app/api/routes/events.py
index d56187e9..92b3f260 100644
--- a/flowsint-api/app/api/routes/events.py
+++ b/flowsint-api/app/api/routes/events.py
@@ -13,6 +13,7 @@ from datetime import datetime, timedelta
router = APIRouter()
+
@router.get("/sketch/{sketch_id}/logs")
def get_logs_by_sketch(
sketch_id: str,
@@ -20,27 +21,33 @@ def get_logs_by_sketch(
since: datetime | None = None,
db: Session = Depends(get_db),
# current_user: Profile = Depends(get_current_user)
-):
+):
"""Get historical logs for a specific sketch with optional filtering"""
# Check if sketch exists
sketch = db.query(Sketch).filter(Sketch.id == sketch_id).first()
if not sketch:
- raise HTTPException(status_code=404, detail=f"Sketch with id {sketch_id} not found")
+ raise HTTPException(
+ status_code=404, detail=f"Sketch with id {sketch_id} not found"
+ )
+
+ print(
+ f"[EventEmitter] Fetching logs for sketch {sketch_id} (limit: {limit}, since: {since})"
+ )
+ query = (
+ db.query(Log).filter(Log.sketch_id == sketch_id).order_by(Log.created_at.desc())
+ )
- print(f"[EventEmitter] Fetching logs for sketch {sketch_id} (limit: {limit}, since: {since})")
- query = db.query(Log).filter(Log.sketch_id == sketch_id).order_by(Log.created_at.desc())
-
if since:
query = query.filter(Log.created_at > since)
else:
# Default to last 24 hours if no since parameter
query = query.filter(Log.created_at > datetime.utcnow() - timedelta(days=1))
-
+
logs = query.limit(limit).all()
-
+
# Reverse to show chronologically (oldest to newest)
logs = list(reversed(logs))
-
+
results = []
for log in logs:
# Ensure payload is always a dictionary
@@ -53,53 +60,58 @@ def get_logs_by_sketch(
else:
# Handle other types by converting to string and wrapping
payload = {"content": str(log.content)}
-
- results.append(Event(
- id=str(log.id),
- sketch_id=str(log.sketch_id) if log.sketch_id else None,
- type=log.type,
- payload=payload
- ))
-
+
+ results.append(
+ Event(
+ id=str(log.id),
+ sketch_id=str(log.sketch_id) if log.sketch_id else None,
+ type=log.type,
+ payload=payload,
+ )
+ )
return results
@router.get("/sketch/{sketch_id}/stream")
-async def stream_events(request: Request, sketch_id: str,
- db: Session = Depends(get_db),
- # current_user: Profile = Depends(get_current_user)
- ):
+async def stream_events(
+ request: Request,
+ sketch_id: str,
+ db: Session = Depends(get_db),
+ # current_user: Profile = Depends(get_current_user)
+):
"""Stream events for a specific scan in real-time"""
# Check if sketch exists
sketch = db.query(Sketch).filter(Sketch.id == sketch_id).first()
if not sketch:
- raise HTTPException(status_code=404, detail=f"Sketch with id {sketch_id} not found")
+ raise HTTPException(
+ status_code=404, detail=f"Sketch with id {sketch_id} not found"
+ )
async def event_generator():
channel = sketch_id
await event_emitter.subscribe(channel)
try:
# Initial connection message
- yield "data: {\"event\": \"connected\", \"data\": \"Connected to log stream\"}\n\n"
+ yield 'data: {"event": "connected", "data": "Connected to log stream"}\n\n'
while True:
if await request.is_disconnected():
break
-
+
data = await event_emitter.get_message(channel)
if data is None:
- await asyncio.sleep(.1) # avoid tight loop on None
+ await asyncio.sleep(0.1) # avoid tight loop on None
continue
-
+
# Handle different types of events
- if isinstance(data, dict) and data.get('type') == 'scanner_complete':
+ if isinstance(data, dict) and data.get("type") == "scanner_complete":
# Send scanner completion event
- yield json.dumps({'event': 'scanner_complete', 'data': data})
+ yield json.dumps({"event": "scanner_complete", "data": data})
else:
# Send regular log event
- yield json.dumps({'event': 'log', 'data': data})
- await asyncio.sleep(.1)
+ yield json.dumps({"event": "log", "data": data})
+ await asyncio.sleep(0.1)
except asyncio.CancelledError:
print(f"[EventEmitter] Client disconnected from sketch_id: {sketch_id}")
@@ -114,16 +126,17 @@ async def stream_events(request: Request, sketch_id: str,
headers={
"Cache-Control": "no-cache",
"Connection": "keep-alive",
- "X-Accel-Buffering": "no"
- }
+ "X-Accel-Buffering": "no",
+ },
)
+
@router.delete("/sketch/{sketch_id}/logs")
def delete_scan_logs(
- sketch_id:str,
+ sketch_id: str,
db: Session = Depends(get_db),
- current_user: Profile = Depends(get_current_user)
- ):
+ current_user: Profile = Depends(get_current_user),
+):
"""Delete all logs for a specific scan"""
try:
db.query(Log).filter(Log.sketch_id == sketch_id).delete()
@@ -133,9 +146,9 @@ def delete_scan_logs(
db.rollback()
raise HTTPException(status_code=500, detail=f"Failed to delete logs: {str(e)}")
+
@router.get("/status/scan/{scan_id}/stream")
-async def stream_status(request: Request, scan_id: str,
- db: Session = Depends(get_db)):
+async def stream_status(request: Request, scan_id: str, db: Session = Depends(get_db)):
"""Stream status updates for a specific scan in real-time"""
async def status_generator():
@@ -143,7 +156,7 @@ async def stream_status(request: Request, scan_id: str,
await event_emitter.subscribe(f"scan_{scan_id}_status")
try:
# Initial connection message
- yield "data: {\"event\": \"connected\", \"data\": \"Connected to status stream\"}\n\n"
+ yield 'data: {"event": "connected", "data": "Connected to status stream"}\n\n'
while True:
data = await event_emitter.get_message(f"scan_{scan_id}_status")
@@ -154,7 +167,9 @@ async def stream_status(request: Request, scan_id: str,
yield f"data: {data}\n\n"
except asyncio.CancelledError:
- print(f"[EventEmitter] Client disconnected from status stream for scan_id: {scan_id}")
+ print(
+ f"[EventEmitter] Client disconnected from status stream for scan_id: {scan_id}"
+ )
finally:
await event_emitter.unsubscribe(f"scan_{scan_id}_status")
@@ -164,6 +179,6 @@ async def stream_status(request: Request, scan_id: str,
headers={
"Cache-Control": "no-cache",
"Connection": "keep-alive",
- "X-Accel-Buffering": "no"
- }
- )
\ No newline at end of file
+ "X-Accel-Buffering": "no",
+ },
+ )
diff --git a/flowsint-api/app/api/routes/investigations.py b/flowsint-api/app/api/routes/investigations.py
index 496e243b..8661e489 100644
--- a/flowsint-api/app/api/routes/investigations.py
+++ b/flowsint-api/app/api/routes/investigations.py
@@ -6,21 +6,37 @@ from sqlalchemy.orm import Session, selectinload
from flowsint_core.core.postgre_db import get_db
from app.models.models import Analysis, Investigation, Profile, Sketch
from app.api.deps import get_current_user
-from app.api.schemas.investigation import InvestigationRead, InvestigationCreate, InvestigationUpdate
+from app.api.schemas.investigation import (
+ InvestigationRead,
+ InvestigationCreate,
+ InvestigationUpdate,
+)
from app.api.schemas.sketch import SketchRead
from flowsint_core.core.graph_db import neo4j_connection
router = APIRouter()
+
# Get the list of all investigations
@router.get("", response_model=List[InvestigationRead])
-def get_investigations(db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
- investigations = db.query(Investigation).filter(Investigation.owner_id == current_user.id).all()
+def get_investigations(
+ db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)
+):
+ investigations = (
+ db.query(Investigation).filter(Investigation.owner_id == current_user.id).all()
+ )
return investigations
+
# Create a new investigation
[email protected]("/create", response_model=InvestigationRead, status_code=status.HTTP_201_CREATED)
-def create_investigation(payload: InvestigationCreate, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
+ "/create", response_model=InvestigationRead, status_code=status.HTTP_201_CREATED
+)
+def create_investigation(
+ payload: InvestigationCreate,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
new_investigation = Investigation(
id=uuid4(),
name=payload.name,
@@ -35,33 +51,57 @@ def create_investigation(payload: InvestigationCreate, db: Session = Depends(get
db.refresh(new_investigation)
return new_investigation
+
# Get a investigation by ID
@router.get("/{investigation_id}", response_model=InvestigationRead)
-def get_investigation_by_id(investigation_id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
- investigation = db.query(Investigation).options(selectinload(Investigation.sketches)).filter(Investigation.id == investigation_id).filter(Investigation.owner_id == current_user.id).first()
+def get_investigation_by_id(
+ investigation_id: UUID,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
+ investigation = (
+ db.query(Investigation)
+ .options(selectinload(Investigation.sketches))
+ .filter(Investigation.id == investigation_id)
+ .filter(Investigation.owner_id == current_user.id)
+ .first()
+ )
if not investigation:
raise HTTPException(status_code=404, detail="Investigation not found")
return investigation
+
# Get a investigation by ID
@router.get("/{investigation_id}/sketches", response_model=List[SketchRead])
def get_sketches_by_investigation(
investigation_id: UUID,
db: Session = Depends(get_db),
- current_user: Profile = Depends(get_current_user)
+ current_user: Profile = Depends(get_current_user),
):
- sketches = db.query(Sketch).filter(Sketch.investigation_id == investigation_id).all()
+ sketches = (
+ db.query(Sketch).filter(Sketch.investigation_id == investigation_id).all()
+ )
if not sketches:
- raise HTTPException(status_code=404, detail="No sketches found for this investigation")
+ raise HTTPException(
+ status_code=404, detail="No sketches found for this investigation"
+ )
return sketches
+
# Update a investigation by ID
@router.put("/{investigation_id}", response_model=InvestigationRead)
-def update_investigation(investigation_id: UUID, payload: InvestigationUpdate, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
- investigation = db.query(Investigation).filter(Investigation.id == investigation_id).first()
+def update_investigation(
+ investigation_id: UUID,
+ payload: InvestigationUpdate,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
+ investigation = (
+ db.query(Investigation).filter(Investigation.id == investigation_id).first()
+ )
if not investigation:
raise HTTPException(status_code=404, detail="Investigation not found")
-
+
investigation.name = payload.name
investigation.description = payload.description
investigation.status = payload.status
@@ -71,18 +111,33 @@ def update_investigation(investigation_id: UUID, payload: InvestigationUpdate, d
db.refresh(investigation)
return investigation
+
# Delete a investigation by ID
@router.delete("/{investigation_id}", status_code=status.HTTP_204_NO_CONTENT)
-def delete_investigation(investigation_id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
- investigation = db.query(Investigation).filter(Investigation.id == investigation_id, Investigation.owner_id == current_user.id).first()
+def delete_investigation(
+ investigation_id: UUID,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
+ investigation = (
+ db.query(Investigation)
+ .filter(
+ Investigation.id == investigation_id,
+ Investigation.owner_id == current_user.id,
+ )
+ .first()
+ )
if not investigation:
raise HTTPException(status_code=404, detail="Investigation not found")
-
+
# Get all sketches related to this investigation
- sketches = db.query(Sketch).filter(Sketch.investigation_id == investigation_id).all()
- analyses = db.query(Analysis).filter(Sketch.investigation_id == investigation_id).all()
+ sketches = (
+ db.query(Sketch).filter(Sketch.investigation_id == investigation_id).all()
+ )
+ analyses = (
+ db.query(Analysis).filter(Sketch.investigation_id == investigation_id).all()
+ )
-
# Delete all nodes and relationships for each sketch in Neo4j
for sketch in sketches:
neo4j_query = """
@@ -94,13 +149,13 @@ def delete_investigation(investigation_id: UUID, db: Session = Depends(get_db),
except Exception as e:
print(f"Neo4j cleanup error for sketch {sketch.id}: {e}")
raise HTTPException(status_code=500, detail="Failed to clean up graph data")
-
+
# Delete all sketches from PostgreSQL
for sketch in sketches:
db.delete(sketch)
for analysis in analyses:
db.delete(analysis)
-
+
# Finally delete the investigation
db.delete(investigation)
db.commit()
diff --git a/flowsint-api/app/api/routes/keys.py b/flowsint-api/app/api/routes/keys.py
index 1ecf8d83..a57a2aba 100644
--- a/flowsint-api/app/api/routes/keys.py
+++ b/flowsint-api/app/api/routes/keys.py
@@ -10,45 +10,62 @@ from datetime import datetime
router = APIRouter()
+
def obfuscate_key(key: str) -> str:
"""Obfuscate a key by showing only the last 4 characters, replacing others with asterisks."""
if len(key) <= 4:
return key
return "*" * (len(key) - 4) + key[-4:]
+
# Get the list of all keys for a user
@router.get("", response_model=List[KeyRead])
-def get_keys(db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
+def get_keys(
+ db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)
+):
keys = db.query(Key).filter(Key.owner_id == current_user.id).all()
- response_data = [KeyRead(
- id=key.id,
- owner_id=key.owner_id,
- encrypted_key=obfuscate_key(key.encrypted_key),
- name=key.name,
- created_at=key.created_at
- ) for key in keys]
+ response_data = [
+ KeyRead(
+ id=key.id,
+ owner_id=key.owner_id,
+ encrypted_key=obfuscate_key(key.encrypted_key),
+ name=key.name,
+ created_at=key.created_at,
+ )
+ for key in keys
+ ]
return response_data
+
# Get a key by ID
@router.get("/{id}", response_model=KeyRead)
-def get_key_by_id(id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
+def get_key_by_id(
+ id: UUID,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
key = db.query(Key).filter(Key.id == id, Key.owner_id == current_user.id).first()
if not key:
raise HTTPException(status_code=404, detail="Key not found")
-
+
# Create a response with obfuscated key
response_data = KeyRead(
id=key.id,
owner_id=key.owner_id,
encrypted_key=obfuscate_key(key.encrypted_key),
name=key.name,
- created_at=key.created_at
+ created_at=key.created_at,
)
return response_data
+
# Create a new key
@router.post("/create", response_model=KeyRead, status_code=status.HTTP_201_CREATED)
-def create_key(payload: KeyCreate, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
+def create_key(
+ payload: KeyCreate,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
new_key = Key(
id=uuid4(),
name=payload.name,
@@ -61,12 +78,17 @@ def create_key(payload: KeyCreate, db: Session = Depends(get_db), current_user:
db.refresh(new_key)
return new_key
+
# Delete a key by ID
@router.delete("/{id}", status_code=status.HTTP_204_NO_CONTENT)
-def delete_key(id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
+def delete_key(
+ id: UUID,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
key = db.query(Key).filter(Key.id == id, Key.owner_id == current_user.id).first()
if not key:
raise HTTPException(status_code=404, detail="Key not found")
db.delete(key)
db.commit()
- return None
\ No newline at end of file
+ return None
diff --git a/flowsint-api/app/api/routes/scan.py b/flowsint-api/app/api/routes/scan.py
index 5436d62b..69577f97 100644
--- a/flowsint-api/app/api/routes/scan.py
+++ b/flowsint-api/app/api/routes/scan.py
@@ -9,20 +9,32 @@ from app.api.schemas.scan import ScanRead
router = APIRouter()
+
# Get the list of all scans
[email protected]("", response_model=List[ScanRead],)
-def get_scans(db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
+ "",
+ response_model=List[ScanRead],
+)
+def get_scans(
+ db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)
+):
scans = db.query(Scan).all()
return scans
+
# Get a scan by ID
@router.get("/{id}", response_model=ScanRead)
-def get_scan_by_id(id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
+def get_scan_by_id(
+ id: UUID,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
scan = db.query(Scan).filter(Scan.id == id).first()
if not scan:
raise HTTPException(status_code=404, detail="Transform not found")
return scan
+
# Delete a scan by ID
@router.delete("", status_code=status.HTTP_204_NO_CONTENT)
def delete_scan(db: Session = Depends(get_db)):
@@ -30,14 +42,21 @@ def delete_scan(db: Session = Depends(get_db)):
db.commit()
return None
+
# Delete a scan by ID
@router.delete("/{id}", status_code=status.HTTP_204_NO_CONTENT)
-def delete_scan(id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
- scan = db.query(Scan).filter(Scan.id == id, Scan.owner_id == current_user.id).first()
+def delete_scan(
+ id: UUID,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
+ scan = (
+ db.query(Scan).filter(Scan.id == id, Scan.owner_id == current_user.id).first()
+ )
if not scan:
raise HTTPException(status_code=404, detail="Scan not found")
scan = db.query(Scan).filter(Scan.id == id).all()
-
+
# Finally delete the scan
db.delete(scan)
db.commit()
diff --git a/flowsint-api/app/api/routes/sketches.py b/flowsint-api/app/api/routes/sketches.py
index 586cfdc1..71e7dd62 100644
--- a/flowsint-api/app/api/routes/sketches.py
+++ b/flowsint-api/app/api/routes/sketches.py
@@ -16,14 +16,15 @@ from app.api.deps import get_current_user
router = APIRouter()
+
@router.post("/create", response_model=SketchRead, status_code=status.HTTP_201_CREATED)
def create_sketch(
data: SketchCreate,
db: Session = Depends(get_db),
- current_user: Profile = Depends(get_current_user)
+ current_user: Profile = Depends(get_current_user),
):
sketch_data = data.dict()
- sketch_data['owner_id'] = current_user.id
+ sketch_data["owner_id"] = current_user.id
sketch = Sketch(**sketch_data)
db.add(sketch)
db.commit()
@@ -32,18 +33,31 @@ def create_sketch(
@router.get("", response_model=List[SketchRead])
-def list_sketches(db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
+def list_sketches(
+ db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)
+):
return db.query(Sketch).filter(Sketch.owner_id == current_user.id).all()
+
@router.get("/{sketch_id}")
-def get_sketch_by_id(sketch_id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
+def get_sketch_by_id(
+ sketch_id: UUID,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
sketch = db.query(Sketch).filter(Sketch.id == sketch_id).first()
if not sketch:
raise HTTPException(status_code=404, detail="Sketch not found")
return sketch
+
@router.put("/{id}", response_model=SketchRead)
-def update_sketch(id: UUID, data: SketchUpdate, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
+def update_sketch(
+ id: UUID,
+ data: SketchUpdate,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
sketch = db.query(Sketch).filter(Sketch.owner_id == current_user.id).get(id)
if not sketch:
raise HTTPException(status_code=404, detail="Sketch not found")
@@ -53,12 +67,21 @@ def update_sketch(id: UUID, data: SketchUpdate, db: Session = Depends(get_db), c
db.refresh(sketch)
return sketch
+
@router.delete("/{id}", status_code=204)
-def delete_sketch(id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
- sketch = db.query(Sketch).filter(Sketch.id == id, Sketch.owner_id == current_user.id).first()
+def delete_sketch(
+ id: UUID,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
+ sketch = (
+ db.query(Sketch)
+ .filter(Sketch.id == id, Sketch.owner_id == current_user.id)
+ .first()
+ )
if not sketch:
raise HTTPException(status_code=404, detail="Sketch not found")
-
+
# Delete all nodes and relationships in Neo4j first
neo4j_query = """
MATCH (n {sketch_id: $sketch_id})
@@ -69,16 +92,17 @@ def delete_sketch(id: UUID, db: Session = Depends(get_db), current_user: Profile
except Exception as e:
print(f"Neo4j cleanup error: {e}")
raise HTTPException(status_code=500, detail="Failed to clean up graph data")
-
+
# Then delete the sketch from PostgreSQL
db.delete(sketch)
db.commit()
+
@router.get("/{id}/graph")
async def get_sketch_nodes(
- id: str,
+ id: str,
format: str = None,
- db: Session = Depends(get_db),
+ db: Session = Depends(get_db),
# current_user: Profile = Depends(get_current_user)
):
"""
@@ -94,12 +118,18 @@ async def get_sketch_nodes(
rls: []
Or if format=inline: List of inline relationship strings
"""
- sketch = db.query(Sketch).filter(Sketch.id == id,
- # Sketch.owner_id == current_user.id
- ).first()
+ sketch = (
+ db.query(Sketch)
+ .filter(
+ Sketch.id == id,
+ # Sketch.owner_id == current_user.id
+ )
+ .first()
+ )
if not sketch:
raise HTTPException(status_code=404, detail="Graph not found")
import random
+
nodes_query = """
MATCH (n)
WHERE n.sketch_id = $sketch_id
@@ -126,11 +156,11 @@ async def get_sketch_nodes(
"label": record["data"].get("label", "Node"),
"type": "custom",
"caption": record["data"].get("label", "Node"),
- "position":{
+ "position": {
"x": random.random() * 1000,
"y": random.random() * 1000,
},
- "idx": idx
+ "idx": idx,
}
for idx, record in enumerate(nodes_result)
]
@@ -150,10 +180,12 @@ async def get_sketch_nodes(
if format == "inline":
from flowsint_core.utils import get_inline_relationships
+
return get_inline_relationships(nodes, rels)
return {"nds": nodes, "rls": rels}
+
class NodeData(BaseModel):
label: str = Field(default="Node", description="Label/name of the node")
color: str = Field(default="Node", description="Color of the node")
@@ -163,17 +195,24 @@ class NodeData(BaseModel):
class Config:
extra = "allow" # Accept any additional fields
+
class NodeInput(BaseModel):
type: str = Field(..., description="Type of the node")
- data: NodeData = Field(default_factory=NodeData, description="Additional data for the node")
+ data: NodeData = Field(
+ default_factory=NodeData, description="Additional data for the node"
+ )
+
def dict_to_cypher_props(props: dict, prefix: str = "") -> str:
return ", ".join(f"{key}: ${prefix}{key}" for key in props)
+
@router.post("/{sketch_id}/nodes/add")
-def add_node(sketch_id: str, node: NodeInput, current_user: Profile = Depends(get_current_user)):
+def add_node(
+ sketch_id: str, node: NodeInput, current_user: Profile = Depends(get_current_user)
+):
node_data = node.data.model_dump()
-
+
node_type = node_data["type"]
properties = {
@@ -186,7 +225,6 @@ def add_node(sketch_id: str, node: NodeInput, current_user: Profile = Depends(ge
if node_data:
flattened_data = flatten(node_data)
properties.update(flattened_data)
-
cypher_props = dict_to_cypher_props(properties)
@@ -202,15 +240,19 @@ def add_node(sketch_id: str, node: NodeInput, current_user: Profile = Depends(ge
raise HTTPException(status_code=500, detail=f"Database error: {str(e)}")
if not create_result:
- raise HTTPException(status_code=400, detail="Node creation failed - no result returned")
+ raise HTTPException(
+ status_code=400, detail="Node creation failed - no result returned"
+ )
try:
new_node = create_result[0]["node"]
new_node["id"] = create_result[0]["id"]
except (IndexError, KeyError) as e:
print(f"Error extracting node_id: {e}, result: {create_result}")
- raise HTTPException(status_code=500, detail="Failed to extract node data from response")
-
+ raise HTTPException(
+ status_code=500, detail="Failed to extract node data from response"
+ )
+
new_node["data"] = node_data
new_node["data"]["id"] = new_node["id"]
@@ -219,14 +261,20 @@ def add_node(sketch_id: str, node: NodeInput, current_user: Profile = Depends(ge
"node": new_node,
}
+
class RelationInput(BaseModel):
source: Any
target: Any
type: Literal["one-way", "two-way"]
label: str = "RELATED_TO" # Optionnel : nom de la relation
+
@router.post("/{sketch_id}/relations/add")
-def add_edge(sketch_id: str, relation: RelationInput, current_user: Profile = Depends(get_current_user)):
+def add_edge(
+ sketch_id: str,
+ relation: RelationInput,
+ current_user: Profile = Depends(get_current_user),
+):
query = f"""
MATCH (a) WHERE elementId(a) = $from_id
@@ -255,85 +303,87 @@ def add_edge(sketch_id: str, relation: RelationInput, current_user: Profile = De
"edge": result[0]["r"],
}
+
class NodeDeleteInput(BaseModel):
nodeIds: List[str]
+
class NodeEditInput(BaseModel):
nodeId: str
- data: NodeData = Field(default_factory=NodeData, description="Updated data for the node")
+ data: NodeData = Field(
+ default_factory=NodeData, description="Updated data for the node"
+ )
+
@router.put("/{sketch_id}/nodes/edit")
def edit_node(
sketch_id: str,
node_edit: NodeEditInput,
db: Session = Depends(get_db),
- current_user: Profile = Depends(get_current_user)
+ current_user: Profile = Depends(get_current_user),
):
# First verify the sketch exists and belongs to the user
sketch = db.query(Sketch).filter(Sketch.id == sketch_id).first()
if not sketch:
raise HTTPException(status_code=404, detail="Sketch not found")
-
+
node_data = node_edit.data.model_dump()
node_type = node_data.get("type", "Node")
-
+
# Prepare properties to update
properties = {
"type": node_type.lower(),
"caption": node_data.get("label", "Node"),
"label": node_data.get("label", "Node"),
}
-
+
# Add any additional data from the flattened node_data
if node_data:
flattened_data = flatten(node_data)
properties.update(flattened_data)
-
+
# Build the SET clause for the Cypher query
set_clause = ", ".join(f"n.{key} = ${key}" for key in properties.keys())
-
+
query = f"""
MATCH (n)
WHERE elementId(n) = $node_id AND n.sketch_id = $sketch_id
SET {set_clause}
RETURN n as node
"""
-
- params = {
- "node_id": node_edit.nodeId,
- "sketch_id": sketch_id,
- **properties
- }
-
+
+ params = {"node_id": node_edit.nodeId, "sketch_id": sketch_id, **properties}
+
try:
result = neo4j_connection.query(query, params)
except Exception as e:
print(f"Node update error: {e}")
raise HTTPException(status_code=500, detail="Failed to update node")
-
+
if not result:
raise HTTPException(status_code=404, detail="Node not found or not accessible")
-
+
updated_node = result[0]["node"]
updated_node["data"] = node_data
-
+
return {
"status": "node updated",
"node": updated_node,
}
+
@router.delete("/{sketch_id}/nodes")
def delete_nodes(
sketch_id: str,
nodes: NodeDeleteInput,
db: Session = Depends(get_db),
- current_user: Profile = Depends(get_current_user)
+ current_user: Profile = Depends(get_current_user),
):
# First verify the sketch exists and belongs to the user
sketch = db.query(Sketch).filter(Sketch.id == sketch_id).first()
if not sketch:
raise HTTPException(status_code=404, detail="Sketch not found")
-
+
# Delete nodes and their relationships
query = """
UNWIND $node_ids AS node_id
@@ -341,18 +391,18 @@ def delete_nodes(
WHERE elementId(n) = node_id AND n.sketch_id = $sketch_id
DETACH DELETE n
"""
-
+
try:
- neo4j_connection.query(query, {
- "node_ids": nodes.nodeIds,
- "sketch_id": sketch_id
- })
+ neo4j_connection.query(
+ query, {"node_ids": nodes.nodeIds, "sketch_id": sketch_id}
+ )
except Exception as e:
print(f"Node deletion error: {e}")
raise HTTPException(status_code=500, detail="Failed to delete nodes")
-
+
return {"status": "nodes deleted", "count": len(nodes.nodeIds)}
+
@router.get("/{sketch_id}/nodes/{node_id}")
def get_related_nodes(
sketch_id: str,
@@ -364,7 +414,7 @@ def get_related_nodes(
# sketch = db.query(Sketch).filter(Sketch.id == sketch_id, Sketch.owner_id == current_user.id).first()
# if not sketch:
# raise HTTPException(status_code=404, detail="Sketch not found")
-
+
# Query to get all direct relationships and connected nodes
# First, let's get the center node
center_query = """
@@ -372,19 +422,18 @@ def get_related_nodes(
WHERE elementId(n) = $node_id AND n.sketch_id = $sketch_id
RETURN elementId(n) as id, labels(n) as labels, properties(n) as data
"""
-
+
try:
- center_result = neo4j_connection.query(center_query, {
- "sketch_id": sketch_id,
- "node_id": node_id
- })
+ center_result = neo4j_connection.query(
+ center_query, {"sketch_id": sketch_id, "node_id": node_id}
+ )
except Exception as e:
print(f"Center node query error: {e}")
raise HTTPException(status_code=500, detail="Failed to retrieve center node")
-
+
if not center_result:
raise HTTPException(status_code=404, detail="Node not found")
-
+
# Now get all relationships and connected nodes
relationships_query = """
MATCH (n)
@@ -415,16 +464,15 @@ def get_related_nodes(
properties(other) as other_node_data,
'incoming' as direction
"""
-
+
try:
- result = neo4j_connection.query(relationships_query, {
- "sketch_id": sketch_id,
- "node_id": node_id
- })
+ result = neo4j_connection.query(
+ relationships_query, {"sketch_id": sketch_id, "node_id": node_id}
+ )
except Exception as e:
print(f"Related nodes query error: {e}")
raise HTTPException(status_code=500, detail="Failed to retrieve related nodes")
-
+
# Extract center node info
center_record = center_result[0]
center_node = {
@@ -433,59 +481,61 @@ def get_related_nodes(
"data": center_record["data"],
"label": center_record["data"].get("label", "Node"),
"type": "custom",
- "caption": center_record["data"].get("label", "Node")
+ "caption": center_record["data"].get("label", "Node"),
}
-
+
# Collect all related nodes and relationships
related_nodes = []
relationships = []
seen_nodes = set()
seen_relationships = set()
-
+
for record in result:
# Skip if no relationship found
if not record["rel_id"]:
continue
-
+
# Add relationship if not seen
if record["rel_id"] not in seen_relationships:
if record["direction"] == "outgoing":
- relationships.append({
- "id": record["rel_id"],
- "type": "straight",
- "source": center_node["id"],
- "target": record["other_node_id"],
- "data": record["rel_data"],
- "caption": record["rel_type"]
- })
+ relationships.append(
+ {
+ "id": record["rel_id"],
+ "type": "straight",
+ "source": center_node["id"],
+ "target": record["other_node_id"],
+ "data": record["rel_data"],
+ "caption": record["rel_type"],
+ }
+ )
else: # incoming
- relationships.append({
- "id": record["rel_id"],
- "type": "straight",
- "source": record["other_node_id"],
- "target": center_node["id"],
- "data": record["rel_data"],
- "caption": record["rel_type"]
- })
+ relationships.append(
+ {
+ "id": record["rel_id"],
+ "type": "straight",
+ "source": record["other_node_id"],
+ "target": center_node["id"],
+ "data": record["rel_data"],
+ "caption": record["rel_type"],
+ }
+ )
seen_relationships.add(record["rel_id"])
-
+
# Add related node if not seen
if record["other_node_id"] and record["other_node_id"] not in seen_nodes:
- related_nodes.append({
- "id": record["other_node_id"],
- "labels": record["other_node_labels"],
- "data": record["other_node_data"],
- "label": record["other_node_data"].get("label", "Node"),
- "type": "custom",
- "caption": record["other_node_data"].get("label", "Node")
- })
+ related_nodes.append(
+ {
+ "id": record["other_node_id"],
+ "labels": record["other_node_labels"],
+ "data": record["other_node_data"],
+ "label": record["other_node_data"].get("label", "Node"),
+ "type": "custom",
+ "caption": record["other_node_data"].get("label", "Node"),
+ }
+ )
seen_nodes.add(record["other_node_id"])
-
+
# Combine center node with related nodes
all_nodes = [center_node] + related_nodes
-
- return {
- "nds": all_nodes,
- "rls": relationships
- }
-
+
+ return {"nds": all_nodes, "rls": relationships}
diff --git a/flowsint-api/app/api/routes/transforms.py b/flowsint-api/app/api/routes/transforms.py
index dd3fa27a..475e3fa3 100644
--- a/flowsint-api/app/api/routes/transforms.py
+++ b/flowsint-api/app/api/routes/transforms.py
@@ -13,46 +13,62 @@ from flowsint_core.core.postgre_db import get_db
from app.models.models import Transform, Profile
from app.api.deps import get_current_user
from app.api.schemas.transform import TransformRead, TransformCreate, TransformUpdate
-from flowsint_types import ASN, CIDR, CryptoWallet, CryptoWalletTransaction, CryptoNFT, Website, Individual
+from flowsint_types import (
+ ASN,
+ CIDR,
+ CryptoWallet,
+ CryptoWalletTransaction,
+ CryptoNFT,
+ Website,
+ Individual,
+)
+
class FlowComputationRequest(BaseModel):
nodes: List[Node]
edges: List[Edge]
inputType: Optional[str] = None
+
class FlowComputationResponse(BaseModel):
transformBranches: List[FlowBranch]
initialData: Any
-
+
+
class StepSimulationRequest(BaseModel):
transformBranches: List[FlowBranch]
currentStepIndex: int
+
class LaunchTransformPayload(BaseModel):
values: List[str]
sketch_id: str
+
router = APIRouter()
+
# Get the list of all transforms
@router.get("", response_model=List[TransformRead])
def get_transforms(
category: Optional[str] = Query(None),
db: Session = Depends(get_db),
- current_user: Profile = Depends(get_current_user)
+ current_user: Profile = Depends(get_current_user),
):
query = db.query(Transform)
-
+
if category is not None and category != "undefined":
# Case-insensitive filtering by checking if any category matches (case-insensitive)
transforms = query.all()
return [
- transform for transform in transforms
+ transform
+ for transform in transforms
if any(cat.lower() == category.lower() for cat in transform.category)
]
-
+
return query.order_by(Transform.last_updated_at.desc()).all()
+
# Returns the "raw_materials" for the transform editor
@router.get("/raw_materials")
async def get_material_list():
@@ -72,7 +88,7 @@ async def get_material_list():
"params": scanner.get("params"),
"params_schema": scanner.get("params_schema"),
"required_params": scanner.get("required_params"),
- "icon": scanner.get("icon")
+ "icon": scanner.get("icon"),
}
for scanner in scanner_list
]
@@ -92,27 +108,33 @@ async def get_material_list():
extract_input_schema_transform(Email),
extract_input_schema_transform(CryptoWallet),
extract_input_schema_transform(CryptoWalletTransaction),
- extract_input_schema_transform(CryptoNFT)
+ extract_input_schema_transform(CryptoNFT),
]
-
+
# Put types first, then add all scanner categories
flattened_scanners = {"types": object_inputs}
flattened_scanners.update(scanner_categories)
return {"items": flattened_scanners}
+
# Returns the "raw_materials" for the transform editor
@router.get("/input_type/{input_type}")
-async def get_material_list(input_type:str):
+async def get_material_list(input_type: str):
scanners = ScannerRegistry.list_by_input_type(input_type)
return {"items": scanners}
-
# Create a new transform
[email protected]("/create", response_model=TransformRead, status_code=status.HTTP_201_CREATED)
-def create_transform(payload: TransformCreate, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
-
+ "/create", response_model=TransformRead, status_code=status.HTTP_201_CREATED
+)
+def create_transform(
+ payload: TransformCreate,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
+
new_transform = Transform(
id=uuid4(),
name=payload.name,
@@ -127,28 +149,39 @@ def create_transform(payload: TransformCreate, db: Session = Depends(get_db), cu
db.refresh(new_transform)
return new_transform
+
# Get a transform by ID
@router.get("/{transform_id}", response_model=TransformRead)
-def get_transform_by_id(transform_id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
+def get_transform_by_id(
+ transform_id: UUID,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
transform = db.query(Transform).filter(Transform.id == transform_id).first()
if not transform:
raise HTTPException(status_code=404, detail="Transform not found")
return transform
+
# Update a transform by ID
@router.put("/{transform_id}", response_model=TransformRead)
-def update_transform(transform_id: UUID, payload: TransformUpdate, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
+def update_transform(
+ transform_id: UUID,
+ payload: TransformUpdate,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
transform = db.query(Transform).filter(Transform.id == transform_id).first()
if not transform:
raise HTTPException(status_code=404, detail="Transform not found")
update_data = payload.dict(exclude_unset=True)
for key, value in update_data.items():
- print(f'only update {key}')
+ print(f"only update {key}")
if key == "category":
if "SocialProfile" in value:
value.append("Username")
setattr(transform, key, value)
-
+
transform.last_updated_at = datetime.utcnow()
db.commit()
@@ -158,7 +191,11 @@ def update_transform(transform_id: UUID, payload: TransformUpdate, db: Session =
# Delete a transform by ID
@router.delete("/{transform_id}", status_code=status.HTTP_204_NO_CONTENT)
-def delete_transform(transform_id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
+def delete_transform(
+ transform_id: UUID,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
transform = db.query(Transform).filter(Transform.id == transform_id).first()
if not transform:
raise HTTPException(status_code=404, detail="Transform not found")
@@ -172,7 +209,7 @@ async def launch_transform(
transform_id: str,
payload: LaunchTransformPayload,
db: Session = Depends(get_db),
- current_user: Profile = Depends(get_current_user)
+ current_user: Profile = Depends(get_current_user),
):
try:
transform = db.query(Transform).filter(Transform.id == transform_id).first()
@@ -180,32 +217,37 @@ async def launch_transform(
raise HTTPException(status_code=404, detail="Transform not found")
nodes = [Node(**node) for node in transform.transform_schema["nodes"]]
edges = [Edge(**edge) for edge in transform.transform_schema["edges"]]
- transform_branches = compute_transform_branches(
- payload.values,
- nodes,
- edges
- )
+ transform_branches = compute_transform_branches(payload.values, nodes, edges)
serializable_branches = [branch.model_dump() for branch in transform_branches]
- task = celery.send_task("run_transform", args=[serializable_branches, payload.values, payload.sketch_id, str(current_user.id)])
+ task = celery.send_task(
+ "run_transform",
+ args=[
+ serializable_branches,
+ payload.values,
+ payload.sketch_id,
+ str(current_user.id),
+ ],
+ )
return {"id": task.id}
except Exception as e:
print(e)
raise HTTPException(status_code=404, detail="Transform not found")
+
@router.post("/{transform_id}/compute", response_model=FlowComputationResponse)
-def compute_transforms(request: FlowComputationRequest, current_user: Profile = Depends(get_current_user)):
+def compute_transforms(
+ request: FlowComputationRequest, current_user: Profile = Depends(get_current_user)
+):
initial_data = generate_sample_data(request.inputType or "string")
transform_branches = compute_transform_branches(
- initial_data,
- request.nodes,
- request.edges
- )
+ initial_data, request.nodes, request.edges
+ )
return FlowComputationResponse(
- transformBranches=transform_branches,
- initialData=initial_data
+ transformBranches=transform_branches, initialData=initial_data
)
-
+
+
def generate_sample_data(type_str: str) -> Any:
type_str = type_str.lower() if type_str else "string"
if type_str == "string":
@@ -229,7 +271,10 @@ def generate_sample_data(type_str: str) -> Any:
else:
return f"sample_{type_str}"
-def compute_transform_branches(initial_value: Any, nodes: List[Node], edges: List[Edge]) -> List[FlowBranch]:
+
+def compute_transform_branches(
+ initial_value: Any, nodes: List[Node], edges: List[Edge]
+) -> List[FlowBranch]:
"""Computes flow branches based on nodes and edges with proper DFS traversal"""
# Find input nodes (starting points)
input_nodes = [node for node in nodes if node.data.get("type") == "type"]
@@ -263,33 +308,38 @@ def compute_transform_branches(initial_value: Any, nodes: List[Node], edges: Lis
"""Calculate the shortest possible path length from a node to any leaf"""
if visited is None:
visited = set()
-
+
if start_node in visited:
- return float('inf')
-
+ return float("inf")
+
visited.add(start_node)
out_edges = [edge for edge in edges if edge.source == start_node]
-
+
if not out_edges:
return 1
-
- min_length = float('inf')
+
+ min_length = float("inf")
for edge in out_edges:
length = calculate_path_length(edge.target, visited.copy())
min_length = min(min_length, length)
-
+
return 1 + min_length
def get_outgoing_edges(node_id: str) -> List[Edge]:
"""Get outgoing edges sorted by the shortest possible path length"""
out_edges = [edge for edge in edges if edge.source == node_id]
# Sort edges by the length of the shortest possible path from their target
- return sorted(
- out_edges,
- key=lambda e: calculate_path_length(e.target)
- )
-
- def create_step(node_id: str, branch_id: str, depth: int, input_data: Dict[str, Any], is_input_node: bool, outputs: Dict[str, Any], node_params: Optional[Dict[str, Any]] = None) -> FlowStep:
+ return sorted(out_edges, key=lambda e: calculate_path_length(e.target))
+
+ def create_step(
+ node_id: str,
+ branch_id: str,
+ depth: int,
+ input_data: Dict[str, Any],
+ is_input_node: bool,
+ outputs: Dict[str, Any],
+ node_params: Optional[Dict[str, Any]] = None,
+ ) -> FlowStep:
return FlowStep(
nodeId=node_id,
params=node_params,
@@ -301,7 +351,17 @@ def compute_transform_branches(initial_value: Any, nodes: List[Node], edges: Lis
depth=depth,
)
- def explore_branch(current_node_id: str, branch_id: str, branch_name: str, depth: int, input_data: Dict[str, Any], path: List[str], branch_visited: set, steps: List[FlowStep], parent_outputs: Dict[str, Any] = None) -> None:
+ def explore_branch(
+ current_node_id: str,
+ branch_id: str,
+ branch_name: str,
+ depth: int,
+ input_data: Dict[str, Any],
+ path: List[str],
+ branch_visited: set,
+ steps: List[FlowStep],
+ parent_outputs: Dict[str, Any] = None,
+ ) -> None:
nonlocal branch_counter
# Skip if node is already in current path (cycle detection)
@@ -316,7 +376,9 @@ def compute_transform_branches(initial_value: Any, nodes: List[Node], edges: Lis
is_input_node = current_node.data.get("type") == "type"
if is_input_node:
outputs_array = current_node.data["outputs"].get("properties", [])
- first_output_name = outputs_array[0].get("name", "output") if outputs_array else "output"
+ first_output_name = (
+ outputs_array[0].get("name", "output") if outputs_array else "output"
+ )
current_outputs = {first_output_name: initial_value}
else:
# Check if we already have outputs for this scanner
@@ -331,14 +393,22 @@ def compute_transform_branches(initial_value: Any, nodes: List[Node], edges: Lis
node_params = current_node.data.get("params", {})
# Create and add current step
- current_step = create_step(current_node_id, branch_id, depth, input_data, is_input_node, current_outputs, node_params)
+ current_step = create_step(
+ current_node_id,
+ branch_id,
+ depth,
+ input_data,
+ is_input_node,
+ current_outputs,
+ node_params,
+ )
steps.append(current_step)
path.append(current_node_id)
branch_visited.add(current_node_id)
# Get all outgoing edges sorted by path length
out_edges = get_outgoing_edges(current_node_id)
-
+
if not out_edges:
# Leaf node reached, save the branch
branches.append(FlowBranch(id=branch_id, name=branch_name, steps=steps[:]))
@@ -347,15 +417,17 @@ def compute_transform_branches(initial_value: Any, nodes: List[Node], edges: Lis
for i, edge in enumerate(out_edges):
if edge.target in path: # Skip if would create cycle
continue
-
+
# Prepare next node's input
output_key = edge.sourceHandle
if not output_key and current_outputs:
output_key = list(current_outputs.keys())[0]
-
+
output_value = current_outputs.get(output_key) if output_key else None
if output_value is None and parent_outputs:
- output_value = parent_outputs.get(output_key) if output_key else None
+ output_value = (
+ parent_outputs.get(output_key) if output_key else None
+ )
next_input = {edge.targetHandle or "input": output_value}
@@ -370,15 +442,17 @@ def compute_transform_branches(initial_value: Any, nodes: List[Node], edges: Lis
path,
branch_visited,
steps,
- current_outputs
+ current_outputs,
)
else:
# Create new branch starting from current node
branch_counter += 1
new_branch_id = f"{branch_id}-{branch_counter}"
new_branch_name = f"{branch_name} (Branch {branch_counter})"
- new_steps = steps[:len(steps)] # Copy steps up to current node
- new_branch_visited = branch_visited.copy() # Create new visited set for the branch
+ new_steps = steps[: len(steps)] # Copy steps up to current node
+ new_branch_visited = (
+ branch_visited.copy()
+ ) # Create new visited set for the branch
explore_branch(
edge.target,
new_branch_id,
@@ -388,7 +462,7 @@ def compute_transform_branches(initial_value: Any, nodes: List[Node], edges: Lis
path[:], # Create new path copy for branch
new_branch_visited,
new_steps,
- current_outputs
+ current_outputs,
)
# Backtrack: remove current node from path and remove its step
@@ -408,70 +482,73 @@ def compute_transform_branches(initial_value: Any, nodes: List[Node], edges: Lis
[], # Use list for path to maintain order
set(), # Use set for visited to check membership
[],
- None
+ None,
)
# Sort branches by length (number of steps)
branches.sort(key=lambda branch: len(branch.steps))
return branches
+
def process_node_data(node: Node, inputs: Dict[str, Any]) -> Dict[str, Any]:
"""Traite les données de nœud en fonction du type de nœud et des entrées"""
outputs = {}
output_types = node.data["outputs"].get("properties", [])
-
+
for output in output_types:
output_name = output.get("name", "output")
class_name = node.data.get("class_name", "")
# For simulation purposes, we'll return a placeholder value based on the scanner type
if class_name in ["ReverseResolveScanner", "ResolveScanner"]:
# IP/Domain resolution scanners
- outputs[output_name] = "192.168.1.1" if "ip" in output_name.lower() else "example.com"
+ outputs[output_name] = (
+ "192.168.1.1" if "ip" in output_name.lower() else "example.com"
+ )
elif class_name == "SubdomainScanner":
# Subdomain scanner
outputs[output_name] = f"sub.{inputs.get('input', 'example.com')}"
-
+
elif class_name == "WhoisScanner":
# WHOIS scanner
outputs[output_name] = {
"domain": inputs.get("input", "example.com"),
"registrar": "Example Registrar",
- "creation_date": "2020-01-01"
+ "creation_date": "2020-01-01",
}
-
+
elif class_name == "GeolocationScanner":
# Geolocation scanner
outputs[output_name] = {
"country": "France",
"city": "Paris",
- "coordinates": {"lat": 48.8566, "lon": 2.3522}
+ "coordinates": {"lat": 48.8566, "lon": 2.3522},
}
-
+
elif class_name == "MaigretScanner":
# Social media scanner
outputs[output_name] = {
"username": inputs.get("input", "user123"),
- "platforms": ["twitter", "github", "linkedin"]
+ "platforms": ["twitter", "github", "linkedin"],
}
-
+
elif class_name == "HoleheScanner":
# Email verification scanner
outputs[output_name] = {
"email": inputs.get("input", "[email protected]"),
"exists": True,
- "platforms": ["gmail", "github"]
+ "platforms": ["gmail", "github"],
}
-
+
elif class_name == "SireneScanner":
# Organization scanner
outputs[output_name] = {
"name": inputs.get("input", "Example Corp"),
"siret": "12345678901234",
- "address": "1 Example Street"
+ "address": "1 Example Street",
}
-
+
else:
# For unknown scanners, pass through the input
outputs[output_name] = inputs.get("input") or f"transformed_{output_name}"
- return outputs
\ No newline at end of file
+ return outputs
diff --git a/flowsint-api/app/api/routes/types.py b/flowsint-api/app/api/routes/types.py
index 7bbd4786..e98e5f02 100644
--- a/flowsint-api/app/api/routes/types.py
+++ b/flowsint-api/app/api/routes/types.py
@@ -3,18 +3,44 @@ from uuid import UUID, uuid4
from fastapi import APIRouter
from pydantic import BaseModel, TypeAdapter
from flowsint_types import (
- Domain, Ip, SocialProfile, Organization, Email, ASN, CIDR,
- CryptoWallet, CryptoWalletTransaction, CryptoNFT, Website, Individual,
- Phone, Leak, Username, Credential, Session,
- DNSRecord, SSLCertificate, Device, Document, File, Message,
- Malware, Weapon, BankAccount, CreditCard, WebTracker, Phrase
+ Domain,
+ Ip,
+ SocialProfile,
+ Organization,
+ Email,
+ ASN,
+ CIDR,
+ CryptoWallet,
+ CryptoWalletTransaction,
+ CryptoNFT,
+ Website,
+ Individual,
+ Phone,
+ Leak,
+ Username,
+ Credential,
+ Session,
+ DNSRecord,
+ SSLCertificate,
+ Device,
+ Document,
+ File,
+ Message,
+ Malware,
+ Weapon,
+ BankAccount,
+ CreditCard,
+ WebTracker,
+ Phrase,
)
+
# from flowsint_types.script import Script
# from flowsint_types.reputation_score import ReputationScore
# from flowsint_types.risk_profile import RiskProfile
router = APIRouter()
+
# Returns the "types" for the sketches
@router.get("/")
async def get_types_list():
@@ -39,9 +65,13 @@ async def get_types_list():
"fields": [],
"children": [
extract_input_schema(Individual, label_key="full_name"),
- extract_input_schema(SocialProfile, label_key="username", icon="socialprofile"),
+ extract_input_schema(
+ SocialProfile, label_key="username", icon="socialprofile"
+ ),
extract_input_schema(Organization, label_key="name"),
- extract_input_schema(Username, label_key="username", icon="socialprofile"),
+ extract_input_schema(
+ Username, label_key="username", icon="socialprofile"
+ ),
# extract_input_schema(Alias, label_key="alias", icon="alias"),
# extract_input_schema(Affiliation, label_key="organization", icon="affiliation"),
],
@@ -67,7 +97,9 @@ async def get_types_list():
"children": [
extract_input_schema(Phone, label_key="number"),
extract_input_schema(Email, label_key="email"),
- extract_input_schema(SocialProfile, label_key="username", icon="socialprofile"),
+ extract_input_schema(
+ SocialProfile, label_key="username", icon="socialprofile"
+ ),
extract_input_schema(Message, label_key="content", icon="message"),
],
},
@@ -97,7 +129,9 @@ async def get_types_list():
"label": "Security & Access",
"fields": [],
"children": [
- extract_input_schema(Credential, label_key="username", icon="credential"),
+ extract_input_schema(
+ Credential, label_key="username", icon="credential"
+ ),
extract_input_schema(Session, label_key="session_id", icon="session"),
extract_input_schema(Device, label_key="device_id", icon="device"),
extract_input_schema(Malware, label_key="name", icon="malware"),
@@ -127,8 +161,12 @@ async def get_types_list():
"label": "Financial Data",
"fields": [],
"children": [
- extract_input_schema(BankAccount, label_key="account_number", icon="creditcard"),
- extract_input_schema(CreditCard, label_key="card_number", icon="creditcard"),
+ extract_input_schema(
+ BankAccount, label_key="account_number", icon="creditcard"
+ ),
+ extract_input_schema(
+ CreditCard, label_key="card_number", icon="creditcard"
+ ),
],
},
{
@@ -150,41 +188,47 @@ async def get_types_list():
"label": "Crypto",
"fields": [],
"children": [
- extract_input_schema(CryptoWallet, label_key="address", icon="cryptowallet"),
- extract_input_schema(CryptoWalletTransaction, label_key="hash", icon="cryptowallet"),
+ extract_input_schema(
+ CryptoWallet, label_key="address", icon="cryptowallet"
+ ),
+ extract_input_schema(
+ CryptoWalletTransaction, label_key="hash", icon="cryptowallet"
+ ),
extract_input_schema(CryptoNFT, label_key="name", icon="cryptowallet"),
],
- }
+ },
]
return types
+def extract_input_schema(
+ model: Type[BaseModel], label_key: str, icon: Optional[str] = None
+) -> Dict[str, Any]:
-def extract_input_schema(model: Type[BaseModel], label_key:str, icon: Optional[str]=None) -> Dict[str, Any]:
-
adapter = TypeAdapter(model)
schema = adapter.json_schema()
# Use the main schema properties, not the $defs
type_name = model.__name__
details = schema
return {
- "id": uuid4(),
- "type": type_name,
- "key": type_name.lower(),
- "label_key": label_key,
- "icon": icon or type_name.lower(),
- "label": type_name,
- "description": details.get("description",""),
- "fields": [resolve_field(prop, details=info, schema=schema)
- for prop, info in details.get("properties", {}).items()
- ]
+ "id": uuid4(),
+ "type": type_name,
+ "key": type_name.lower(),
+ "label_key": label_key,
+ "icon": icon or type_name.lower(),
+ "label": type_name,
+ "description": details.get("description", ""),
+ "fields": [
+ resolve_field(prop, details=info, schema=schema)
+ for prop, info in details.get("properties", {}).items()
+ ],
}
-def resolve_field(prop:str, details: dict, schema: dict = None) -> Dict:
+def resolve_field(prop: str, details: dict, schema: dict = None) -> Dict:
"""_summary_
- The fields can sometimes contain nested complex objects, like:
+ The fields can sometimes contain nested complex objects, like:
- Organization having Individual[] as dirigeants, so we want to skip those.
Args:
details (dict): _description_
@@ -193,28 +237,35 @@ def resolve_field(prop:str, details: dict, schema: dict = None) -> Dict:
Returns:
str: _description_
"""
- field = { "name": prop,"label": details["title"], "description": details["description"], "type":"text"}
+ field = {
+ "name": prop,
+ "label": details["title"],
+ "description": details["description"],
+ "type": "text",
+ }
if has_enum(details):
- field["type"] = "select"
- field["options"]= [
- { "label": label, "value": label } for label in get_enum_values(details)
+ field["type"] = "select"
+ field["options"] = [
+ {"label": label, "value": label} for label in get_enum_values(details)
]
field["required"] = is_required(details)
-
+
return field
def has_enum(schema: dict) -> bool:
- any_of = schema.get('anyOf', [])
- return any(isinstance(entry, dict) and 'enum' in entry for entry in any_of)
+ any_of = schema.get("anyOf", [])
+ return any(isinstance(entry, dict) and "enum" in entry for entry in any_of)
+
def is_required(schema: dict) -> bool:
- any_of = schema.get('anyOf', [])
- return not any(entry == {'type': 'null'} for entry in any_of)
-
+ any_of = schema.get("anyOf", [])
+ return not any(entry == {"type": "null"} for entry in any_of)
+
+
def get_enum_values(schema: dict) -> list:
enum_values = []
- for entry in schema.get('anyOf', []):
- if isinstance(entry, dict) and 'enum' in entry:
- enum_values.extend(entry['enum'])
- return enum_values
\ No newline at end of file
+ for entry in schema.get("anyOf", []):
+ if isinstance(entry, dict) and "enum" in entry:
+ enum_values.extend(entry["enum"])
+ return enum_values
diff --git a/flowsint-api/app/api/schemas/analysis.py b/flowsint-api/app/api/schemas/analysis.py
index 0dbd8933..0ff2f57e 100644
--- a/flowsint-api/app/api/schemas/analysis.py
+++ b/flowsint-api/app/api/schemas/analysis.py
@@ -3,6 +3,7 @@ from pydantic import UUID4, BaseModel
from typing import Optional, Any
from datetime import datetime
+
class AnalysisCreate(BaseModel):
title: str
description: Optional[str] = None
@@ -10,6 +11,7 @@ class AnalysisCreate(BaseModel):
owner_id: Optional[UUID4] = None
investigation_id: Optional[UUID4] = None
+
class AnalysisRead(ORMBase):
id: UUID4
title: str
@@ -20,10 +22,11 @@ class AnalysisRead(ORMBase):
owner_id: Optional[UUID4]
investigation_id: Optional[UUID4]
+
class AnalysisUpdate(BaseModel):
title: Optional[str] = None
description: Optional[str] = None
content: Optional[Any] = None
last_updated_at: Optional[datetime] = None
owner_id: Optional[UUID4] = None
- investigation_id: Optional[UUID4] = None
\ No newline at end of file
+ investigation_id: Optional[UUID4] = None
diff --git a/flowsint-api/app/api/schemas/base.py b/flowsint-api/app/api/schemas/base.py
index 972379c3..1823eb11 100644
--- a/flowsint-api/app/api/schemas/base.py
+++ b/flowsint-api/app/api/schemas/base.py
@@ -1,5 +1,6 @@
from pydantic import BaseModel
+
class ORMBase(BaseModel):
class Config:
from_attributes = True
diff --git a/flowsint-api/app/api/schemas/chat.py b/flowsint-api/app/api/schemas/chat.py
index c99a33b2..62f49468 100644
--- a/flowsint-api/app/api/schemas/chat.py
+++ b/flowsint-api/app/api/schemas/chat.py
@@ -3,6 +3,7 @@ from pydantic import UUID4, BaseModel
from typing import List, Optional, Any
from datetime import datetime
+
class ChatMessageRead(BaseModel):
id: UUID4
content: Optional[Any] = None
@@ -14,12 +15,14 @@ class ChatMessageRead(BaseModel):
class Config:
from_attributes = True
+
class ChatCreate(BaseModel):
title: str
description: Optional[str] = None
owner_id: Optional[UUID4] = None
investigation_id: Optional[UUID4] = None
+
class ChatRead(ORMBase):
id: UUID4
title: str
@@ -29,4 +32,3 @@ class ChatRead(ORMBase):
owner_id: Optional[UUID4]
investigation_id: Optional[UUID4]
messages: List[ChatMessageRead]
-
\ No newline at end of file
diff --git a/flowsint-api/app/api/schemas/feedback.py b/flowsint-api/app/api/schemas/feedback.py
index 32b783ab..36596a10 100644
--- a/flowsint-api/app/api/schemas/feedback.py
+++ b/flowsint-api/app/api/schemas/feedback.py
@@ -3,10 +3,12 @@ from pydantic import UUID4, BaseModel
from typing import Optional
from datetime import datetime
+
class FeedbackCreate(BaseModel):
content: Optional[str] = None
owner_id: Optional[UUID4] = None
+
class FeedbackRead(ORMBase):
id: int
created_at: datetime
diff --git a/flowsint-api/app/api/schemas/investigation.py b/flowsint-api/app/api/schemas/investigation.py
index 7680e45e..cccb1c06 100644
--- a/flowsint-api/app/api/schemas/investigation.py
+++ b/flowsint-api/app/api/schemas/investigation.py
@@ -3,12 +3,15 @@ from pydantic import UUID4, BaseModel
from typing import Optional
from datetime import datetime
from .sketch import SketchRead
+
+
class InvestigationCreate(BaseModel):
name: str
description: str
owner_id: Optional[UUID4] = None
status: Optional[str] = "active"
+
class InvestigationRead(ORMBase):
id: UUID4
created_at: datetime
@@ -19,11 +22,13 @@ class InvestigationRead(ORMBase):
status: str
sketches: list[SketchRead] = []
+
class InvestigationProfileCreate(BaseModel):
investigation_id: UUID4
profile_id: UUID4
role: Optional[str] = "member"
+
class InvestigationProfileRead(ORMBase):
id: int
created_at: datetime
@@ -31,6 +36,7 @@ class InvestigationProfileRead(ORMBase):
profile_id: UUID4
role: str
+
class InvestigationUpdate(BaseModel):
name: str
description: Optional[str] = None
diff --git a/flowsint-api/app/api/schemas/investigation_profiles.py b/flowsint-api/app/api/schemas/investigation_profiles.py
index a9e15346..fd56ca7e 100644
--- a/flowsint-api/app/api/schemas/investigation_profiles.py
+++ b/flowsint-api/app/api/schemas/investigation_profiles.py
@@ -3,11 +3,13 @@ from pydantic import UUID4, BaseModel
from typing import Optional
from datetime import datetime
+
class InvestigationProfileCreate(BaseModel):
investigation_id: UUID4
profile_id: UUID4
role: Optional[str] = "member"
+
class InvestigationProfileRead(ORMBase):
id: int
created_at: datetime
diff --git a/flowsint-api/app/api/schemas/key.py b/flowsint-api/app/api/schemas/key.py
index acaf1f09..ed07be78 100644
--- a/flowsint-api/app/api/schemas/key.py
+++ b/flowsint-api/app/api/schemas/key.py
@@ -2,10 +2,12 @@ from .base import ORMBase
from pydantic import UUID4, BaseModel
from datetime import datetime
+
class KeyCreate(BaseModel):
key: str
name: str
-
+
+
class KeyRead(ORMBase):
id: UUID4
owner_id: UUID4
diff --git a/flowsint-api/app/api/schemas/profile.py b/flowsint-api/app/api/schemas/profile.py
index fbed722d..62c473fa 100644
--- a/flowsint-api/app/api/schemas/profile.py
+++ b/flowsint-api/app/api/schemas/profile.py
@@ -2,13 +2,14 @@ from .base import ORMBase
from pydantic import UUID4, BaseModel, EmailStr
from typing import Optional
+
class ProfileCreate(BaseModel):
email: EmailStr
password: str
+
class ProfileRead(ORMBase):
id: UUID4
first_name: Optional[str]
last_name: Optional[str]
avatar_url: Optional[str]
-
diff --git a/flowsint-api/app/api/schemas/scan.py b/flowsint-api/app/api/schemas/scan.py
index a1854fe4..b39d4aa1 100644
--- a/flowsint-api/app/api/schemas/scan.py
+++ b/flowsint-api/app/api/schemas/scan.py
@@ -3,12 +3,14 @@ from .base import ORMBase
from pydantic import UUID4, BaseModel
from typing import Optional
+
class ScanCreate(BaseModel):
values: Optional[List[str]] = None
sketch_id: Optional[UUID4] = None
status: Optional[str] = None
results: Optional[Any] = None
+
class ScanRead(ORMBase):
id: UUID4
sketch_id: Optional[UUID4]
diff --git a/flowsint-api/app/api/schemas/sketch.py b/flowsint-api/app/api/schemas/sketch.py
index 4fda7bb4..aeba81fd 100644
--- a/flowsint-api/app/api/schemas/sketch.py
+++ b/flowsint-api/app/api/schemas/sketch.py
@@ -4,6 +4,7 @@ from typing import Optional
from datetime import datetime
from uuid import UUID
+
class SketchCreate(BaseModel):
title: str
description: str
@@ -11,6 +12,7 @@ class SketchCreate(BaseModel):
investigation_id: UUID4
status: Optional[str] = "active"
+
class SketchRead(ORMBase):
id: UUID4
title: str
@@ -21,18 +23,21 @@ class SketchRead(ORMBase):
last_updated_at: datetime
status: str
+
class SketchProfileCreate(BaseModel):
sketch_id: UUID4
profile_id: UUID4
role: Optional[str] = "editor"
+
class SketchProfileRead(ORMBase):
id: int
created_at: datetime
sketch_id: UUID4
profile_id: UUID4
role: str
-
+
+
class SketchUpdate(BaseModel):
title: Optional[str] = None
description: Optional[str] = None
@@ -40,11 +45,10 @@ class SketchUpdate(BaseModel):
status: Optional[str] = None
investigation_id: Optional[UUID] = None
-
+
class SketchIn(BaseModel):
title: Optional[str] = None
description: Optional[str] = None
owner_id: Optional[UUID] = None
status: Optional[str] = "active"
investigation_id: UUID # requis
-
diff --git a/flowsint-api/app/api/schemas/transform.py b/flowsint-api/app/api/schemas/transform.py
index 1f65384a..bc282506 100644
--- a/flowsint-api/app/api/schemas/transform.py
+++ b/flowsint-api/app/api/schemas/transform.py
@@ -4,12 +4,14 @@ from typing import Optional
from datetime import datetime
from typing import List, Optional, Dict, Any
+
class TransformCreate(BaseModel):
name: str
description: Optional[str] = None
category: Optional[List[str]] = None
transform_schema: Optional[Dict[str, Any]] = None
+
class TransformRead(ORMBase):
id: UUID4
name: str
@@ -19,6 +21,7 @@ class TransformRead(ORMBase):
created_at: datetime
last_updated_at: datetime
+
class TransformUpdate(BaseModel):
name: Optional[str] = None
description: Optional[str] = None
diff --git a/flowsint-api/app/main.py b/flowsint-api/app/main.py
index 9c31c15d..8a91b89d 100644
--- a/flowsint-api/app/main.py
+++ b/flowsint-api/app/main.py
@@ -4,6 +4,7 @@ import os
from typing import List
from dotenv import load_dotenv
from fastapi.middleware.cors import CORSMiddleware
+
# Routes to be included
from app.api.routes import auth
from app.api.routes import investigations
@@ -53,11 +54,13 @@ app.add_middleware(
app.include_router(auth.router, prefix="/api/auth", tags=["auth"])
app.include_router(sketches.router, prefix="/api/sketches", tags=["sketches"])
-app.include_router(investigations.router, prefix="/api/investigations", tags=["investigations"])
+app.include_router(
+ investigations.router, prefix="/api/investigations", tags=["investigations"]
+)
app.include_router(transforms.router, prefix="/api/transforms", tags=["transforms"])
app.include_router(events.router, prefix="/api/events", tags=["events"])
app.include_router(analysis.router, prefix="/api/analyses", tags=["analyses"])
app.include_router(chat.router, prefix="/api/chats", tags=["chats"])
app.include_router(scan.router, prefix="/api/scans", tags=["scans"])
app.include_router(keys.router, prefix="/api/keys", tags=["keys"])
-app.include_router(types.router, prefix="/api/types", tags=["types"])
\ No newline at end of file
+app.include_router(types.router, prefix="/api/types", tags=["types"])
diff --git a/flowsint-api/app/models/models.py b/flowsint-api/app/models/models.py
index b5bbca15..86cd56f7 100644
--- a/flowsint-api/app/models/models.py
+++ b/flowsint-api/app/models/models.py
@@ -1,13 +1,21 @@
import uuid
from datetime import datetime
from sqlalchemy import (
- String, Text, DateTime, ForeignKey,
- Index, func, JSON, Column, Enum as SQLEnum
+ String,
+ Text,
+ DateTime,
+ ForeignKey,
+ Index,
+ func,
+ JSON,
+ Column,
+ Enum as SQLEnum,
)
from sqlalchemy.dialects.postgresql import UUID as PGUUID, ARRAY, JSONB
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
from flowsint_core.core.enums import EventLevel
+
class Base(DeclarativeBase):
pass
@@ -18,19 +26,27 @@ class Feedback(Base):
id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
created_at = mapped_column(DateTime(timezone=True), server_default=func.now())
content = mapped_column(Text, nullable=True)
- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id"), nullable=True)
+ owner_id = mapped_column(
+ PGUUID(as_uuid=True), ForeignKey("profiles.id"), nullable=True
+ )
class Investigation(Base):
__tablename__ = "investigations"
- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+ id: Mapped[uuid.UUID] = mapped_column(
+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
+ )
created_at = mapped_column(DateTime(timezone=True), server_default=func.now())
name = mapped_column(Text)
description = mapped_column(Text)
- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)
+ owner_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),
+ nullable=True,
+ )
last_updated_at = mapped_column(DateTime(timezone=True), server_default=func.now())
- status = mapped_column(String, server_default='active')
+ status = mapped_column(String, server_default="active")
sketches = relationship("Sketch", back_populates="investigation")
analyses = relationship("Analysis", back_populates="investigation")
chats = relationship("Chat", back_populates="investigation")
@@ -45,32 +61,51 @@ class InvestigationsProfiles(Base):
id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
created_at = mapped_column(DateTime(timezone=True), server_default=func.now())
- investigation_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"))
- profile_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"))
+ investigation_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"),
+ )
+ profile_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),
+ )
role = mapped_column(String, server_default="member")
__table_args__ = (
Index("idx_investigations_profiles_investigation_id", "investigation_id"),
Index("idx_investigations_profiles_profile_id", "profile_id"),
# Uniqueness constraint
- Index("projects_profiles_unique_profile_project", "profile_id", "investigation_id", unique=True),
+ Index(
+ "projects_profiles_unique_profile_project",
+ "profile_id",
+ "investigation_id",
+ unique=True,
+ ),
)
class Log(Base):
__tablename__ = "logs"
- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+ id: Mapped[uuid.UUID] = mapped_column(
+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
+ )
content = mapped_column(JSONB, nullable=True)
created_at = mapped_column(DateTime(timezone=True), server_default=func.now())
- sketch_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)
+ sketch_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"),
+ nullable=True,
+ )
type = Column(SQLEnum(EventLevel), default=EventLevel.INFO)
class Profile(Base):
__tablename__ = "profiles"
- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+ id: Mapped[uuid.UUID] = mapped_column(
+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
+ )
first_name = mapped_column(Text, nullable=True)
last_name = mapped_column(Text, nullable=True)
avatar_url = mapped_column(Text, nullable=True)
@@ -82,17 +117,23 @@ class Profile(Base):
class Scan(Base):
__tablename__ = "scans"
- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
- sketch_id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)
+ id: Mapped[uuid.UUID] = mapped_column(
+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
+ )
+ sketch_id: Mapped[uuid.UUID] = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"),
+ nullable=True,
+ )
status = Column(SQLEnum(EventLevel), default=EventLevel.PENDING)
started_at = Column(DateTime, default=datetime.utcnow)
completed_at = Column(DateTime, nullable=True)
error = Column(Text, nullable=True)
details = Column(JSON, nullable=True)
-
+
# Relationships
sketch = relationship("Sketch", back_populates="scans")
-
+
def __repr__(self):
return f"<Scan(id={self.id}, status={self.status})>"
@@ -100,13 +141,22 @@ class Scan(Base):
class Sketch(Base):
__tablename__ = "sketches"
- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+ id: Mapped[uuid.UUID] = mapped_column(
+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
+ )
title = mapped_column(Text)
description = mapped_column(Text)
created_at = mapped_column(DateTime(timezone=True), server_default=func.now())
- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)
+ owner_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),
+ nullable=True,
+ )
status = mapped_column(String, server_default="active")
- investigation_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"))
+ investigation_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"),
+ )
investigation = relationship("Investigation", back_populates="sketches")
last_updated_at = mapped_column(DateTime(timezone=True), server_default=func.now())
scans = relationship("Scan", back_populates="sketch")
@@ -122,21 +172,34 @@ class SketchesProfiles(Base):
id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
created_at = mapped_column(DateTime(timezone=True), server_default=func.now())
- profile_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"))
- sketch_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"))
+ profile_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),
+ )
+ sketch_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"),
+ )
role = mapped_column(String, server_default="editor")
__table_args__ = (
Index("idx_sketches_profiles_sketch_id", "sketch_id"),
Index("idx_sketches_profiles_profile_id", "profile_id"),
- Index("investigations_profiles_unique_profile_investigation", "profile_id", "sketch_id", unique=True),
+ Index(
+ "investigations_profiles_unique_profile_investigation",
+ "profile_id",
+ "sketch_id",
+ unique=True,
+ ),
)
class Transform(Base):
__tablename__ = "transforms"
- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+ id: Mapped[uuid.UUID] = mapped_column(
+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
+ )
name = mapped_column(Text, nullable=False)
description = mapped_column(Text, nullable=True)
category = mapped_column(ARRAY(Text), nullable=True)
@@ -148,14 +211,24 @@ class Transform(Base):
class Analysis(Base):
__tablename__ = "analyses"
- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+ id: Mapped[uuid.UUID] = mapped_column(
+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
+ )
title = mapped_column(Text, nullable=False)
description = mapped_column(Text, nullable=True)
content = mapped_column(JSON, nullable=True)
created_at = mapped_column(DateTime(timezone=True), server_default=func.now())
last_updated_at = mapped_column(DateTime(timezone=True), server_default=func.now())
- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)
- investigation_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)
+ owner_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),
+ nullable=True,
+ )
+ investigation_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"),
+ nullable=True,
+ )
investigation = relationship("Investigation", back_populates="analyses")
__table_args__ = (
@@ -163,16 +236,27 @@ class Analysis(Base):
Index("idx_analyses_investigation_id", "investigation_id"),
)
+
class Chat(Base):
__tablename__ = "chats"
- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+ id: Mapped[uuid.UUID] = mapped_column(
+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
+ )
title = mapped_column(Text, nullable=False)
description = mapped_column(Text, nullable=True)
created_at = mapped_column(DateTime(timezone=True), server_default=func.now())
last_updated_at = mapped_column(DateTime(timezone=True), server_default=func.now())
- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)
- investigation_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)
+ owner_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),
+ nullable=True,
+ )
+ investigation_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"),
+ nullable=True,
+ )
investigation = relationship("Investigation", back_populates="chats")
messages = relationship("ChatMessage", back_populates="chat", cascade="all, delete")
__table_args__ = (
@@ -180,30 +264,42 @@ class Chat(Base):
Index("idx_chats_investigation_id", "investigation_id"),
)
+
class ChatMessage(Base):
__tablename__ = "messages"
- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+ id: Mapped[uuid.UUID] = mapped_column(
+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
+ )
content = mapped_column(JSON, nullable=True)
context = mapped_column(JSON, nullable=True)
is_bot: Mapped[bool] = mapped_column(default=False)
created_at = mapped_column(DateTime(timezone=True), server_default=func.now())
- chat_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("chats.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=False)
- chat = relationship("Chat", back_populates="messages")
- __table_args__ = (
- Index("idx_messages_chat_id", "chat_id"),
+ chat_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("chats.id", onupdate="CASCADE", ondelete="CASCADE"),
+ nullable=False,
)
+ chat = relationship("Chat", back_populates="messages")
+ __table_args__ = (Index("idx_messages_chat_id", "chat_id"),)
+
class Key(Base):
__tablename__ = "keys"
- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+ id: Mapped[uuid.UUID] = mapped_column(
+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
+ )
name = mapped_column(String, nullable=False)
- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)
+ owner_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),
+ nullable=True,
+ )
encrypted_key = mapped_column(String, nullable=False)
created_at = mapped_column(DateTime(timezone=True), server_default=func.now())
__table_args__ = (
Index("idx_keys_owner_id", "owner_id"),
Index("idx_keys_service", "name"),
- )
\ No newline at end of file
+ )
diff --git a/flowsint-api/app/utils.py b/flowsint-api/app/utils.py
index b1e02a8e..238eed7c 100644
--- a/flowsint-api/app/utils.py
+++ b/flowsint-api/app/utils.py
@@ -13,6 +13,7 @@ import inspect
from typing import Any, Dict, Type
from pydantic import BaseModel, TypeAdapter
+
def is_valid_ip(address: str) -> bool:
try:
ipaddress.ip_address(address)
@@ -20,11 +21,13 @@ def is_valid_ip(address: str) -> bool:
except ValueError:
return False
+
def is_valid_username(username: str) -> bool:
if not re.match(r"^[a-zA-Z0-9_-]{3,30}$", username):
return False
return True
+
def is_valid_email(email: str) -> bool:
if not re.match(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$", email):
return False
@@ -34,26 +37,29 @@ def is_valid_email(email: str) -> bool:
def is_valid_domain(url_or_domain: str) -> str:
try:
- parsed = urlparse(url_or_domain if "://" in url_or_domain else "http://" + url_or_domain)
+ parsed = urlparse(
+ url_or_domain if "://" in url_or_domain else "http://" + url_or_domain
+ )
hostname = parsed.hostname or url_or_domain
-
+
if not hostname or "." not in hostname:
return False
-
+
if not re.match(r"^[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$", hostname):
return False
-
+
return True
except Exception as e:
return False
+
def is_root_domain(domain: str) -> bool:
"""
Determine if a domain is a root domain or subdomain.
-
+
Args:
domain: The domain string to check
-
+
Returns:
True if it's a root domain (e.g., example.com), False if it's a subdomain (e.g., sub.example.com)
"""
@@ -62,20 +68,32 @@ def is_root_domain(domain: str) -> bool:
if "://" in domain:
parsed = urlparse(domain)
domain = parsed.hostname or domain
-
+
# Split by dots
- parts = domain.split('.')
-
+ parts = domain.split(".")
+
# Handle common country code TLDs that have 2 parts (e.g., .co.uk, .com.au, .org.uk)
- common_cc_tlds = ['.co.uk', '.com.au', '.org.uk', '.net.uk', '.gov.uk', '.ac.uk',
- '.co.nz', '.com.sg', '.co.jp', '.co.kr', '.com.br', '.com.mx']
-
+ common_cc_tlds = [
+ ".co.uk",
+ ".com.au",
+ ".org.uk",
+ ".net.uk",
+ ".gov.uk",
+ ".ac.uk",
+ ".co.nz",
+ ".com.sg",
+ ".co.jp",
+ ".co.kr",
+ ".com.br",
+ ".com.mx",
+ ]
+
# Check if the domain ends with a common country code TLD
for cc_tld in common_cc_tlds:
if domain.endswith(cc_tld):
# For country code TLDs, we need exactly 3 parts (e.g., example.co.uk)
return len(parts) == 3
-
+
# For regular TLDs, a root domain has 2 parts (e.g., example.com)
# A subdomain has 3 or more parts (e.g., sub.example.com, www.sub.example.com)
return len(parts) == 2
@@ -83,6 +101,7 @@ def is_root_domain(domain: str) -> bool:
# If we can't parse it, assume it's not a root domain
return False
+
def is_valid_number(phone: str, region: str = "FR") -> None:
"""
Validates a phone number. Raises InvalidPhoneNumberError if invalid.
@@ -94,19 +113,21 @@ def is_valid_number(phone: str, region: str = "FR") -> None:
return False
except NumberParseException:
return False
-
+
+
def parse_asn(asn: str) -> int:
if not is_valid_asn(asn):
raise ValueError(f"Invalid ASN format: {asn}")
return int(re.sub(r"(?i)^AS", "", asn))
-
+
def is_valid_asn(asn: str) -> bool:
if not re.fullmatch(r"(AS)?\d+", asn, re.IGNORECASE):
return False
asn_num = int(re.sub(r"(?i)^AS", "", asn))
return 0 <= asn_num <= 4294967295
+
def resolve_type(details: dict, schema_context: dict = None) -> str:
if "anyOf" in details:
types = []
@@ -121,22 +142,23 @@ def resolve_type(details: dict, schema_context: dict = None) -> str:
else:
types.append(option.get("type", "unknown"))
return " | ".join(types)
-
+
if "type" in details:
if details["type"] == "array":
item_type = resolve_type(details.get("items", {}), schema_context)
return f"{item_type}[]"
return details["type"]
-
+
# Handle $ref in array items or other contexts
if "$ref" in details and schema_context:
ref_path = details["$ref"]
if ref_path.startswith("#/$defs/"):
ref_name = ref_path.split("/")[-1]
return ref_name
-
+
return "any"
+
def extract_input_schema_transform(model: Type[BaseModel]) -> Dict[str, Any]:
adapter = TypeAdapter(model)
schema = adapter.json_schema()
@@ -153,37 +175,30 @@ def extract_input_schema_transform(model: Type[BaseModel]) -> Dict[str, Any]:
"outputs": {
"type": type_name,
"properties": [
- {
- "name": prop,
- "type": resolve_type(info, schema)
- }
+ {"name": prop, "type": resolve_type(info, schema)}
for prop, info in details.get("properties", {}).items()
- ]
- },
- "inputs": {
- "type": "",
- "properties": []
+ ],
},
+ "inputs": {"type": "", "properties": []},
"type": "type",
"category": model.__name__,
}
-
def get_domain_from_ssl(ip: str, port: int = 443) -> str | None:
try:
context = ssl.create_default_context()
with socket.create_connection((ip, port), timeout=3) as sock:
with context.wrap_socket(sock, server_hostname=ip) as ssock:
cert = ssock.getpeercert()
- subject = cert.get('subject', [])
+ subject = cert.get("subject", [])
for entry in subject:
- if entry[0][0] == 'commonName':
+ if entry[0][0] == "commonName":
return entry[0][1]
# Alternative: check subjectAltName
- san = cert.get('subjectAltName', [])
+ san = cert.get("subjectAltName", [])
for typ, val in san:
- if typ == 'DNS':
+ if typ == "DNS":
return val
except Exception as e:
print(f"SSL extraction failed for {ip}: {e}")
@@ -208,12 +223,14 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:
scanner_node = node_lookup.get(target_id)
if scanner_node and scanner_node["data"]["type"] == "scanner":
- scanners.append({
- "scanner_name": scanner_node["data"]["name"],
- "module": scanner_node["data"]["module"],
- "input": source_handle,
- "output": target_handle
- })
+ scanners.append(
+ {
+ "scanner_name": scanner_node["data"]["name"],
+ "module": scanner_node["data"]["module"],
+ "input": source_handle,
+ "output": target_handle,
+ }
+ )
return {
"input": {
@@ -221,25 +238,22 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:
"outputs": input_output,
},
"scanners": scanners,
- "scanner_names" : [scanner["scanner_name"] for scanner in scanners]
-
+ "scanner_names": [scanner["scanner_name"] for scanner in scanners],
}
-
+
+
def get_label_color(label: str) -> str:
- color_map = {
- 'subdomain': '#A5ABB6',
- 'domain': '#68BDF6',
- 'default': '#A5ABB6'
- }
-
+ color_map = {"subdomain": "#A5ABB6", "domain": "#68BDF6", "default": "#A5ABB6"}
+
return color_map.get(label, color_map["default"])
+
def flatten(data_dict, prefix=""):
"""
Flattens a dictionary to contain only Neo4j-compatible property values.
Neo4j supports primitive types (string, number, boolean) and arrays of those types.
Args:
- data_dict (dict): Dictionary to flatten
+ data_dict (dict): Dictionary to flatten
Returns:
dict: Flattened dictionary with only Neo4j-compatible values
"""
@@ -250,13 +264,14 @@ def flatten(data_dict, prefix=""):
if value is None:
continue
if isinstance(value, (str, int, float, bool)) or (
- isinstance(value, list) and all(isinstance(item, (str, int, float, bool)) for item in value)
+ isinstance(value, list)
+ and all(isinstance(item, (str, int, float, bool)) for item in value)
):
key = f"{prefix}{key}"
- flattened[key] = value
+ flattened[key] = value
return flattened
-
+
def get_inline_relationships(nodes: List[Any], edges: List[Any]) -> List[str]:
"""
Get the inline relationships for a list of nodes and edges.
@@ -266,14 +281,15 @@ def get_inline_relationships(nodes: List[Any], edges: List[Any]) -> List[str]:
source = next((node for node in nodes if node["id"] == edge["source"]), None)
target = next((node for node in nodes if node["id"] == edge["target"]), None)
if source and target:
- relationships.append({"source":source, "edge":edge, "target":target})
+ relationships.append({"source": source, "edge": edge, "target": target})
return relationships
+
def to_json_serializable(obj):
"""Convert any object to a JSON-serializable format."""
import json
from pydantic import BaseModel
-
+
try:
# Test if already JSON serializable
json.dumps(obj)
@@ -282,11 +298,15 @@ def to_json_serializable(obj):
# Handle common cases
if isinstance(obj, BaseModel):
# Use mode='json' to ensure all Pydantic types are properly serialized
- return obj.model_dump(mode='json') if hasattr(obj, 'model_dump') else obj.dict()
+ return (
+ obj.model_dump(mode="json")
+ if hasattr(obj, "model_dump")
+ else obj.dict()
+ )
elif isinstance(obj, (list, tuple)):
return [to_json_serializable(item) for item in obj]
elif isinstance(obj, dict):
return {key: to_json_serializable(value) for key, value in obj.items()}
else:
# Convert anything else to string
- return str(obj)
\ No newline at end of file
+ return str(obj)
diff --git a/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx b/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx
index 38492352..e5e37a55 100644
--- a/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx
@@ -71,19 +71,19 @@ export default function DetailsPanel({ node }: { node: GraphNode | null }) {
)}
<div className="flex-1 min-h-0">
<ResizablePanelGroup direction="vertical" className="h-full">
- <ResizablePanel defaultSize={30} minSize={20}>
+ <ResizablePanel defaultSize={20} minSize={20}>
<div className="h-full overflow-auto">
<KeyValueDisplay data={node.data} />
</div>
</ResizablePanel>
<ResizableHandle />
- <ResizablePanel defaultSize={40} minSize={25}>
+ <ResizablePanel defaultSize={35} minSize={25}>
<div className="h-full p-3">
<NeighborsGraph sketchId={sketchId as string} nodeId={node.id} />
</div>
</ResizablePanel>
<ResizableHandle />
- <ResizablePanel defaultSize={30} minSize={20}>
+ <ResizablePanel defaultSize={45} minSize={20}>
<Relationships sketchId={sketchId as string} nodeId={node.id} />
</ResizablePanel>
</ResizablePanelGroup>
diff --git a/flowsint-core/src/flowsint_core/__init__.py b/flowsint-core/src/flowsint_core/__init__.py
index c5f63f49..c6c2baa3 100644
--- a/flowsint-core/src/flowsint_core/__init__.py
+++ b/flowsint-core/src/flowsint_core/__init__.py
@@ -16,4 +16,4 @@ from .core.vault import *
from .utils import *
__version__ = "0.1.0"
-__author__ = "EliottElek <[email protected]>"
\ No newline at end of file
+__author__ = "EliottElek <[email protected]>"
diff --git a/flowsint-core/src/flowsint_core/core/__init__.py b/flowsint-core/src/flowsint_core/core/__init__.py
deleted file mode 100644
index 5e2b07a9..00000000
--- a/flowsint-core/src/flowsint_core/core/__init__.py
+++ /dev/null
@@ -1,35 +0,0 @@
-"""
-Core module initialization.
-"""
-
-from .registry import ScannerRegistry
-
-# Register scanner modules for discovery
-ScannerRegistry.register_module("flowsint_transforms.domains.subdomains")
-ScannerRegistry.register_module("flowsint_transforms.domains.whois")
-ScannerRegistry.register_module("flowsint_transforms.domains.resolve")
-ScannerRegistry.register_module("flowsint_transforms.ips.reverse_resolve")
-ScannerRegistry.register_module("flowsint_transforms.ips.geolocation")
-ScannerRegistry.register_module("flowsint_transforms.socials.maigret")
-ScannerRegistry.register_module("flowsint_transforms.ips.ip_to_asn")
-ScannerRegistry.register_module("flowsint_transforms.ips.asn_to_cidrs")
-ScannerRegistry.register_module("flowsint_transforms.ips.cidr_to_ips")
-ScannerRegistry.register_module("flowsint_transforms.organizations.org_to_asn")
-ScannerRegistry.register_module("flowsint_transforms.domains.domain_to_asn")
-ScannerRegistry.register_module("flowsint_transforms.crypto.wallet_to_transactions")
-ScannerRegistry.register_module("flowsint_transforms.crypto.wallet_to_nfts")
-ScannerRegistry.register_module("flowsint_transforms.domains.to_website")
-ScannerRegistry.register_module("flowsint_transforms.websites.to_crawler")
-ScannerRegistry.register_module("flowsint_transforms.websites.to_links")
-ScannerRegistry.register_module("flowsint_transforms.websites.to_domain")
-ScannerRegistry.register_module("flowsint_transforms.emails.to_gravatar")
-ScannerRegistry.register_module("flowsint_transforms.emails.to_leaks")
-ScannerRegistry.register_module("flowsint_transforms.individuals.to_org")
-ScannerRegistry.register_module("flowsint_transforms.organizations.to_infos")
-ScannerRegistry.register_module("flowsint_transforms.websites.to_webtrackers")
-ScannerRegistry.register_module("flowsint_transforms.n8n.connector")
-
-# Discover and register all scanners
-ScannerRegistry.discover_scanners()
-
-__all__ = ["ScannerRegistry"]
diff --git a/flowsint-core/src/flowsint_core/core/auth.py b/flowsint-core/src/flowsint_core/core/auth.py
index cedab67b..6b601bc2 100644
--- a/flowsint-core/src/flowsint_core/core/auth.py
+++ b/flowsint-core/src/flowsint_core/core/auth.py
@@ -10,21 +10,28 @@ load_dotenv()
# Get AUTH_SECRET after loading environment variables
AUTH_SECRET = os.getenv("AUTH_SECRET")
if not AUTH_SECRET:
- raise ValueError("AUTH_SECRET environment variable is not set. Please set it in your .env file.")
+ raise ValueError(
+ "AUTH_SECRET environment variable is not set. Please set it in your .env file."
+ )
ALGORITHM = "HS256"
ACCESS_TOKEN_EXPIRE_MINUTES = 60 * 60
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+
def verify_password(plain_password: str, hashed_password: str) -> bool:
return pwd_context.verify(plain_password, hashed_password)
+
def get_password_hash(password: str) -> str:
return pwd_context.hash(password)
+
def create_access_token(data: dict, expires_delta: timedelta | None = None):
to_encode = data.copy()
- expire = datetime.utcnow() + (expires_delta or timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES))
+ expire = datetime.utcnow() + (
+ expires_delta or timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES)
+ )
to_encode.update({"exp": expire})
encoded_jwt = jwt.encode(to_encode, AUTH_SECRET, algorithm=ALGORITHM)
- return encoded_jwt
\ No newline at end of file
+ return encoded_jwt
diff --git a/flowsint-core/src/flowsint_core/core/celery.py b/flowsint-core/src/flowsint_core/core/celery.py
index 58cd63b5..1da80106 100644
--- a/flowsint-core/src/flowsint_core/core/celery.py
+++ b/flowsint-core/src/flowsint_core/core/celery.py
@@ -8,7 +8,7 @@ celery = Celery(
include=[
"flowsint_core.tasks.event",
"flowsint_core.tasks.transform",
- ]
+ ],
)
celery.conf.update(
@@ -20,5 +20,5 @@ celery.conf.update(
task_track_started=True,
task_time_limit=3600, # 1 hour
worker_max_tasks_per_child=1000,
- worker_prefetch_multiplier=1
-)
\ No newline at end of file
+ worker_prefetch_multiplier=1,
+)
diff --git a/flowsint-core/src/flowsint_core/core/config.py b/flowsint-core/src/flowsint_core/core/config.py
index e7a744be..bec1c203 100644
--- a/flowsint-core/src/flowsint_core/core/config.py
+++ b/flowsint-core/src/flowsint_core/core/config.py
@@ -3,8 +3,10 @@ from dotenv import load_dotenv
load_dotenv()
+
class Settings:
CELERY_BROKER_URL = os.getenv("REDIS_URI", "redis://localhost:6379/0")
CELERY_RESULT_BACKEND = os.getenv("REDIS_URI", "redis://localhost:6379/0")
-settings = Settings()
\ No newline at end of file
+
+settings = Settings()
diff --git a/flowsint-core/src/flowsint_core/core/enums.py b/flowsint-core/src/flowsint_core/core/enums.py
index f0c37bb6..49471c73 100644
--- a/flowsint-core/src/flowsint_core/core/enums.py
+++ b/flowsint-core/src/flowsint_core/core/enums.py
@@ -1,5 +1,6 @@
from enum import Enum
+
class EventLevel(str, Enum):
# Standard log levels
INFO = "INFO"
@@ -21,4 +22,4 @@ class EventLevel(str, Enum):
@property
def lowercase(self) -> str:
"""Get the lowercase version of the enum value"""
- return self.value.lower()
\ No newline at end of file
+ return self.value.lower()
diff --git a/flowsint-core/src/flowsint_core/core/events.py b/flowsint-core/src/flowsint_core/core/events.py
index 744fca85..a2d9a88b 100644
--- a/flowsint-core/src/flowsint_core/core/events.py
+++ b/flowsint-core/src/flowsint_core/core/events.py
@@ -7,6 +7,7 @@ from uuid import UUID
import redis.asyncio as redis
import os
+
class EventEmitter:
def __init__(self):
self.id = uuid.uuid4()
@@ -31,12 +32,14 @@ class EventEmitter:
"""Get the next message from Redis for a specific channel"""
if channel not in self.pubsubs:
return None
-
- message = await self.pubsubs[channel].get_message(ignore_subscribe_messages=True)
+
+ message = await self.pubsubs[channel].get_message(
+ ignore_subscribe_messages=True
+ )
if message is None:
await asyncio.sleep(0.1)
return None
-
+
if message:
data = message["data"]
if isinstance(data, bytes):
@@ -57,8 +60,10 @@ class EventEmitter:
except ValueError:
return False
+
event_emitter = EventEmitter()
+
def init_events(app: FastAPI):
"""Initialize the event system in the FastAPI app"""
- print("[EventEmitter] Events initialized in FastAPI app")
\ No newline at end of file
+ print("[EventEmitter] Events initialized in FastAPI app")
diff --git a/flowsint-core/src/flowsint_core/core/graph_db.py b/flowsint-core/src/flowsint_core/core/graph_db.py
index 1d8d1503..84a256c2 100644
--- a/flowsint-core/src/flowsint_core/core/graph_db.py
+++ b/flowsint-core/src/flowsint_core/core/graph_db.py
@@ -4,18 +4,21 @@ from dotenv import load_dotenv
load_dotenv()
+
class Neo4jConnection:
def __init__(self, uri: str, user: str, password: str):
self.__uri = uri
self.__user = user
self.__password = password
- self.__driver = GraphDatabase.driver(self.__uri, auth=(self.__user, self.__password))
+ self.__driver = GraphDatabase.driver(
+ self.__uri, auth=(self.__user, self.__password)
+ )
def query(self, query: str, parameters: dict = None):
with self.__driver.session() as session:
result = session.run(query, parameters)
return result.data()
-
+
URI = os.getenv("NEO4J_URI_BOLT")
URI = "bolt://localhost:7687"
@@ -25,7 +28,6 @@ PASSWORD = os.getenv("NEO4J_PASSWORD")
neo4j_connection = Neo4jConnection(URI, USERNAME, PASSWORD)
-
# import os
# from neo4j import GraphDatabase
# from dotenv import load_dotenv
@@ -43,7 +45,7 @@ neo4j_connection = Neo4jConnection(URI, USERNAME, PASSWORD)
# with self.__driver.session() as session:
# result = session.run(query, parameters)
# return result.data()
-
+
# URI = os.getenv("NEO4J_URI_BOLT")
# USERNAME = os.getenv("NEO4J_USERNAME")
diff --git a/flowsint-core/src/flowsint_core/core/logger.py b/flowsint-core/src/flowsint_core/core/logger.py
index 88627814..eb6905b1 100644
--- a/flowsint-core/src/flowsint_core/core/logger.py
+++ b/flowsint-core/src/flowsint_core/core/logger.py
@@ -8,17 +8,16 @@ import logging
logger = logging.getLogger(__name__)
+
class Logger:
@staticmethod
- def _create_log(sketch_id: Union[str, UUID], level: EventLevel, content: str) -> Log:
+ def _create_log(
+ sketch_id: Union[str, UUID], level: EventLevel, content: str
+ ) -> Log:
"""Create a log entry in the database"""
db = next(get_db())
try:
- log = Log(
- sketch_id=str(sketch_id),
- type=level.value,
- content=content
- )
+ log = Log(sketch_id=str(sketch_id), type=level.value, content=content)
db.add(log)
db.commit()
db.refresh(log)
@@ -30,46 +29,62 @@ class Logger:
def info(sketch_id: Union[str, UUID], message: Dict):
"""Log an info message"""
log = Logger._create_log(sketch_id, EventLevel.INFO, message)
- emit_event_task.apply(args=[str(log.id), str(sketch_id), EventLevel.INFO, message])
+ emit_event_task.apply(
+ args=[str(log.id), str(sketch_id), EventLevel.INFO, message]
+ )
@staticmethod
def error(sketch_id: Union[str, UUID], message: Dict):
"""Log an error message"""
log = Logger._create_log(sketch_id, EventLevel.FAILED, message)
- emit_event_task.apply(args=[str(log.id), str(sketch_id), EventLevel.FAILED, message])
+ emit_event_task.apply(
+ args=[str(log.id), str(sketch_id), EventLevel.FAILED, message]
+ )
@staticmethod
def warn(sketch_id: Union[str, UUID], message: Dict):
"""Log a warning message"""
log = Logger._create_log(sketch_id, EventLevel.WARNING, message)
- emit_event_task.apply(args=[str(log.id), str(sketch_id), EventLevel.WARNING, message])
+ emit_event_task.apply(
+ args=[str(log.id), str(sketch_id), EventLevel.WARNING, message]
+ )
@staticmethod
def debug(sketch_id: Union[str, UUID], message: Dict):
"""Log a debug message"""
log = Logger._create_log(sketch_id, EventLevel.DEBUG, message)
- emit_event_task.apply(args=[str(log.id), str(sketch_id), EventLevel.DEBUG, message])
-
+ emit_event_task.apply(
+ args=[str(log.id), str(sketch_id), EventLevel.DEBUG, message]
+ )
+
@staticmethod
def graph_append(sketch_id: Union[str, UUID], message: Dict):
"""Log an insert event to the graph"""
log = Logger._create_log(sketch_id, EventLevel.GRAPH_APPEND, message)
- emit_event_task.apply(args=[str(log.id), str(sketch_id), EventLevel.GRAPH_APPEND, message])
+ emit_event_task.apply(
+ args=[str(log.id), str(sketch_id), EventLevel.GRAPH_APPEND, message]
+ )
@staticmethod
def success(sketch_id: Union[str, UUID], message: Dict):
"""Log a success message"""
log = Logger._create_log(sketch_id, EventLevel.SUCCESS, message)
- emit_event_task.apply(args=[str(log.id), str(sketch_id), EventLevel.SUCCESS, message])
-
+ emit_event_task.apply(
+ args=[str(log.id), str(sketch_id), EventLevel.SUCCESS, message]
+ )
+
@staticmethod
def completed(sketch_id: Union[str, UUID], message: Dict):
"""Log a completed message"""
log = Logger._create_log(sketch_id, EventLevel.COMPLETED, message)
- emit_event_task.apply(args=[str(log.id), str(sketch_id), EventLevel.COMPLETED, message])
-
+ emit_event_task.apply(
+ args=[str(log.id), str(sketch_id), EventLevel.COMPLETED, message]
+ )
+
@staticmethod
def pending(sketch_id: Union[str, UUID], message: Dict):
"""Log a pending message"""
log = Logger._create_log(sketch_id, EventLevel.PENDING, message)
- emit_event_task.apply(args=[str(log.id), str(sketch_id), EventLevel.PENDING, message])
+ emit_event_task.apply(
+ args=[str(log.id), str(sketch_id), EventLevel.PENDING, message]
+ )
diff --git a/flowsint-core/src/flowsint_core/core/models.py b/flowsint-core/src/flowsint_core/core/models.py
index b385335a..f12a8a18 100644
--- a/flowsint-core/src/flowsint_core/core/models.py
+++ b/flowsint-core/src/flowsint_core/core/models.py
@@ -1,13 +1,21 @@
import uuid
from datetime import datetime
from sqlalchemy import (
- String, Text, DateTime, ForeignKey,
- Index, func, JSON, Column, Enum as SQLEnum
+ String,
+ Text,
+ DateTime,
+ ForeignKey,
+ Index,
+ func,
+ JSON,
+ Column,
+ Enum as SQLEnum,
)
from sqlalchemy.dialects.postgresql import UUID as PGUUID, ARRAY, JSONB
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
from .enums import EventLevel
+
class Base(DeclarativeBase):
pass
@@ -18,19 +26,27 @@ class Feedback(Base):
id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
created_at = mapped_column(DateTime(timezone=True), server_default=func.now())
content = mapped_column(Text, nullable=True)
- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id"), nullable=True)
+ owner_id = mapped_column(
+ PGUUID(as_uuid=True), ForeignKey("profiles.id"), nullable=True
+ )
class Investigation(Base):
__tablename__ = "investigations"
- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+ id: Mapped[uuid.UUID] = mapped_column(
+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
+ )
created_at = mapped_column(DateTime(timezone=True), server_default=func.now())
name = mapped_column(Text)
description = mapped_column(Text)
- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)
+ owner_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),
+ nullable=True,
+ )
last_updated_at = mapped_column(DateTime(timezone=True), server_default=func.now())
- status = mapped_column(String, server_default='active')
+ status = mapped_column(String, server_default="active")
sketches = relationship("Sketch", back_populates="investigation")
analyses = relationship("Analysis", back_populates="investigation")
chats = relationship("Chat", back_populates="investigation")
@@ -45,32 +61,51 @@ class InvestigationsProfiles(Base):
id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
created_at = mapped_column(DateTime(timezone=True), server_default=func.now())
- investigation_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"))
- profile_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"))
+ investigation_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"),
+ )
+ profile_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),
+ )
role = mapped_column(String, server_default="member")
__table_args__ = (
Index("idx_investigations_profiles_investigation_id", "investigation_id"),
Index("idx_investigations_profiles_profile_id", "profile_id"),
# Uniqueness constraint
- Index("projects_profiles_unique_profile_project", "profile_id", "investigation_id", unique=True),
+ Index(
+ "projects_profiles_unique_profile_project",
+ "profile_id",
+ "investigation_id",
+ unique=True,
+ ),
)
class Log(Base):
__tablename__ = "logs"
- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+ id: Mapped[uuid.UUID] = mapped_column(
+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
+ )
content = mapped_column(JSONB, nullable=True)
created_at = mapped_column(DateTime(timezone=True), server_default=func.now())
- sketch_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)
+ sketch_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"),
+ nullable=True,
+ )
type = Column(SQLEnum(EventLevel), default=EventLevel.INFO)
class Profile(Base):
__tablename__ = "profiles"
- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+ id: Mapped[uuid.UUID] = mapped_column(
+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
+ )
first_name = mapped_column(Text, nullable=True)
last_name = mapped_column(Text, nullable=True)
avatar_url = mapped_column(Text, nullable=True)
@@ -82,17 +117,23 @@ class Profile(Base):
class Scan(Base):
__tablename__ = "scans"
- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
- sketch_id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)
+ id: Mapped[uuid.UUID] = mapped_column(
+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
+ )
+ sketch_id: Mapped[uuid.UUID] = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"),
+ nullable=True,
+ )
status = Column(SQLEnum(EventLevel), default=EventLevel.PENDING)
started_at = Column(DateTime, default=datetime.utcnow)
completed_at = Column(DateTime, nullable=True)
error = Column(Text, nullable=True)
details = Column(JSON, nullable=True)
-
+
# Relationships
sketch = relationship("Sketch", back_populates="scans")
-
+
def __repr__(self):
return f"<Scan(id={self.id}, status={self.status})>"
@@ -100,13 +141,22 @@ class Scan(Base):
class Sketch(Base):
__tablename__ = "sketches"
- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+ id: Mapped[uuid.UUID] = mapped_column(
+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
+ )
title = mapped_column(Text)
description = mapped_column(Text)
created_at = mapped_column(DateTime(timezone=True), server_default=func.now())
- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)
+ owner_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),
+ nullable=True,
+ )
status = mapped_column(String, server_default="active")
- investigation_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"))
+ investigation_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"),
+ )
investigation = relationship("Investigation", back_populates="sketches")
last_updated_at = mapped_column(DateTime(timezone=True), server_default=func.now())
scans = relationship("Scan", back_populates="sketch")
@@ -122,21 +172,34 @@ class SketchesProfiles(Base):
id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
created_at = mapped_column(DateTime(timezone=True), server_default=func.now())
- profile_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"))
- sketch_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"))
+ profile_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),
+ )
+ sketch_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"),
+ )
role = mapped_column(String, server_default="editor")
__table_args__ = (
Index("idx_sketches_profiles_sketch_id", "sketch_id"),
Index("idx_sketches_profiles_profile_id", "profile_id"),
- Index("investigations_profiles_unique_profile_investigation", "profile_id", "sketch_id", unique=True),
+ Index(
+ "investigations_profiles_unique_profile_investigation",
+ "profile_id",
+ "sketch_id",
+ unique=True,
+ ),
)
class Transform(Base):
__tablename__ = "transforms"
- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+ id: Mapped[uuid.UUID] = mapped_column(
+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
+ )
name = mapped_column(Text, nullable=False)
description = mapped_column(Text, nullable=True)
category = mapped_column(ARRAY(Text), nullable=True)
@@ -148,14 +211,24 @@ class Transform(Base):
class Analysis(Base):
__tablename__ = "analyses"
- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+ id: Mapped[uuid.UUID] = mapped_column(
+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
+ )
title = mapped_column(Text, nullable=False)
description = mapped_column(Text, nullable=True)
content = mapped_column(JSON, nullable=True)
created_at = mapped_column(DateTime(timezone=True), server_default=func.now())
last_updated_at = mapped_column(DateTime(timezone=True), server_default=func.now())
- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)
- investigation_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)
+ owner_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),
+ nullable=True,
+ )
+ investigation_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"),
+ nullable=True,
+ )
investigation = relationship("Investigation", back_populates="analyses")
__table_args__ = (
@@ -163,16 +236,27 @@ class Analysis(Base):
Index("idx_analyses_investigation_id", "investigation_id"),
)
+
class Chat(Base):
__tablename__ = "chats"
- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+ id: Mapped[uuid.UUID] = mapped_column(
+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
+ )
title = mapped_column(Text, nullable=False)
description = mapped_column(Text, nullable=True)
created_at = mapped_column(DateTime(timezone=True), server_default=func.now())
last_updated_at = mapped_column(DateTime(timezone=True), server_default=func.now())
- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)
- investigation_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)
+ owner_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),
+ nullable=True,
+ )
+ investigation_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"),
+ nullable=True,
+ )
investigation = relationship("Investigation", back_populates="chats")
messages = relationship("ChatMessage", back_populates="chat", cascade="all, delete")
__table_args__ = (
@@ -180,30 +264,42 @@ class Chat(Base):
Index("idx_chats_investigation_id", "investigation_id"),
)
+
class ChatMessage(Base):
__tablename__ = "messages"
- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+ id: Mapped[uuid.UUID] = mapped_column(
+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
+ )
content = mapped_column(JSON, nullable=True)
context = mapped_column(JSON, nullable=True)
is_bot: Mapped[bool] = mapped_column(default=False)
created_at = mapped_column(DateTime(timezone=True), server_default=func.now())
- chat_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("chats.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=False)
- chat = relationship("Chat", back_populates="messages")
- __table_args__ = (
- Index("idx_messages_chat_id", "chat_id"),
+ chat_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("chats.id", onupdate="CASCADE", ondelete="CASCADE"),
+ nullable=False,
)
+ chat = relationship("Chat", back_populates="messages")
+ __table_args__ = (Index("idx_messages_chat_id", "chat_id"),)
+
class Key(Base):
__tablename__ = "keys"
- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+ id: Mapped[uuid.UUID] = mapped_column(
+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
+ )
name = mapped_column(String, nullable=False)
- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)
+ owner_id = mapped_column(
+ PGUUID(as_uuid=True),
+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),
+ nullable=True,
+ )
encrypted_key = mapped_column(String, nullable=False)
created_at = mapped_column(DateTime(timezone=True), server_default=func.now())
__table_args__ = (
Index("idx_keys_owner_id", "owner_id"),
Index("idx_keys_service", "name"),
- )
\ No newline at end of file
+ )
diff --git a/flowsint-core/src/flowsint_core/core/orchestrator.py b/flowsint-core/src/flowsint_core/core/orchestrator.py
index fd584f14..ba8f8bc6 100644
--- a/flowsint-core/src/flowsint_core/core/orchestrator.py
+++ b/flowsint-core/src/flowsint_core/core/orchestrator.py
@@ -11,11 +11,20 @@ import asyncio
import json
import os
+
class TransformOrchestrator(Scanner):
"""
Orchestrator for running a list of scanners.
"""
- def __init__(self, sketch_id: str, scan_id: str, transform_branches: List[FlowBranch], neo4j_conn=None, vault=None):
+
+ def __init__(
+ self,
+ sketch_id: str,
+ scan_id: str,
+ transform_branches: List[FlowBranch],
+ neo4j_conn=None,
+ vault=None,
+ ):
super().__init__(sketch_id, scan_id, neo4j_conn=neo4j_conn, vault=vault)
self.transform_branches = transform_branches
self.scanners = {} # Map of nodeId -> scanner instance
@@ -31,14 +40,14 @@ class TransformOrchestrator(Scanner):
# Create a directory for storing transform files if it doesn't exist
transform_dir = "transform_logs"
os.makedirs(transform_dir, exist_ok=True)
-
+
# Create filename with sketch_id and scan_id
filename = f"transform_execution_{self.sketch_id}_{self.scan_id}.json"
self.execution_log_file = os.path.join(transform_dir, filename)
-
+
# Serialize the transform branches
serialized_branches = to_json_serializable(self.transform_branches)
-
+
# Create initial log structure
initial_log = {
"sketch_id": self.sketch_id,
@@ -52,11 +61,11 @@ class TransformOrchestrator(Scanner):
"total_steps": 0,
"completed_steps": 0,
"failed_steps": 0,
- "total_execution_time_ms": 0
+ "total_execution_time_ms": 0,
},
- "final_results": {}
+ "final_results": {},
}
-
+
# Count total steps
total_steps = 0
for branch in self.transform_branches:
@@ -64,55 +73,68 @@ class TransformOrchestrator(Scanner):
if step.type != "type":
total_steps += 1
initial_log["summary"]["total_steps"] = total_steps
-
+
# Save initial log file
- with open(self.execution_log_file, 'w', encoding='utf-8') as f:
+ with open(self.execution_log_file, "w", encoding="utf-8") as f:
json.dump(initial_log, f, indent=2, ensure_ascii=False)
-
- Logger.info(self.sketch_id, {"message": f"Transform execution log created at {self.execution_log_file}"})
-
+
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"Transform execution log created at {self.execution_log_file}"
+ },
+ )
+
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"Failed to create execution log: {str(e)}"})
+ Logger.error(
+ self.sketch_id, {"message": f"Failed to create execution log: {str(e)}"}
+ )
self.execution_log_file = None
- def _update_execution_log(self, step_entry: Dict[str, Any], status: str = None) -> None:
+ def _update_execution_log(
+ self, step_entry: Dict[str, Any], status: str = None
+ ) -> None:
"""
Update the execution log file with a new step entry or status update.
"""
if not self.execution_log_file:
return
-
+
try:
# Read current log
- with open(self.execution_log_file, 'r', encoding='utf-8') as f:
+ with open(self.execution_log_file, "r", encoding="utf-8") as f:
log_data = json.load(f)
-
+
# Update timestamp
log_data["updated_at"] = datetime.now().isoformat()
-
+
# Update status if provided
if status:
log_data["status"] = status
-
+
# Add step entry if provided
if step_entry:
log_data["execution_log"].append(step_entry)
-
+
# Update summary
if step_entry["status"] == "completed":
log_data["summary"]["completed_steps"] += 1
elif step_entry["status"] == "error":
log_data["summary"]["failed_steps"] += 1
-
+
if "execution_time_ms" in step_entry:
- log_data["summary"]["total_execution_time_ms"] += step_entry["execution_time_ms"]
-
+ log_data["summary"]["total_execution_time_ms"] += step_entry[
+ "execution_time_ms"
+ ]
+
# Write updated log
- with open(self.execution_log_file, 'w', encoding='utf-8') as f:
+ with open(self.execution_log_file, "w", encoding="utf-8") as f:
json.dump(log_data, f, indent=2, ensure_ascii=False)
-
+
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"Failed to update execution log: {str(e)}"})
+ Logger.error(
+ self.sketch_id, {"message": f"Failed to update execution log: {str(e)}"}
+ )
def _finalize_execution_log(self, final_results: Dict[str, Any]) -> None:
"""
@@ -120,25 +142,33 @@ class TransformOrchestrator(Scanner):
"""
if not self.execution_log_file:
return
-
+
try:
# Read current log
- with open(self.execution_log_file, 'r', encoding='utf-8') as f:
+ with open(self.execution_log_file, "r", encoding="utf-8") as f:
log_data = json.load(f)
-
+
# Update final data
log_data["updated_at"] = datetime.now().isoformat()
log_data["status"] = "completed"
log_data["final_results"] = to_json_serializable(final_results)
-
+
# Write final log
- with open(self.execution_log_file, 'w', encoding='utf-8') as f:
+ with open(self.execution_log_file, "w", encoding="utf-8") as f:
json.dump(log_data, f, indent=2, ensure_ascii=False)
-
- Logger.info(self.sketch_id, {"message": f"Transform execution log finalized at {self.execution_log_file}"})
-
+
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"Transform execution log finalized at {self.execution_log_file}"
+ },
+ )
+
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"Failed to finalize execution log: {str(e)}"})
+ Logger.error(
+ self.sketch_id,
+ {"message": f"Failed to finalize execution log: {str(e)}"},
+ )
def _save_transform_branches(self) -> None:
"""
@@ -148,27 +178,37 @@ class TransformOrchestrator(Scanner):
# Create a directory for storing transform files if it doesn't exist
transform_dir = "transform_logs"
os.makedirs(transform_dir, exist_ok=True)
-
+
# Create filename with sketch_id and scan_id
filename = f"transform_branches_{self.sketch_id}_{self.scan_id}.json"
filepath = os.path.join(transform_dir, filename)
-
+
# Serialize the transform branches
serialized_branches = to_json_serializable(self.transform_branches)
-
+
# Save to JSON file
- with open(filepath, 'w', encoding='utf-8') as f:
- json.dump({
- "sketch_id": self.sketch_id,
- "scan_id": self.scan_id,
- "timestamp": datetime.now().isoformat(),
- "transform_branches": serialized_branches
- }, f, indent=2, ensure_ascii=False)
-
- Logger.info(self.sketch_id, {"message": f"Transform branches saved to {filepath}"})
-
+ with open(filepath, "w", encoding="utf-8") as f:
+ json.dump(
+ {
+ "sketch_id": self.sketch_id,
+ "scan_id": self.scan_id,
+ "timestamp": datetime.now().isoformat(),
+ "transform_branches": serialized_branches,
+ },
+ f,
+ indent=2,
+ ensure_ascii=False,
+ )
+
+ Logger.info(
+ self.sketch_id, {"message": f"Transform branches saved to {filepath}"}
+ )
+
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"Failed to save transform branches: {str(e)}"})
+ Logger.error(
+ self.sketch_id,
+ {"message": f"Failed to save transform branches: {str(e)}"},
+ )
def _load_scanners(self) -> None:
if not self.transform_branches:
@@ -181,29 +221,31 @@ class TransformOrchestrator(Scanner):
if step.type == "type":
continue
scanner_nodes.append(step)
-
+
if not scanner_nodes:
raise ValueError("No scanner nodes found in transform branches")
-
+
# Create scanner instances for each node
for node in scanner_nodes:
node_id = node.nodeId
-
+
# Extract scanner name from nodeId (assuming format like "scanner_name-1234567890")
- scanner_name = node_id.split('-')[0]
-
+ scanner_name = node_id.split("-")[0]
+
if not ScannerRegistry.scanner_exists(scanner_name):
raise ValueError(f"Scanner '{scanner_name}' not found in registry")
# Pass the step params to the scanner instance
- scanner_params = node.params if hasattr(node, 'params') and node.params else {}
+ scanner_params = (
+ node.params if hasattr(node, "params") and node.params else {}
+ )
scanner = ScannerRegistry.get_scanner(
- scanner_name,
- self.sketch_id,
- self.scan_id,
+ scanner_name,
+ self.sketch_id,
+ self.scan_id,
neo4j_conn=self.neo4j_conn,
vault=self.vault,
- params=scanner_params
+ params=scanner_params,
)
self.scanners[node_id] = scanner
@@ -216,13 +258,15 @@ class TransformOrchestrator(Scanner):
return results_mapping[ref_value]
return None
- def prepare_scanner_inputs(self, step: FlowStep, results_mapping: Dict[str, Any], initial_values: List[str]) -> List[Any]:
+ def prepare_scanner_inputs(
+ self, step: FlowStep, results_mapping: Dict[str, Any], initial_values: List[str]
+ ) -> List[Any]:
"""
Prepare the inputs for a scanner based on the references and previous results.
Handles single references, lists, and direct values.
"""
inputs = {}
-
+
for input_key, input_ref in step.inputs.items():
# Cas 1 : une seule référence (string)
if isinstance(input_ref, str):
@@ -251,10 +295,15 @@ class TransformOrchestrator(Scanner):
primary_key = scanner.key()
return {primary_key: initial_values}
else:
- scanner = self.scanners.get(step.nodeId)
+ scanner = self.scanners.get(step.nodeId)
return inputs[input_key]
- def update_results_mapping(self, outputs: Dict[str, Any], step_outputs: Dict[str, str], results_mapping: Dict[str, Any]) -> None:
+ def update_results_mapping(
+ self,
+ outputs: Dict[str, Any],
+ step_outputs: Dict[str, str],
+ results_mapping: Dict[str, Any],
+ ) -> None:
"""
Update the results mapping with new outputs from a scanner.
"""
@@ -280,10 +329,7 @@ class TransformOrchestrator(Scanner):
@classmethod
def output_schema(cls) -> Dict[str, str]:
- return {
- "branches": "array",
- "results": "dict"
- }
+ return {"branches": "array", "results": "dict"}
def scan(self, values: List[str]) -> Dict[str, Any]:
"""
@@ -303,54 +349,49 @@ class TransformOrchestrator(Scanner):
"""
# Update execution log to indicate scan has started
self._update_execution_log(None, "running")
-
- results = {
- "initial_values": values,
- "branches": [],
- "results": {}
- }
+
+ results = {"initial_values": values, "branches": [], "results": {}}
Logger.pending(self.sketch_id, {"message": "Starting transform..."})
-
+
# Global mapping of output references to actual values
results_mapping = {}
# Cache for scanner results to avoid recomputation
scanner_results_cache = {}
-
+
total_steps = sum(len(branch.steps) for branch in self.transform_branches)
completed_steps = 0
-
+
# Process each branch
for branch in self.transform_branches:
branch_id = branch.id
branch_name = branch.name
- branch_results = {
- "id": branch_id,
- "name": branch_name,
- "steps": []
- }
-
+ branch_results = {"id": branch_id, "name": branch_name, "steps": []}
+
# Process each step in the branch
scanner_inputs = values
for step in branch.steps:
if step.type == "type":
continue
-
+
node_id = step.nodeId
scanner = self.scanners.get(node_id)
-
+
if not scanner:
- Logger.error(self.sketch_id, {"message": f"Scanner not found for node {node_id}"})
+ Logger.error(
+ self.sketch_id,
+ {"message": f"Scanner not found for node {node_id}"},
+ )
continue
-
+
scanner_name = scanner.name()
step_start_time = time.time()
-
+
step_result = {
"nodeId": node_id,
"scanner": scanner_name,
- "status": "error" # Default to error, will update on success
+ "status": "error", # Default to error, will update on success
}
-
+
# Create execution log entry
log_entry = {
"step_id": f"{branch_id}_{node_id}",
@@ -363,9 +404,9 @@ class TransformOrchestrator(Scanner):
"status": "running",
"error": None,
"timestamp": datetime.now().isoformat(),
- "execution_time_ms": 0
+ "execution_time_ms": 0,
}
-
+
try:
# Update status for current step
completed_steps += 1
@@ -374,11 +415,13 @@ class TransformOrchestrator(Scanner):
step_result["error"] = error_msg
log_entry["status"] = "error"
log_entry["error"] = error_msg
- log_entry["execution_time_ms"] = int((time.time() - step_start_time) * 1000)
+ log_entry["execution_time_ms"] = int(
+ (time.time() - step_start_time) * 1000
+ )
self._update_execution_log(log_entry)
branch_results["steps"].append(step_result)
continue
-
+
# Check if we already have results for this scanner with these inputs
cache_key = f"{node_id}:{str(scanner_inputs)}"
if cache_key in scanner_results_cache:
@@ -388,20 +431,24 @@ class TransformOrchestrator(Scanner):
# Execute the scanner
outputs = await scanner.execute(scanner_inputs)
if not isinstance(outputs, (dict, list)):
- raise ValueError(f"Scanner '{scanner_name}' returned unsupported output format")
+ raise ValueError(
+ f"Scanner '{scanner_name}' returned unsupported output format"
+ )
# Cache the results
scanner_results_cache[cache_key] = outputs
log_entry["cache_hit"] = False
-
+
# Store the outputs in the step result (serialize to avoid JSON issues)
step_result["outputs"] = to_json_serializable(outputs)
step_result["status"] = "completed"
-
+
# Update log entry with success
log_entry["outputs"] = to_json_serializable(outputs)
log_entry["status"] = "completed"
- log_entry["execution_time_ms"] = int((time.time() - step_start_time) * 1000)
-
+ log_entry["execution_time_ms"] = int(
+ (time.time() - step_start_time) * 1000
+ )
+
# Update the global results mapping with the outputs
self.update_results_mapping(outputs, step.outputs, results_mapping)
# Also store the raw outputs in the main results
@@ -414,35 +461,40 @@ class TransformOrchestrator(Scanner):
step_result["error"] = error_msg
log_entry["status"] = "error"
log_entry["error"] = error_msg
- log_entry["execution_time_ms"] = int((time.time() - step_start_time) * 1000)
+ log_entry["execution_time_ms"] = int(
+ (time.time() - step_start_time) * 1000
+ )
results["results"][node_id] = {"error": error_msg}
self._update_execution_log(log_entry)
return results
-
+
except Exception as e:
error_msg = f"Error during scan: {str(e)}"
- Logger.error(self.sketch_id, {"message":error_msg})
+ Logger.error(self.sketch_id, {"message": error_msg})
step_result["error"] = error_msg
log_entry["status"] = "error"
log_entry["error"] = error_msg
- log_entry["execution_time_ms"] = int((time.time() - step_start_time) * 1000)
+ log_entry["execution_time_ms"] = int(
+ (time.time() - step_start_time) * 1000
+ )
results["results"][node_id] = {"error": error_msg}
self._update_execution_log(log_entry)
return results
-
+
# Update execution log with this step
self._update_execution_log(log_entry)
branch_results["steps"].append(step_result)
-
+
results["branches"].append(branch_results)
-
- Logger.completed(self.sketch_id, {"message": "Transform completed successfully."})
-
+
+ Logger.completed(
+ self.sketch_id, {"message": "Transform completed successfully."}
+ )
+
# Include the final reference mapping for debugging
results["reference_mapping"] = results_mapping
-
+
# Finalize execution log
self._finalize_execution_log(results)
-
+
return results
-
\ No newline at end of file
diff --git a/flowsint-core/src/flowsint_core/core/postgre_db.py b/flowsint-core/src/flowsint_core/core/postgre_db.py
index bb4bf25e..516dea1a 100644
--- a/flowsint-core/src/flowsint_core/core/postgre_db.py
+++ b/flowsint-core/src/flowsint_core/core/postgre_db.py
@@ -19,4 +19,4 @@ def get_db() -> Session:
try:
yield db
finally:
- db.close()
\ No newline at end of file
+ db.close()
diff --git a/flowsint-core/src/flowsint_core/core/registry.py b/flowsint-core/src/flowsint_core/core/registry.py
index 8ac501b8..eaa053bf 100644
--- a/flowsint-core/src/flowsint_core/core/registry.py
+++ b/flowsint-core/src/flowsint_core/core/registry.py
@@ -1,61 +1,47 @@
import inspect
-from typing import Dict, Type, List, Optional
-from .scanner_base import Scanner
+from typing import Dict, Type, List
+from flowsint_core.core.scanner_base import Scanner
+from flowsint_transforms.domains.subdomains import SubdomainScanner
+from flowsint_transforms.domains.whois import WhoisScanner
+from flowsint_transforms.domains.resolve import ResolveScanner
+from flowsint_transforms.ips.reverse_resolve import ReverseResolveScanner
+from flowsint_transforms.ips.geolocation import GeolocationScanner
+from flowsint_transforms.socials.maigret import MaigretScanner
+from flowsint_transforms.ips.ip_to_asn import IpToAsnScanner
+from flowsint_transforms.ips.asn_to_cidrs import AsnToCidrsScanner
+from flowsint_transforms.ips.cidr_to_ips import CidrToIpsScanner
+from flowsint_transforms.organizations.org_to_asn import OrgToAsnScanner
+from flowsint_transforms.domains.domain_to_asn import DomainToAsnScanner
+from flowsint_transforms.crypto.wallet_to_transactions import (
+ CryptoWalletAddressToTransactions,
+)
+from flowsint_transforms.crypto.wallet_to_nfts import CryptoWalletAddressToNFTs
+from flowsint_transforms.domains.to_website import DomainToWebsiteScanner
+from flowsint_transforms.websites.to_crawler import WebsiteToCrawler
+from flowsint_transforms.websites.to_links import WebsiteToLinks
+from flowsint_transforms.websites.to_domain import WebsiteToDomainScanner
+from flowsint_transforms.emails.to_gravatar import EmailToGravatarScanner
+from flowsint_transforms.emails.to_leaks import EmailToBreachesScanner
+from flowsint_transforms.individuals.to_org import IndividualToOrgScanner
+from flowsint_transforms.organizations.to_infos import OrgToInfosScanner
+from flowsint_transforms.websites.to_webtrackers import WebsiteToWebtrackersScanner
+from flowsint_transforms.n8n.connector import N8nConnector
class ScannerRegistry:
- """
- Registry for managing scanner discovery and instantiation.
- Uses a dynamic discovery system to avoid circular dependencies.
- """
-
+
_scanners: Dict[str, Type[Scanner]] = {}
- _scanner_modules: List[str] = []
-
+
@classmethod
def register(cls, scanner_class: Type[Scanner]) -> None:
- """Register a scanner class in the registry."""
- if not issubclass(scanner_class, Scanner):
- raise ValueError(f"Class {scanner_class.__name__} must inherit from Scanner")
-
- scanner_name = scanner_class.name()
- cls._scanners[scanner_name] = scanner_class
-
- @classmethod
- def register_module(cls, module_path: str) -> None:
- """Register a module path for dynamic scanner discovery."""
- if module_path not in cls._scanner_modules:
- cls._scanner_modules.append(module_path)
-
- @classmethod
- def discover_scanners(cls) -> None:
- """Dynamically discover and register scanners from registered modules."""
- import importlib
- import pkgutil
-
- for module_path in cls._scanner_modules:
- try:
- module = importlib.import_module(module_path)
-
- # Walk through the module to find scanner classes
- for name, obj in inspect.getmembers(module):
- if (inspect.isclass(obj) and
- issubclass(obj, Scanner) and
- obj != Scanner and
- hasattr(obj, 'name')):
- cls.register(obj)
-
- except ImportError as e:
- print(f"Warning: Could not import module {module_path}: {e}")
-
+ cls._scanners[scanner_class.name()] = scanner_class
+
@classmethod
def scanner_exists(cls, name: str) -> bool:
- """Check if a scanner with the given name exists."""
return name in cls._scanners
-
+
@classmethod
def get_scanner(cls, name: str, sketch_id: str, scan_id: str, **kwargs) -> Scanner:
- """Get a scanner instance by name."""
if name not in cls._scanners:
raise Exception(f"Scanner '{name}' not found")
return cls._scanners[name](sketch_id=sketch_id, scan_id=scan_id, **kwargs)
@@ -80,15 +66,13 @@ class ScannerRegistry:
@classmethod
def list(cls) -> Dict[str, Dict[str, str]]:
- """List all registered scanners with their metadata."""
return {
name: cls._create_scanner_metadata(scanner)
for name, scanner in cls._scanners.items()
}
-
+
@classmethod
def list_by_categories(cls) -> Dict[str, List[Dict[str, str]]]:
- """List scanners grouped by category."""
scanners_by_category = {}
for _, scanner in cls._scanners.items():
category = scanner.category()
@@ -96,23 +80,45 @@ class ScannerRegistry:
scanners_by_category[category] = []
scanners_by_category[category].append(cls._create_scanner_metadata(scanner))
return scanners_by_category
-
+
@classmethod
def list_by_input_type(cls, input_type: str) -> List[Dict[str, str]]:
- """List scanners that accept a specific input type."""
input_type_lower = input_type.lower()
-
+
if input_type_lower == "any":
- return [cls._create_scanner_metadata(scanner) for scanner in cls._scanners.values()]
-
+ return [
+ cls._create_scanner_metadata(scanner)
+ for scanner in cls._scanners.values()
+ ]
+
return [
cls._create_scanner_metadata(scanner)
for scanner in cls._scanners.values()
if scanner.input_schema()["type"].lower() in ["any", input_type_lower]
]
-
- @classmethod
- def clear(cls) -> None:
- """Clear all registered scanners (mainly for testing)."""
- cls._scanners.clear()
- cls._scanner_modules.clear()
+
+
+# Register all scanners
+ScannerRegistry.register(ReverseResolveScanner)
+ScannerRegistry.register(ResolveScanner)
+ScannerRegistry.register(SubdomainScanner)
+ScannerRegistry.register(WhoisScanner)
+ScannerRegistry.register(GeolocationScanner)
+ScannerRegistry.register(MaigretScanner)
+ScannerRegistry.register(IpToAsnScanner)
+ScannerRegistry.register(AsnToCidrsScanner)
+ScannerRegistry.register(CidrToIpsScanner)
+ScannerRegistry.register(OrgToAsnScanner)
+ScannerRegistry.register(DomainToAsnScanner)
+ScannerRegistry.register(CryptoWalletAddressToTransactions)
+ScannerRegistry.register(CryptoWalletAddressToNFTs)
+ScannerRegistry.register(DomainToWebsiteScanner)
+ScannerRegistry.register(WebsiteToCrawler)
+ScannerRegistry.register(WebsiteToLinks)
+ScannerRegistry.register(WebsiteToDomainScanner)
+ScannerRegistry.register(EmailToGravatarScanner)
+ScannerRegistry.register(EmailToBreachesScanner)
+ScannerRegistry.register(IndividualToOrgScanner)
+ScannerRegistry.register(OrgToInfosScanner)
+ScannerRegistry.register(WebsiteToWebtrackersScanner)
+ScannerRegistry.register(N8nConnector)
diff --git a/flowsint-core/src/flowsint_core/core/scanner_base.py b/flowsint-core/src/flowsint_core/core/scanner_base.py
index d8b1b56f..013fc53e 100644
--- a/flowsint-core/src/flowsint_core/core/scanner_base.py
+++ b/flowsint-core/src/flowsint_core/core/scanner_base.py
@@ -7,9 +7,11 @@ from .logger import Logger
from .vault import VaultProtocol
from ..utils import resolve_type
+
class InvalidScannerParams(Exception):
pass
+
def build_params_model(params_schema: list) -> BaseModel:
"""
Build a strict Pydantic model from a params_schema.
@@ -22,75 +24,75 @@ def build_params_model(params_schema: list) -> BaseModel:
type = str # You can later enhance this to support int, bool, etc.
required = param.get("required", False)
default = ... if required else param.get("default")
- fields[name] = (Optional[type], Field(default=default, description=param.get("description", "")))
+ fields[name] = (
+ Optional[type],
+ Field(default=default, description=param.get("description", "")),
+ )
- model = create_model(
- "ParamsModel",
- __config__=ConfigDict(extra="forbid"),
- **fields
- )
+ model = create_model("ParamsModel", __config__=ConfigDict(extra="forbid"), **fields)
return model
+
class Scanner(ABC):
"""
Abstract base class for all scanners.
-
+
## InputType and OutputType Pattern
-
+
Scanners only need to define InputType and OutputType as class attributes.
The base class automatically handles schema generation:
-
+
```python
from typing import List
from flowsint_types import Domain
from flowsint_types import Ip
-
+
class MyScanner(Scanner):
# Define types as class attributes
InputType = List[Domain]
OutputType = List[Ip]
-
+
@classmethod
def name(cls):
return "my_scanner"
-
+
@classmethod
def category(cls):
return "Domain"
-
+
@classmethod
def key(cls):
return "domain"
-
+
def preprocess(self, data: InputType) -> InputType:
cleaned: InputType = []
# ... implementation
return cleaned
-
+
async def scan(self, data: InputType) -> OutputType:
results: OutputType = []
# ... implementation
return results
-
+
# Make types available at module level for easy access
InputType = MyScanner.InputType
OutputType = MyScanner.OutputType
```
-
+
The base class automatically provides:
- input_schema() method using InputType
- output_schema() method using OutputType
- Error handling for missing type definitions
- Consistent schema generation across all scanners
-
+
Subclasses can override input_schema() or output_schema() if needed for special cases.
"""
-
+
# Abstract type aliases that must be defined in subclasses for runtime use
InputType = NotImplemented
OutputType = NotImplemented
-
+
def __init__(
self,
sketch_id: Optional[str] = None,
@@ -98,7 +100,7 @@ class Scanner(ABC):
neo4j_conn: Optional[Neo4jConnection] = None,
params_schema: Optional[List[Dict[str, Any]]] = None,
vault: Optional[VaultProtocol] = None,
- params: Optional[Dict[str, Any]] = None
+ params: Optional[Dict[str, Any]] = None,
):
self.scan_id = scan_id or "default"
self.sketch_id = sketch_id or "system"
@@ -107,7 +109,7 @@ class Scanner(ABC):
self.params_schema = params_schema or []
self.ParamsModel = build_params_model(self.params_schema)
self.params: Dict[str, Any] = params or {}
- # Params is filled synchronously by the constructor. This params is generally constructed of
+ # Params is filled synchronously by the constructor. This params is generally constructed of
# vaultSecret references, not the key directly. The idea is that the real key values are resolved after calling
# async_init(), right before the execution.
@@ -116,7 +118,9 @@ class Scanner(ABC):
# Resolve parameters (e.g. replace vaultSecret by real secrets)
if self.params:
resolved_params = self.resolve_params()
- Logger.debug(self.sketch_id, {"message": f"Resolved params: {str(resolved_params)}"})
+ Logger.debug(
+ self.sketch_id, {"message": f"Resolved params: {str(resolved_params)}"}
+ )
else:
resolved_params = {}
# Strict validation after resolution
@@ -124,13 +128,20 @@ class Scanner(ABC):
validated = self.ParamsModel(**resolved_params)
self.params = validated.model_dump()
except ValidationError as e:
- raise InvalidScannerParams(f"Scanner '{self.name()}' received invalid parameters: {e}")
+ raise InvalidScannerParams(
+ f"Scanner '{self.name()}' received invalid parameters: {e}"
+ )
def resolve_params(self) -> Dict[str, Any]:
resolved = {}
- Logger.warn(self.sketch_id, {"message": f"Params schema: {str(self.params_schema)}"})
+ Logger.warn(
+ self.sketch_id, {"message": f"Params schema: {str(self.params_schema)}"}
+ )
Logger.warn(self.sketch_id, {"message": f"Params: {str(self.params)}"})
- Logger.warn(self.sketch_id, {"message": f"Params schema length: {len(self.params_schema)}"})
+ Logger.warn(
+ self.sketch_id,
+ {"message": f"Params schema length: {len(self.params_schema)}"},
+ )
i = 1
for param in self.params_schema:
Logger.warn(self.sketch_id, {"message": f"Param {i}: {str(param)}"})
@@ -160,16 +171,15 @@ class Scanner(ABC):
continue
return resolved
-
@classmethod
def required_params(self) -> bool:
- return False
+ return False
@classmethod
@abstractmethod
def name(cls) -> str:
pass
-
+
@classmethod
def icon(cls) -> str | None:
return None
@@ -193,6 +203,7 @@ class Scanner(ABC):
Falls back to cleaned docstring if not overridden.
"""
import inspect
+
return inspect.cleandoc(cls.__doc__ or "No documentation available.")
@classmethod
@@ -215,21 +226,21 @@ class Scanner(ABC):
Subclasses don't need to override this unless they have special requirements.
"""
return cls.generate_output_schema()
-
+
@classmethod
def generate_input_schema(cls) -> Dict[str, Any]:
"""
Helper method to generate input schema from InputType class attribute.
-
+
Raises:
NotImplementedError: If InputType is not defined in the subclass
"""
if cls.InputType is NotImplemented:
raise NotImplementedError(f"InputType must be defined in {cls.__name__}")
-
+
adapter = TypeAdapter(cls.InputType)
schema = adapter.json_schema()
-
+
# Handle different schema structures
if "$defs" in schema and schema["$defs"]:
# Follow the $ref in items to get the correct type (not just the first one)
@@ -241,35 +252,35 @@ class Scanner(ABC):
else:
# Fallback: get the first type definition (for backward compatibility)
type_name, details = list(schema["$defs"].items())[0]
-
+
return {
"type": type_name,
"properties": [
{"name": prop, "type": resolve_type(info, schema)}
for prop, info in details["properties"].items()
- ]
+ ],
}
else:
# Handle simpler schemas
return {
"type": schema.get("title", "Any"),
- "properties": [{"name": "value", "type": "object"}]
+ "properties": [{"name": "value", "type": "object"}],
}
-
- @classmethod
+
+ @classmethod
def generate_output_schema(cls) -> Dict[str, Any]:
"""
Helper method to generate output schema from OutputType class attribute.
-
+
Raises:
NotImplementedError: If OutputType is not defined in the subclass
"""
if cls.OutputType is NotImplemented:
raise NotImplementedError(f"OutputType must be defined in {cls.__name__}")
-
+
adapter = TypeAdapter(cls.OutputType)
schema = adapter.json_schema()
-
+
# Handle different schema structures
if "$defs" in schema and schema["$defs"]:
# Follow the $ref in items to get the correct type (not just the first one)
@@ -281,19 +292,19 @@ class Scanner(ABC):
else:
# Fallback: get the first type definition (for backward compatibility)
type_name, details = list(schema["$defs"].items())[0]
-
+
return {
"type": type_name,
"properties": [
{"name": prop, "type": resolve_type(info, schema)}
for prop, info in details["properties"].items()
- ]
+ ],
}
else:
# Handle simpler schemas
return {
"type": schema.get("title", "Any"),
- "properties": [{"name": "value", "type": "object"}]
+ "properties": [{"name": "value", "type": "object"}],
}
@abstractmethod
@@ -309,7 +320,9 @@ class Scanner(ABC):
def preprocess(self, values: List[str]) -> List[str]:
return values
- def postprocess(self, results: List[Dict[str, Any]], input_data: List[str] = None) -> List[Dict[str, Any]]:
+ def postprocess(
+ self, results: List[Dict[str, Any]], input_data: List[str] = None
+ ) -> List[Dict[str, Any]]:
return results
async def execute(self, values: List[str]) -> List[Dict[str, Any]]:
@@ -322,51 +335,68 @@ class Scanner(ABC):
processed = self.postprocess(results, preprocessed)
if self.name() != "transform_orchestrator":
- Logger.completed(self.sketch_id, {"message": f"Scanner {self.name()} finished."})
+ Logger.completed(
+ self.sketch_id, {"message": f"Scanner {self.name()} finished."}
+ )
return processed
except Exception as e:
if self.name() != "transform_orchestrator":
- Logger.error(self.sketch_id, {"message": f"Scanner {self.name()} errored: '{str(e)}'."})
+ Logger.error(
+ self.sketch_id,
+ {"message": f"Scanner {self.name()} errored: '{str(e)}'."},
+ )
return []
- def create_node(self, node_type: str, key_prop: str, key_value: str, **properties) -> None:
+ def create_node(
+ self, node_type: str, key_prop: str, key_value: str, **properties
+ ) -> None:
"""Simple helper to create a single Neo4j node."""
if not self.neo4j_conn:
return
-
- properties['sketch_id'] = self.sketch_id
- properties['label'] = properties.get('label', key_value)
-
+
+ properties["sketch_id"] = self.sketch_id
+ properties["label"] = properties.get("label", key_value)
+
set_clauses = [f"n.{prop} = ${prop}" for prop in properties.keys()]
params = {key_prop: key_value, **properties}
-
+
query = f"""
MERGE (n:{node_type} {{{key_prop}: ${key_prop}}})
SET {', '.join(set_clauses)}
"""
self.neo4j_conn.query(query, params)
-
- def create_relationship(self, from_type: str, from_key: str, from_value: str,
- to_type: str, to_key: str, to_value: str, rel_type: str) -> None:
+
+ def create_relationship(
+ self,
+ from_type: str,
+ from_key: str,
+ from_value: str,
+ to_type: str,
+ to_key: str,
+ to_value: str,
+ rel_type: str,
+ ) -> None:
"""Simple helper to create a relationship between two nodes."""
if not self.neo4j_conn:
return
-
+
query = f"""
MATCH (from:{from_type} {{{from_key}: $from_value}})
MATCH (to:{to_type} {{{to_key}: $to_value}})
MERGE (from)-[:{rel_type} {{sketch_id: $sketch_id}}]->(to)
"""
-
- self.neo4j_conn.query(query, {
- 'from_value': from_value,
- 'to_value': to_value,
- 'sketch_id': self.sketch_id
- })
-
+
+ self.neo4j_conn.query(
+ query,
+ {
+ "from_value": from_value,
+ "to_value": to_value,
+ "sketch_id": self.sketch_id,
+ },
+ )
+
def log_graph_message(self, message: str) -> None:
"""Simple helper to log a graph message."""
Logger.graph_append(self.sketch_id, {"message": message})
-
diff --git a/flowsint-core/src/flowsint_core/core/types.py b/flowsint-core/src/flowsint_core/core/types.py
index b221df47..c32c0fe7 100644
--- a/flowsint-core/src/flowsint_core/core/types.py
+++ b/flowsint-core/src/flowsint_core/core/types.py
@@ -4,42 +4,89 @@ from datetime import datetime
from typing import Dict, List, Any, Optional, Literal
from .enums import EventLevel
+
class Event(BaseModel):
id: str = Field(..., description="Unique identifier for the event")
sketch_id: str = Field(..., description="ID of the sketch")
type: EventLevel = Field(..., description="Type of event")
payload: Dict[str, Any] = Field(..., description="Payload of the event")
- created_at: datetime = Field(default_factory=datetime.now, description="Timestamp when the event was created")
+ created_at: datetime = Field(
+ default_factory=datetime.now, description="Timestamp when the event was created"
+ )
+
class Node(BaseModel):
"""Represents a node in a transformation flow with position and data."""
+
id: str = Field(..., description="Unique identifier for the node", title="Node ID")
- data: Dict[str, Any] = Field(..., description="Data payload associated with the node", title="Node Data")
- position: Optional[Dict[str, float]] = Field(None, description="X and Y coordinates for node positioning", title="Position")
- type: Optional[str] = Field(None, description="Type of node (e.g., input, output, process)", title="Node Type")
-
+ data: Dict[str, Any] = Field(
+ ..., description="Data payload associated with the node", title="Node Data"
+ )
+ position: Optional[Dict[str, float]] = Field(
+ None, description="X and Y coordinates for node positioning", title="Position"
+ )
+ type: Optional[str] = Field(
+ None,
+ description="Type of node (e.g., input, output, process)",
+ title="Node Type",
+ )
+
+
class Edge(BaseModel):
"""Represents an edge connecting two nodes in a transformation flow."""
+
id: str = Field(..., description="Unique identifier for the edge", title="Edge ID")
- data: Optional[Dict[str, Any]] = Field(None, description="Data payload associated with the edge", title="Edge Data")
+ data: Optional[Dict[str, Any]] = Field(
+ None, description="Data payload associated with the edge", title="Edge Data"
+ )
source: str = Field(..., description="ID of the source node", title="Source Node")
target: str = Field(..., description="ID of the target node", title="Target Node")
- sourceHandle: Optional[str] = Field(None, description="Handle ID on the source node", title="Source Handle")
- targetHandle: Optional[str] = Field(None, description="Handle ID on the target node", title="Target Handle")
+ sourceHandle: Optional[str] = Field(
+ None, description="Handle ID on the source node", title="Source Handle"
+ )
+ targetHandle: Optional[str] = Field(
+ None, description="Handle ID on the target node", title="Target Handle"
+ )
+
class FlowStep(BaseModel):
"""Represents a single step in a transformation flow with execution status."""
+
nodeId: str = Field(..., description="ID of the associated node", title="Node ID")
- params: Optional[Dict[str, Any]] = Field(None, description="Parameters for the step", title="Parameters")
- type: Literal["type", "scanner"] = Field(..., description="Type of step - either type transformation or scanner", title="Step Type")
- inputs: Dict[str, Any] = Field(..., description="Input data for this step", title="Inputs")
- outputs: Dict[str, Any] = Field(..., description="Output data from this step", title="Outputs")
- status: Literal["pending", "processing", "completed", "error"] = Field(..., description="Current execution status of the step", title="Status")
- branchId: str = Field(..., description="ID of the branch this step belongs to", title="Branch ID")
- depth: int = Field(..., description="Depth level of this step in the flow", title="Depth")
+ params: Optional[Dict[str, Any]] = Field(
+ None, description="Parameters for the step", title="Parameters"
+ )
+ type: Literal["type", "scanner"] = Field(
+ ...,
+ description="Type of step - either type transformation or scanner",
+ title="Step Type",
+ )
+ inputs: Dict[str, Any] = Field(
+ ..., description="Input data for this step", title="Inputs"
+ )
+ outputs: Dict[str, Any] = Field(
+ ..., description="Output data from this step", title="Outputs"
+ )
+ status: Literal["pending", "processing", "completed", "error"] = Field(
+ ..., description="Current execution status of the step", title="Status"
+ )
+ branchId: str = Field(
+ ..., description="ID of the branch this step belongs to", title="Branch ID"
+ )
+ depth: int = Field(
+ ..., description="Depth level of this step in the flow", title="Depth"
+ )
+
class FlowBranch(BaseModel):
"""Represents a branch containing a sequence of transformation steps."""
- id: str = Field(..., description="Unique identifier for the branch", title="Branch ID")
- name: str = Field(..., description="Human-readable name of the branch", title="Branch Name")
- steps: List[FlowStep] = Field(..., description="List of steps in this branch", title="Steps")
\ No newline at end of file
+
+ id: str = Field(
+ ..., description="Unique identifier for the branch", title="Branch ID"
+ )
+ name: str = Field(
+ ..., description="Human-readable name of the branch", title="Branch Name"
+ )
+ steps: List[FlowStep] = Field(
+ ..., description="List of steps in this branch", title="Steps"
+ )
diff --git a/flowsint-core/src/flowsint_core/core/vault.py b/flowsint-core/src/flowsint_core/core/vault.py
index fad08029..10f51e91 100644
--- a/flowsint-core/src/flowsint_core/core/vault.py
+++ b/flowsint-core/src/flowsint_core/core/vault.py
@@ -4,9 +4,10 @@ from sqlalchemy.orm import Session
from sqlalchemy import select
from .models import Key
+
class VaultProtocol(Protocol):
- def get_secret(self, vault_ref: str) -> Optional[str]:
- ...
+ def get_secret(self, vault_ref: str) -> Optional[str]: ...
+
class Vault(VaultProtocol):
def __init__(self, db: Session, owner_id: uuid.UUID):
diff --git a/flowsint-core/src/flowsint_core/tasks/event.py b/flowsint-core/src/flowsint_core/tasks/event.py
index 767f1d9e..2a2a6bb3 100644
--- a/flowsint-core/src/flowsint_core/tasks/event.py
+++ b/flowsint-core/src/flowsint_core/tasks/event.py
@@ -9,17 +9,17 @@ from ..core.types import Event
logger = logging.getLogger(__name__)
+
@celery.task(name="emit_event")
def emit_event_task(log_id: str, sketch_id: str, log_type: EventLevel, content: Dict):
"""Celery task to emit a log event"""
try:
event = Event(
- id=log_id,
- sketch_id=sketch_id,
- type=log_type,
- payload=content
+ id=log_id, sketch_id=sketch_id, type=log_type, payload=content
).model_dump_json()
- redis_client = redis.from_url(os.getenv("REDIS_URI", "redis://127.0.0.1:6379/0"))
+ redis_client = redis.from_url(
+ os.getenv("REDIS_URI", "redis://127.0.0.1:6379/0")
+ )
redis_client.publish(sketch_id, event)
except Exception as e:
raise
diff --git a/flowsint-core/src/flowsint_core/tasks/transform.py b/flowsint-core/src/flowsint_core/tasks/transform.py
index b81ed304..600f1fda 100644
--- a/flowsint-core/src/flowsint_core/tasks/transform.py
+++ b/flowsint-core/src/flowsint_core/tasks/transform.py
@@ -14,10 +14,11 @@ from sqlalchemy.orm import Session
from ..core.logger import Logger
from ..core.enums import EventLevel
from flowsint_core.utils import to_json_serializable
+
load_dotenv()
URI = os.getenv("NEO4J_URI_BOLT")
-URI="bolt://localhost:7687"
+URI = "bolt://localhost:7687"
USERNAME = os.getenv("NEO4J_USERNAME")
PASSWORD = os.getenv("NEO4J_PASSWORD")
@@ -25,8 +26,15 @@ neo4j_connection = Neo4jConnection(URI, USERNAME, PASSWORD)
db: Session = next(get_db())
logger = Logger()
+
@celery.task(name="run_transform", bind=True)
-def run_transform(self, transform_branches, values: List[str], sketch_id: str | None, owner_id: Optional[str] = None):
+def run_transform(
+ self,
+ transform_branches,
+ values: List[str],
+ sketch_id: str | None,
+ owner_id: Optional[str] = None,
+):
session = SessionLocal()
try:
@@ -49,7 +57,9 @@ def run_transform(self, transform_branches, values: List[str], sketch_id: str |
try:
vault = Vault(session, uuid.UUID(owner_id))
except Exception as e:
- Logger.error(sketch_id, {"message": f"Failed to create vault: {str(e)}"})
+ Logger.error(
+ sketch_id, {"message": f"Failed to create vault: {str(e)}"}
+ )
transform_branches = [FlowBranch(**branch) for branch in transform_branches]
scanner = TransformOrchestrator(
@@ -59,7 +69,7 @@ def run_transform(self, transform_branches, values: List[str], sketch_id: str |
neo4j_conn=neo4j_connection,
vault=vault,
)
-
+
# Use the synchronous scan method which internally handles the async operations
results = scanner.scan(values=values)
diff --git a/flowsint-core/src/flowsint_core/tests/fixtures/database.py b/flowsint-core/src/flowsint_core/tests/fixtures/database.py
index 0bd300f7..dac66630 100644
--- a/flowsint-core/src/flowsint_core/tests/fixtures/database.py
+++ b/flowsint-core/src/flowsint_core/tests/fixtures/database.py
@@ -5,8 +5,6 @@ from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from app.models.models import Profile, Key
from flowsint_core.core.vault import Vault
-import tempfile
-import os
@pytest.fixture(scope="function")
@@ -14,17 +12,17 @@ def test_db_session():
"""Create an in-memory SQLite database for testing"""
# Create in-memory SQLite database
engine = create_engine("sqlite:///:memory:", echo=False)
-
+
# Only create specific tables we need for testing (to avoid ARRAY type issues)
Profile.__table__.create(engine)
Key.__table__.create(engine)
-
+
# Create session
TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
session = TestingSessionLocal()
-
+
yield session
-
+
# Clean up
session.close()
engine.dispose()
@@ -45,13 +43,13 @@ def test_profile(test_db_session, test_owner_id):
hashed_password="hashed_test_password",
is_active=True,
first_name="Test",
- last_name="User"
+ last_name="User",
)
-
+
test_db_session.add(profile)
test_db_session.commit()
test_db_session.refresh(profile)
-
+
return profile
@@ -62,13 +60,13 @@ def test_api_key(test_db_session, test_owner_id):
id=uuid.uuid4(),
name="test_etherscan_key",
owner_id=test_owner_id,
- encrypted_key="test_api_key_12345"
+ encrypted_key="test_api_key_12345",
)
-
+
test_db_session.add(key)
test_db_session.commit()
test_db_session.refresh(key)
-
+
return key
@@ -90,46 +88,45 @@ def mock_logger():
return logger
[email protected](scope="function")
[email protected](scope="function")
def test_sketch_scan_ids():
"""Generate test sketch and scan IDs"""
- return {
- "sketch_id": str(uuid.uuid4()),
- "scan_id": str(uuid.uuid4())
- }
+ return {"sketch_id": str(uuid.uuid4()), "scan_id": str(uuid.uuid4())}
@pytest.fixture(scope="function")
def multiple_test_keys(test_db_session, test_owner_id):
"""Create multiple test API keys for comprehensive testing"""
keys = []
-
+
# Create keys with different names
key_configs = [
{"name": "etherscan_api_key", "value": "test_etherscan_key_123"},
{"name": "another_service_key", "value": "test_another_key_456"},
- {"name": "vaultKey1", "value": "test_vault_key_789"}
+ {"name": "vaultKey1", "value": "test_vault_key_789"},
]
-
+
for config in key_configs:
key = Key(
id=uuid.uuid4(),
name=config["name"],
owner_id=test_owner_id,
- encrypted_key=config["value"]
+ encrypted_key=config["value"],
)
test_db_session.add(key)
keys.append(key)
-
+
test_db_session.commit()
-
+
for key in keys:
test_db_session.refresh(key)
-
+
return keys
@pytest.fixture(scope="function")
-def test_vault_with_multiple_keys(test_db_session, test_owner_id, test_profile, multiple_test_keys):
+def test_vault_with_multiple_keys(
+ test_db_session, test_owner_id, test_profile, multiple_test_keys
+):
"""Create a test vault with multiple keys for comprehensive testing"""
- return Vault(db=test_db_session, owner_id=test_owner_id)
\ No newline at end of file
+ return Vault(db=test_db_session, owner_id=test_owner_id)
diff --git a/flowsint-core/src/flowsint_core/tests/orchestrator.py b/flowsint-core/src/flowsint_core/tests/orchestrator.py
index 5bf09abe..97deb6ed 100644
--- a/flowsint-core/src/flowsint_core/tests/orchestrator.py
+++ b/flowsint-core/src/flowsint_core/tests/orchestrator.py
@@ -4,76 +4,108 @@ from flowsint_types.whois import Whois
import pytest
+
def test_preprocess_valid_domains():
- scanner = TransformOrchestrator("123",scanner_names= ["domain_resolve_scanner", "domain_whois_scanner"])
+ scanner = TransformOrchestrator(
+ "123", scanner_names=["domain_resolve_scanner", "domain_whois_scanner"]
+ )
assert isinstance(scanner.scanners, list)
assert len(scanner.scanners) == 2
-
+
+
def test_preprocess_valid_domains():
- with pytest.raises(ValueError, match="Scanner 'this_scan_is_wrong' not found in registry"):
- TransformOrchestrator("123",scanner_names= ["domain_resolve_scanner", "this_scan_is_wrong"])
+ with pytest.raises(
+ ValueError, match="Scanner 'this_scan_is_wrong' not found in registry"
+ ):
+ TransformOrchestrator(
+ "123", scanner_names=["domain_resolve_scanner", "this_scan_is_wrong"]
+ )
+
def test_execute_domain_subdomains_scanner():
domains = ["example.com"]
- scanner = TransformOrchestrator("123", scanner_names = ["domain_subdomains_scanner"])
+ scanner = TransformOrchestrator("123", scanner_names=["domain_subdomains_scanner"])
results = scanner.execute(values=domains)
assert results["initial_values"] == domains
- assert results['scanners'] == ['domain_subdomains_scanner']
- assert isinstance(results['results']['domain_subdomains_scanner'], list)
- assert isinstance(results['results']['domain_subdomains_scanner'][0], Domain)
+ assert results["scanners"] == ["domain_subdomains_scanner"]
+ assert isinstance(results["results"]["domain_subdomains_scanner"], list)
+ assert isinstance(results["results"]["domain_subdomains_scanner"][0], Domain)
def test_execute_domain_whois_scanner():
domains = ["example.com"]
- scanner = TransformOrchestrator("123", scanner_names = ["domain_whois_scanner"])
+ scanner = TransformOrchestrator("123", scanner_names=["domain_whois_scanner"])
results = scanner.execute(values=domains)
assert results["initial_values"] == domains
- assert results['scanners'] == ['domain_whois_scanner']
- assert isinstance(results['results']['domain_whois_scanner'], list)
- assert isinstance(results['results']['domain_whois_scanner'][0], Whois)
-
+ assert results["scanners"] == ["domain_whois_scanner"]
+ assert isinstance(results["results"]["domain_whois_scanner"], list)
+ assert isinstance(results["results"]["domain_whois_scanner"][0], Whois)
+
+
def test_execute_ip_resolve():
- ips = ['91.199.212.73']
- scanner = TransformOrchestrator("123", scanner_names = ["ip_reverse_resolve_scanner"])
+ ips = ["91.199.212.73"]
+ scanner = TransformOrchestrator("123", scanner_names=["ip_reverse_resolve_scanner"])
results = scanner.execute(values=ips)
assert results["initial_values"] == ips
- assert results['scanners'] == ['ip_reverse_resolve_scanner']
- assert isinstance(results['results']['ip_reverse_resolve_scanner'], list)
- if(len(results['results']['ip_reverse_resolve_scanner'])!= 0):
- assert isinstance(results['results']['ip_reverse_resolve_scanner'][0], Domain)
-
+ assert results["scanners"] == ["ip_reverse_resolve_scanner"]
+ assert isinstance(results["results"]["ip_reverse_resolve_scanner"], list)
+ if len(results["results"]["ip_reverse_resolve_scanner"]) != 0:
+ assert isinstance(results["results"]["ip_reverse_resolve_scanner"][0], Domain)
+
+
def test_execute_ip_resolve_and_whois():
- ips = ['91.199.212.73']
- scanner = TransformOrchestrator("123", scanner_names = ["ip_reverse_resolve_scanner", "domain_whois_scanner"])
+ ips = ["91.199.212.73"]
+ scanner = TransformOrchestrator(
+ "123", scanner_names=["ip_reverse_resolve_scanner", "domain_whois_scanner"]
+ )
results = scanner.execute(values=ips)
assert results["initial_values"] == ips
- assert results['scanners'] == ['ip_reverse_resolve_scanner', 'domain_whois_scanner']
-
+ assert results["scanners"] == ["ip_reverse_resolve_scanner", "domain_whois_scanner"]
+
+
def test_execute_ip_resolve_and_whois_multiple():
- ips = ['91.199.212.73', '76.76.21.21']
- scanner = TransformOrchestrator("123", scanner_names = ["ip_reverse_resolve_scanner", "domain_whois_scanner"])
+ ips = ["91.199.212.73", "76.76.21.21"]
+ scanner = TransformOrchestrator(
+ "123", scanner_names=["ip_reverse_resolve_scanner", "domain_whois_scanner"]
+ )
results = scanner.execute(values=ips)
assert results["initial_values"] == ips
- assert results['scanners'] == ['ip_reverse_resolve_scanner', 'domain_whois_scanner']
+ assert results["scanners"] == ["ip_reverse_resolve_scanner", "domain_whois_scanner"]
print(results)
+
def test_execute_ip_resolve_and_whois_multiple():
- ips = ['162.19.81.222']
- scanner = TransformOrchestrator("123", scanner_names = ["ip_reverse_resolve_scanner", "domain_whois_scanner", "domain_subdomains_scanner"])
+ ips = ["162.19.81.222"]
+ scanner = TransformOrchestrator(
+ "123",
+ scanner_names=[
+ "ip_reverse_resolve_scanner",
+ "domain_whois_scanner",
+ "domain_subdomains_scanner",
+ ],
+ )
results = scanner.execute(values=ips)
assert results["initial_values"] == ips
- assert results['scanners'] == ['ip_reverse_resolve_scanner', 'domain_whois_scanner', "domain_subdomains_scanner"]
-
+ assert results["scanners"] == [
+ "ip_reverse_resolve_scanner",
+ "domain_whois_scanner",
+ "domain_subdomains_scanner",
+ ]
+
+
def test_execute_domain_whois_and_subdomains():
domains = ["alliage.io"]
- scanner = TransformOrchestrator("123", scanner_names = ["domain_whois_scanner", "domain_subdomains_scanner"])
+ scanner = TransformOrchestrator(
+ "123", scanner_names=["domain_whois_scanner", "domain_subdomains_scanner"]
+ )
results = scanner.execute(values=domains)
assert results["initial_values"] == domains
- assert results['scanners'] == ['domain_whois_scanner', "domain_subdomains_scanner"]
+ assert results["scanners"] == ["domain_whois_scanner", "domain_subdomains_scanner"]
+
def test_execute_domain_subdomains():
domains = ["alliage.io"]
- scanner = TransformOrchestrator("123", scanner_names = ["domain_subdomains_scanner"])
+ scanner = TransformOrchestrator("123", scanner_names=["domain_subdomains_scanner"])
results = scanner.execute(values=domains)
assert results["initial_values"] == domains
- assert results['scanners'] == ["domain_subdomains_scanner"]
+ assert results["scanners"] == ["domain_subdomains_scanner"]
diff --git a/flowsint-core/src/flowsint_core/tests/test_base_scanner.py b/flowsint-core/src/flowsint_core/tests/test_base_scanner.py
index 0eb04761..7d2078c3 100644
--- a/flowsint-core/src/flowsint_core/tests/test_base_scanner.py
+++ b/flowsint-core/src/flowsint_core/tests/test_base_scanner.py
@@ -7,31 +7,31 @@ from flowsint_types.ip import Ip
class MockScanner(Scanner):
"""Mock scanner for testing base functionality"""
-
+
# Define InputType and OutputType
InputType = List[Domain]
OutputType = List[Ip]
-
+
@classmethod
def name(cls) -> str:
return "mock_scanner"
-
+
@classmethod
def category(cls) -> str:
return "Test"
-
+
@classmethod
def key(cls) -> str:
return "domain"
-
+
@classmethod
def input_schema(cls) -> Dict[str, Any]:
return cls.generate_input_schema()
-
+
@classmethod
def output_schema(cls) -> Dict[str, Any]:
return cls.generate_output_schema()
-
+
async def scan(self, values: List[str]) -> List[Dict[str, Any]]:
# Mock implementation
return [{"address": "1.2.3.4"}]
@@ -39,165 +39,165 @@ class MockScanner(Scanner):
class IncompleteScanner(Scanner):
"""Scanner without InputType/OutputType for testing error cases"""
-
+
@classmethod
def name(cls) -> str:
return "incomplete_scanner"
-
+
@classmethod
def category(cls) -> str:
return "Test"
-
+
@classmethod
def key(cls) -> str:
return "test"
-
+
@classmethod
def input_schema(cls) -> Dict[str, Any]:
return cls.generate_input_schema()
-
+
@classmethod
def output_schema(cls) -> Dict[str, Any]:
return cls.generate_output_schema()
-
+
async def scan(self, values: List[str]) -> List[Dict[str, Any]]:
return []
class TestBaseScannerInputOutputTypes:
"""Test suite for Scanner InputType/OutputType functionality"""
-
+
def test_input_type_output_type_class_attributes(self):
"""Test that InputType and OutputType are properly set as class attributes"""
- assert hasattr(MockScanner, 'InputType')
- assert hasattr(MockScanner, 'OutputType')
+ assert hasattr(MockScanner, "InputType")
+ assert hasattr(MockScanner, "OutputType")
assert MockScanner.InputType == List[Domain]
assert MockScanner.OutputType == List[Ip]
-
+
def test_generate_input_schema_success(self):
"""Test that generate_input_schema works correctly with valid InputType"""
schema = MockScanner.generate_input_schema()
-
+
assert isinstance(schema, dict)
assert "type" in schema
assert "properties" in schema
assert schema["type"] == "Domain"
-
+
# Check that properties are correctly extracted
properties = schema["properties"]
assert isinstance(properties, list)
-
+
# Should have domain property
domain_prop = next((p for p in properties if p["name"] == "domain"), None)
assert domain_prop is not None
assert domain_prop["type"] == "string"
-
+
def test_generate_output_schema_success(self):
"""Test that generate_output_schema works correctly with valid OutputType"""
schema = MockScanner.generate_output_schema()
-
+
assert isinstance(schema, dict)
assert "type" in schema
assert "properties" in schema
assert schema["type"] == "Ip"
-
+
# Check that properties are correctly extracted
properties = schema["properties"]
assert isinstance(properties, list)
-
+
# Should have address property
address_prop = next((p for p in properties if p["name"] == "address"), None)
assert address_prop is not None
assert address_prop["type"] == "string"
-
+
def test_generate_input_schema_not_implemented_error(self):
"""Test that generate_input_schema raises error when InputType is not defined"""
with pytest.raises(NotImplementedError) as exc_info:
IncompleteScanner.generate_input_schema()
-
+
assert "InputType must be defined" in str(exc_info.value)
assert "IncompleteScanner" in str(exc_info.value)
-
+
def test_generate_output_schema_not_implemented_error(self):
"""Test that generate_output_schema raises error when OutputType is not defined"""
with pytest.raises(NotImplementedError) as exc_info:
IncompleteScanner.generate_output_schema()
-
+
assert "OutputType must be defined" in str(exc_info.value)
assert "IncompleteScanner" in str(exc_info.value)
-
+
def test_input_output_schema_methods_use_generate_methods(self):
"""Test that the schema methods properly use the generate methods"""
input_schema = MockScanner.input_schema()
output_schema = MockScanner.output_schema()
-
+
# These should be identical to calling generate methods directly
assert input_schema == MockScanner.generate_input_schema()
assert output_schema == MockScanner.generate_output_schema()
-
+
def test_base_scanner_has_not_implemented_defaults(self):
"""Test that base Scanner class has NotImplemented defaults"""
assert Scanner.InputType is NotImplemented
assert Scanner.OutputType is NotImplemented
-
+
def test_inheritance_preserves_input_output_types(self):
"""Test that InputType and OutputType are properly inherited"""
-
+
class ChildScanner(MockScanner):
pass
-
+
# Child should inherit the types from MockScanner
assert ChildScanner.InputType == List[Domain]
assert ChildScanner.OutputType == List[Ip]
-
+
# And schema generation should work
input_schema = ChildScanner.generate_input_schema()
output_schema = ChildScanner.generate_output_schema()
-
+
assert input_schema["type"] == "Domain"
assert output_schema["type"] == "Ip"
-
+
def test_scanner_instance_can_access_class_types(self):
"""Test that scanner instances can access InputType and OutputType"""
scanner = MockScanner("test_sketch", "test_scan")
-
+
assert scanner.InputType == List[Domain]
assert scanner.OutputType == List[Ip]
-
+
# Instance should be able to call generate methods
input_schema = scanner.generate_input_schema()
output_schema = scanner.generate_output_schema()
-
+
assert input_schema["type"] == "Domain"
assert output_schema["type"] == "Ip"
class TestScannerFunctionality:
"""Test other Scanner base functionality"""
-
+
def test_scanner_initialization(self):
"""Test that Scanner initializes correctly"""
scanner = MockScanner("test_sketch", "test_scan")
-
+
assert scanner.sketch_id == "test_sketch"
assert scanner.scan_id == "test_scan"
assert scanner.params == {}
assert scanner.params_schema == []
-
+
def test_scanner_initialization_with_defaults(self):
"""Test Scanner initialization with default values"""
scanner = MockScanner()
-
+
assert scanner.sketch_id == "system"
assert scanner.scan_id == "default"
-
+
@pytest.mark.asyncio
async def test_scanner_execute_method(self):
"""Test the execute method workflow"""
scanner = MockScanner("test_sketch", "test_scan")
-
+
result = await scanner.execute(["test.com"])
-
+
assert isinstance(result, list)
assert len(result) == 1
- assert result[0]["address"] == "1.2.3.4"
\ No newline at end of file
+ assert result[0]["address"] == "1.2.3.4"
diff --git a/flowsint-core/src/flowsint_core/tests/test_crypto_with_vault.py b/flowsint-core/src/flowsint_core/tests/test_crypto_with_vault.py
index c226ea31..03b5551d 100644
--- a/flowsint-core/src/flowsint_core/tests/test_crypto_with_vault.py
+++ b/flowsint-core/src/flowsint_core/tests/test_crypto_with_vault.py
@@ -1,12 +1,19 @@
import pytest
import uuid
-from flowsint_transforms.crypto.wallet_to_transactions import CryptoWalletAddressToTransactions
+from flowsint_transforms.crypto.wallet_to_transactions import (
+ CryptoWalletAddressToTransactions,
+)
from flowsint_transforms.crypto.wallet_to_nfts import CryptoWalletAddressToNFTs
from ..core.vault import Vault
from .fixtures.database import (
- test_db_session, test_owner_id, test_profile, test_api_key,
- test_vault, test_vault_with_multiple_keys, multiple_test_keys,
- test_sketch_scan_ids
+ test_db_session,
+ test_owner_id,
+ test_profile,
+ test_api_key,
+ test_vault,
+ test_vault_with_multiple_keys,
+ multiple_test_keys,
+ test_sketch_scan_ids,
)
@@ -28,7 +35,9 @@ class TestVaultIntegration:
secret = test_vault_with_multiple_keys.get_secret("vaultKey1")
assert secret == "test_vault_key_789"
- def test_vault_get_secret_by_uuid(self, test_vault_with_multiple_keys, multiple_test_keys):
+ def test_vault_get_secret_by_uuid(
+ self, test_vault_with_multiple_keys, multiple_test_keys
+ ):
"""Test retrieving secret by key UUID"""
# Get the UUID of the first key
key_uuid = str(multiple_test_keys[0].id)
@@ -59,10 +68,10 @@ class TestCryptoWalletAddressToTransactions:
vault=test_vault,
params={
"ETHERSCAN_API_KEY": "test_etherscan_key",
- "ETHERSCAN_API_URL": "https://api.etherscan.io/api"
- }
+ "ETHERSCAN_API_URL": "https://api.etherscan.io/api",
+ },
)
-
+
assert scanner.sketch_id == test_sketch_scan_ids["sketch_id"]
assert scanner.scan_id == test_sketch_scan_ids["scan_id"]
assert scanner.vault == test_vault
@@ -74,14 +83,16 @@ class TestCryptoWalletAddressToTransactions:
scan_id=test_sketch_scan_ids["scan_id"],
params={
"ETHERSCAN_API_KEY": "direct_api_key",
- "ETHERSCAN_API_URL": "https://api.etherscan.io/api"
- }
+ "ETHERSCAN_API_URL": "https://api.etherscan.io/api",
+ },
)
-
+
assert scanner.vault is None
@pytest.mark.asyncio
- async def test_async_init_with_vault_key_exists(self, test_vault_with_multiple_keys, test_sketch_scan_ids):
+ async def test_async_init_with_vault_key_exists(
+ self, test_vault_with_multiple_keys, test_sketch_scan_ids
+ ):
"""Test async_init when vault key exists"""
scanner = CryptoWalletAddressToTransactions(
sketch_id=test_sketch_scan_ids["sketch_id"],
@@ -89,23 +100,25 @@ class TestCryptoWalletAddressToTransactions:
vault=test_vault_with_multiple_keys,
params={
"ETHERSCAN_API_KEY": "vaultKey1", # This key exists in vault
- "ETHERSCAN_API_URL": "https://api.etherscan.io/api"
- }
+ "ETHERSCAN_API_URL": "https://api.etherscan.io/api",
+ },
)
-
+
# Get params before async_init
params_before = scanner.get_params()
assert params_before["ETHERSCAN_API_KEY"] == "vaultKey1"
-
+
# Initialize scanner
await scanner.async_init()
-
+
# Get params after async_init - should be resolved from vault
params_after = scanner.get_params()
assert params_after["ETHERSCAN_API_KEY"] == "test_vault_key_789"
@pytest.mark.asyncio
- async def test_async_init_with_vault_key_not_exists(self, test_vault, test_sketch_scan_ids):
+ async def test_async_init_with_vault_key_not_exists(
+ self, test_vault, test_sketch_scan_ids
+ ):
"""Test async_init when vault key doesn't exist"""
scanner = CryptoWalletAddressToTransactions(
sketch_id=test_sketch_scan_ids["sketch_id"],
@@ -113,17 +126,17 @@ class TestCryptoWalletAddressToTransactions:
vault=test_vault,
params={
"ETHERSCAN_API_KEY": "nonexistent_key", # This key doesn't exist
- "ETHERSCAN_API_URL": "https://api.etherscan.io/api"
- }
+ "ETHERSCAN_API_URL": "https://api.etherscan.io/api",
+ },
)
-
+
# Get params before async_init
params_before = scanner.get_params()
assert params_before["ETHERSCAN_API_KEY"] == "nonexistent_key"
-
+
# Initialize scanner
await scanner.async_init()
-
+
# Get params after async_init - should remain unchanged since key not found
params_after = scanner.get_params()
assert params_after["ETHERSCAN_API_KEY"] == "nonexistent_key"
@@ -136,17 +149,17 @@ class TestCryptoWalletAddressToTransactions:
scan_id=test_sketch_scan_ids["scan_id"],
params={
"ETHERSCAN_API_KEY": "direct_api_key",
- "ETHERSCAN_API_URL": "https://api.etherscan.io/api"
- }
+ "ETHERSCAN_API_URL": "https://api.etherscan.io/api",
+ },
)
-
+
# Get params before async_init
params_before = scanner.get_params()
assert params_before["ETHERSCAN_API_KEY"] == "direct_api_key"
-
+
# Initialize scanner
await scanner.async_init()
-
+
# Get params after async_init - should remain unchanged
params_after = scanner.get_params()
assert params_after["ETHERSCAN_API_KEY"] == "direct_api_key"
@@ -169,16 +182,18 @@ class TestCryptoWalletAddressToNFTs:
vault=test_vault,
params={
"ETHERSCAN_API_KEY": "test_etherscan_key",
- "ETHERSCAN_API_URL": "https://api.etherscan.io/api"
- }
+ "ETHERSCAN_API_URL": "https://api.etherscan.io/api",
+ },
)
-
+
assert scanner.sketch_id == test_sketch_scan_ids["sketch_id"]
assert scanner.scan_id == test_sketch_scan_ids["scan_id"]
assert scanner.vault == test_vault
@pytest.mark.asyncio
- async def test_async_init_with_vault_multiple_keys(self, test_vault_with_multiple_keys, test_sketch_scan_ids):
+ async def test_async_init_with_vault_multiple_keys(
+ self, test_vault_with_multiple_keys, test_sketch_scan_ids
+ ):
"""Test async_init with multiple vault keys"""
scanner = CryptoWalletAddressToNFTs(
sketch_id=test_sketch_scan_ids["sketch_id"],
@@ -186,13 +201,13 @@ class TestCryptoWalletAddressToNFTs:
vault=test_vault_with_multiple_keys,
params={
"ETHERSCAN_API_KEY": "etherscan_api_key", # This key exists in vault
- "ETHERSCAN_API_URL": "https://api.etherscan.io/api"
- }
+ "ETHERSCAN_API_URL": "https://api.etherscan.io/api",
+ },
)
-
+
# Initialize scanner
await scanner.async_init()
-
+
# Get params after async_init - should be resolved from vault
params_after = scanner.get_params()
assert params_after["ETHERSCAN_API_KEY"] == "test_etherscan_key_123"
@@ -208,7 +223,9 @@ class TestBothCryptoScannersIntegration:
"""Test suite for testing both crypto scanners together"""
@pytest.mark.asyncio
- async def test_multiple_scanners_same_vault(self, test_vault_with_multiple_keys, test_sketch_scan_ids):
+ async def test_multiple_scanners_same_vault(
+ self, test_vault_with_multiple_keys, test_sketch_scan_ids
+ ):
"""Test multiple scanners using the same vault"""
scanner1 = CryptoWalletAddressToTransactions(
sketch_id=test_sketch_scan_ids["sketch_id"],
@@ -216,33 +233,35 @@ class TestBothCryptoScannersIntegration:
vault=test_vault_with_multiple_keys,
params={
"ETHERSCAN_API_KEY": "vaultKey1",
- "ETHERSCAN_API_URL": "https://api.etherscan.io/api"
- }
+ "ETHERSCAN_API_URL": "https://api.etherscan.io/api",
+ },
)
-
+
scanner2 = CryptoWalletAddressToNFTs(
sketch_id=test_sketch_scan_ids["sketch_id"],
scan_id=test_sketch_scan_ids["scan_id"],
vault=test_vault_with_multiple_keys,
params={
"ETHERSCAN_API_KEY": "vaultKey1",
- "ETHERSCAN_API_URL": "https://api.etherscan.io/api"
- }
+ "ETHERSCAN_API_URL": "https://api.etherscan.io/api",
+ },
)
-
+
# Initialize both scanners
await scanner1.async_init()
await scanner2.async_init()
-
+
# Both should have resolved the same key
params1 = scanner1.get_params()
params2 = scanner2.get_params()
-
+
assert params1["ETHERSCAN_API_KEY"] == "test_vault_key_789"
assert params2["ETHERSCAN_API_KEY"] == "test_vault_key_789"
@pytest.mark.asyncio
- async def test_multiple_scanners_different_keys(self, test_vault_with_multiple_keys, test_sketch_scan_ids):
+ async def test_multiple_scanners_different_keys(
+ self, test_vault_with_multiple_keys, test_sketch_scan_ids
+ ):
"""Test multiple scanners using different vault keys"""
scanner1 = CryptoWalletAddressToTransactions(
sketch_id=test_sketch_scan_ids["sketch_id"],
@@ -250,28 +269,28 @@ class TestBothCryptoScannersIntegration:
vault=test_vault_with_multiple_keys,
params={
"ETHERSCAN_API_KEY": "vaultKey1",
- "ETHERSCAN_API_URL": "https://api.etherscan.io/api"
- }
+ "ETHERSCAN_API_URL": "https://api.etherscan.io/api",
+ },
)
-
+
scanner2 = CryptoWalletAddressToNFTs(
sketch_id=test_sketch_scan_ids["sketch_id"],
scan_id=test_sketch_scan_ids["scan_id"],
vault=test_vault_with_multiple_keys,
params={
"ETHERSCAN_API_KEY": "etherscan_api_key",
- "ETHERSCAN_API_URL": "https://api.etherscan.io/api"
- }
+ "ETHERSCAN_API_URL": "https://api.etherscan.io/api",
+ },
)
-
+
# Initialize both scanners
await scanner1.async_init()
await scanner2.async_init()
-
+
# Should have resolved different keys
params1 = scanner1.get_params()
params2 = scanner2.get_params()
-
+
assert params1["ETHERSCAN_API_KEY"] == "test_vault_key_789"
assert params2["ETHERSCAN_API_KEY"] == "test_etherscan_key_123"
@@ -279,24 +298,28 @@ class TestBothCryptoScannersIntegration:
"""Test that both crypto scanners have consistent schemas"""
transactions_schema = CryptoWalletAddressToTransactions.input_schema()
nfts_schema = CryptoWalletAddressToNFTs.input_schema()
-
+
# Both should accept the same input type (crypto wallet)
assert transactions_schema["type"] == nfts_schema["type"]
-
+
# Both should have param schemas with ETHERSCAN_API_KEY
transactions_params = CryptoWalletAddressToTransactions.get_params_schema()
nfts_params = CryptoWalletAddressToNFTs.get_params_schema()
-
+
# Check that both have ETHERSCAN_API_KEY parameter
transactions_api_key = next(
- (param for param in transactions_params if param["name"] == "ETHERSCAN_API_KEY"),
- None
+ (
+ param
+ for param in transactions_params
+ if param["name"] == "ETHERSCAN_API_KEY"
+ ),
+ None,
)
nfts_api_key = next(
- (param for param in nfts_params if param["name"] == "ETHERSCAN_API_KEY"),
- None
+ (param for param in nfts_params if param["name"] == "ETHERSCAN_API_KEY"),
+ None,
)
-
+
assert transactions_api_key is not None
assert nfts_api_key is not None
- assert transactions_api_key["required"] == nfts_api_key["required"]
\ No newline at end of file
+ assert transactions_api_key["required"] == nfts_api_key["required"]
diff --git a/flowsint-core/src/flowsint_core/tests/test_input_output_migration.py b/flowsint-core/src/flowsint_core/tests/test_input_output_migration.py
index 0143393b..62d223fd 100644
--- a/flowsint-core/src/flowsint_core/tests/test_input_output_migration.py
+++ b/flowsint-core/src/flowsint_core/tests/test_input_output_migration.py
@@ -3,6 +3,7 @@ Test migration from old TypeAlias pattern to new InputType/OutputType class attr
This test demonstrates the benefits and proper usage of the new pattern.
"""
+
import pytest
from typing import List, Dict, Any, TypeAlias
from pydantic import TypeAdapter
@@ -15,19 +16,19 @@ from app.utils import resolve_type
class OldPatternScanner(Scanner):
"""Example of old pattern using module-level TypeAlias"""
-
+
@classmethod
def name(cls) -> str:
return "old_pattern_scanner"
-
+
@classmethod
def category(cls) -> str:
return "Test"
-
+
@classmethod
def key(cls) -> str:
return "domain"
-
+
@classmethod
def input_schema(cls) -> Dict[str, Any]:
# Old pattern: manually defining TypeAlias and building schema
@@ -40,9 +41,9 @@ class OldPatternScanner(Scanner):
"properties": [
{"name": prop, "type": resolve_type(info, schema)}
for prop, info in details["properties"].items()
- ]
+ ],
}
-
+
@classmethod
def output_schema(cls) -> Dict[str, Any]:
# Old pattern: manually defining TypeAlias and building schema
@@ -55,34 +56,34 @@ class OldPatternScanner(Scanner):
"properties": [
{"name": prop, "type": resolve_type(info, schema)}
for prop, info in details["properties"].items()
- ]
+ ],
}
-
+
async def scan(self, values: List[str]) -> List[Dict[str, Any]]:
return [{"address": "1.2.3.4"}]
class NewPatternScanner(Scanner):
"""Example of new pattern using class attributes with automatic schema generation"""
-
+
# New pattern: just define class attributes - base class handles the rest!
InputType = List[Domain]
OutputType = List[Ip]
-
+
@classmethod
def name(cls) -> str:
return "new_pattern_scanner"
-
+
@classmethod
def category(cls) -> str:
return "Test"
-
+
@classmethod
def key(cls) -> str:
return "domain"
-
+
# No need to implement input_schema() or output_schema() - base class does it automatically!
-
+
# Methods can use InputType/OutputType directly (once made available at module level)
def preprocess(self, data: List[Domain]) -> List[Domain]:
# Using concrete type for clarity in test, but would use InputType in real implementation
@@ -91,7 +92,7 @@ class NewPatternScanner(Scanner):
if isinstance(item, Domain):
cleaned.append(item)
return cleaned
-
+
async def scan(self, values: List[Domain]) -> List[Ip]:
# Using concrete type for clarity in test, but would use InputType/OutputType in real implementation
results: List[Ip] = []
@@ -99,6 +100,7 @@ class NewPatternScanner(Scanner):
results.append(Ip(address="1.2.3.4"))
return results
+
# Make types available at module level (this is what enables clean usage)
NewPatternInputType = NewPatternScanner.InputType
NewPatternOutputType = NewPatternScanner.OutputType
@@ -106,230 +108,262 @@ NewPatternOutputType = NewPatternScanner.OutputType
class AdvancedNewPatternScanner(Scanner):
"""Example showing advanced usage with different types"""
-
+
InputType = List[Email]
OutputType = List[Domain]
-
+
@classmethod
def name(cls) -> str:
return "advanced_pattern_scanner"
-
+
@classmethod
def category(cls) -> str:
return "Test"
-
+
@classmethod
def key(cls) -> str:
return "email"
-
+
# Schema methods automatic!
-
+
async def scan(self, values: List[str]) -> List[Dict[str, Any]]:
return [{"domain": "example.com"}]
class TestInputOutputMigrationPattern:
"""Test migration from old to new pattern"""
-
+
def test_both_patterns_produce_same_schema(self):
"""Test that old and new patterns produce identical schemas"""
old_input_schema = OldPatternScanner.input_schema()
new_input_schema = NewPatternScanner.input_schema()
-
+
old_output_schema = OldPatternScanner.output_schema()
new_output_schema = NewPatternScanner.output_schema()
-
+
# Schemas should be identical
assert old_input_schema == new_input_schema
assert old_output_schema == new_output_schema
-
+
def test_new_pattern_benefits_code_reuse(self):
"""Test that new pattern reduces code duplication"""
# With the new pattern, multiple scanners can easily reuse the same logic
-
+
class Scanner1(Scanner):
InputType = List[Domain]
OutputType = List[Ip]
-
+
@classmethod
- def name(cls): return "scanner1"
+ def name(cls):
+ return "scanner1"
+
@classmethod
- def category(cls): return "Test"
+ def category(cls):
+ return "Test"
+
@classmethod
- def key(cls): return "domain"
+ def key(cls):
+ return "domain"
+
# No need for input_schema() or output_schema() - automatic!
- async def scan(self, values): return []
-
+ async def scan(self, values):
+ return []
+
class Scanner2(Scanner):
InputType = List[Domain]
OutputType = List[Ip]
-
+
@classmethod
- def name(cls): return "scanner2"
+ def name(cls):
+ return "scanner2"
+
@classmethod
- def category(cls): return "Test"
+ def category(cls):
+ return "Test"
+
@classmethod
- def key(cls): return "domain"
+ def key(cls):
+ return "domain"
+
# No need for input_schema() or output_schema() - automatic!
- async def scan(self, values): return []
-
+ async def scan(self, values):
+ return []
+
# Both should produce identical schemas with minimal code
assert Scanner1.input_schema() == Scanner2.input_schema()
assert Scanner1.output_schema() == Scanner2.output_schema()
-
+
def test_new_pattern_type_introspection(self):
"""Test that new pattern allows for better type introspection"""
# Can easily check what types a scanner uses
assert NewPatternScanner.InputType == List[Domain]
assert NewPatternScanner.OutputType == List[Ip]
-
+
assert AdvancedNewPatternScanner.InputType == List[Email]
assert AdvancedNewPatternScanner.OutputType == List[Domain]
-
+
# This wasn't easily possible with the old pattern
-
+
def test_new_pattern_inheritance_works(self):
"""Test that new pattern works well with inheritance"""
-
+
class BaseDomainScanner(Scanner):
InputType = List[Domain]
OutputType = List[Ip]
-
+
@classmethod
- def name(cls): return "base_domain"
+ def name(cls):
+ return "base_domain"
+
@classmethod
- def category(cls): return "Test"
+ def category(cls):
+ return "Test"
+
@classmethod
- def key(cls): return "domain"
+ def key(cls):
+ return "domain"
+
# Schema methods automatic!
- async def scan(self, values): return []
-
+ async def scan(self, values):
+ return []
+
class SpecializedDomainScanner(BaseDomainScanner):
@classmethod
- def name(cls): return "specialized_domain"
+ def name(cls):
+ return "specialized_domain"
+
# Inherits InputType and OutputType
-
+
# Child should inherit the types and schemas
assert SpecializedDomainScanner.InputType == List[Domain]
assert SpecializedDomainScanner.OutputType == List[Ip]
-
+
specialized_input = SpecializedDomainScanner.input_schema()
specialized_output = SpecializedDomainScanner.output_schema()
-
+
assert specialized_input["type"] == "Domain"
assert specialized_output["type"] == "Ip"
-
+
def test_new_pattern_error_handling(self):
"""Test that new pattern provides better error handling"""
-
+
class IncompleteDomainScanner(Scanner):
# Forgot to define InputType and OutputType
@classmethod
- def name(cls): return "incomplete"
+ def name(cls):
+ return "incomplete"
+
@classmethod
- def category(cls): return "Test"
+ def category(cls):
+ return "Test"
+
@classmethod
- def key(cls): return "domain"
+ def key(cls):
+ return "domain"
+
# Base class will try to generate schemas automatically and fail appropriately
- async def scan(self, values): return []
-
+ async def scan(self, values):
+ return []
+
# Should get clear error messages
with pytest.raises(NotImplementedError) as exc_info:
IncompleteDomainScanner.input_schema()
assert "InputType must be defined" in str(exc_info.value)
assert "IncompleteDomainScanner" in str(exc_info.value)
-
+
with pytest.raises(NotImplementedError) as exc_info:
IncompleteDomainScanner.output_schema()
assert "OutputType must be defined" in str(exc_info.value)
assert "IncompleteDomainScanner" in str(exc_info.value)
-
+
def test_new_pattern_runtime_accessibility(self):
"""Test that types are accessible at runtime for dynamic operations"""
-
+
# Can build registries or perform operations based on types
scanners = [NewPatternScanner, AdvancedNewPatternScanner]
-
+
domain_input_scanners = [
- scanner for scanner in scanners
- if hasattr(scanner, 'InputType') and scanner.InputType == List[Domain]
+ scanner
+ for scanner in scanners
+ if hasattr(scanner, "InputType") and scanner.InputType == List[Domain]
]
-
+
email_input_scanners = [
- scanner for scanner in scanners
- if hasattr(scanner, 'InputType') and scanner.InputType == List[Email]
+ scanner
+ for scanner in scanners
+ if hasattr(scanner, "InputType") and scanner.InputType == List[Email]
]
-
+
assert len(domain_input_scanners) == 1
assert domain_input_scanners[0] == NewPatternScanner
-
+
assert len(email_input_scanners) == 1
assert email_input_scanners[0] == AdvancedNewPatternScanner
-
+
def test_new_pattern_with_clean_type_usage(self):
"""Test that the new pattern allows clean type usage without quotes"""
scanner = NewPatternScanner("test", "test")
-
+
# Test that we can create data of the expected types
test_domains = [Domain(domain="example.com"), Domain(domain="test.com")]
-
+
# Preprocess should work with clean type annotations
result = scanner.preprocess(test_domains)
assert len(result) == 2
assert all(isinstance(d, Domain) for d in result)
-
+
@pytest.mark.asyncio
async def test_new_pattern_async_methods(self):
"""Test that async methods work correctly with clean type annotations"""
scanner = NewPatternScanner("test", "test")
-
+
test_domains = [Domain(domain="example.com")]
result = await scanner.scan(test_domains)
-
+
assert len(result) == 1
assert isinstance(result[0], Ip)
assert result[0].address == "1.2.3.4"
-
+
def test_module_level_type_access(self):
"""Test that types are properly accessible at module level"""
# These should be available after the class definition
assert NewPatternInputType == List[Domain]
assert NewPatternOutputType == List[Ip]
-
+
# And they should match the class attributes
assert NewPatternInputType == NewPatternScanner.InputType
assert NewPatternOutputType == NewPatternScanner.OutputType
-
+
def test_migration_checklist(self):
"""Test that demonstrates a complete migration checklist"""
-
+
# Migration steps:
# 1. Define InputType and OutputType as class attributes
# 2. Remove input_schema() and output_schema() method implementations (base class handles automatically)
# 3. Add module-level assignments: InputType = MyScanner.InputType (optional, for clean usage)
-
+
# Verify the new pattern is simpler and more maintainable
new_scanner_benefits = [
"Just define InputType and OutputType class attributes",
- "Automatic schema generation by base class",
+ "Automatic schema generation by base class",
"No boilerplate schema methods needed",
"Consistent schema generation across all scanners",
- "Clean type usage throughout class methods"
+ "Clean type usage throughout class methods",
]
-
+
assert len(new_scanner_benefits) == 5
-
+
# Verify functionality is preserved and automatic
schema = NewPatternScanner.input_schema()
assert schema["type"] == "Domain"
assert any(prop["name"] == "domain" for prop in schema["properties"])
-
+
# Verify schemas are generated automatically without manual implementation
- assert hasattr(NewPatternScanner, 'input_schema')
- assert hasattr(NewPatternScanner, 'output_schema')
-
+ assert hasattr(NewPatternScanner, "input_schema")
+ assert hasattr(NewPatternScanner, "output_schema")
+
# The base class should be handling the schema generation
input_schema = NewPatternScanner.input_schema()
output_schema = NewPatternScanner.output_schema()
assert input_schema is not None
- assert output_schema is not None
\ No newline at end of file
+ assert output_schema is not None
diff --git a/flowsint-core/src/flowsint_core/tests/test_registry.py b/flowsint-core/src/flowsint_core/tests/test_registry.py
index 5388ef1c..ef0c7a7a 100644
--- a/flowsint-core/src/flowsint_core/tests/test_registry.py
+++ b/flowsint-core/src/flowsint_core/tests/test_registry.py
@@ -5,40 +5,41 @@ from flowsint_types.domain import Domain
from flowsint_types.ip import Ip
from typing import List
+
class MockScanner(Scanner):
"""Mock scanner for testing registry functionality"""
-
+
InputType = List[Domain]
OutputType = List[Ip]
-
+
@classmethod
def name(cls) -> str:
return "mock_scanner"
-
+
@classmethod
def category(cls) -> str:
return "Test"
-
+
@classmethod
def key(cls) -> str:
return "domain"
-
+
@classmethod
def documentation(cls) -> str:
return "Mock scanner for testing"
-
+
@classmethod
def get_params_schema(cls) -> List[dict]:
return []
-
+
@classmethod
def required_params(cls) -> bool:
return False
-
+
@classmethod
def icon(cls) -> str:
return "test-icon"
-
+
async def scan(self, data: InputType) -> OutputType:
return []
@@ -54,29 +55,28 @@ class TestScannerRegistry:
"""Test registering a scanner"""
ScannerRegistry.register(MockScanner)
assert ScannerRegistry.scanner_exists("mock_scanner")
-
+
scanners = ScannerRegistry.list()
assert "mock_scanner" in scanners
assert scanners["mock_scanner"]["class_name"] == "MockScanner"
def test_register_invalid_scanner(self):
"""Test that registering a non-scanner class raises error"""
+
class NotAScanner:
pass
-
+
with pytest.raises(ValueError, match="must inherit from Scanner"):
ScannerRegistry.register(NotAScanner)
def test_get_scanner_instance(self):
"""Test getting a scanner instance"""
ScannerRegistry.register(MockScanner)
-
+
scanner = ScannerRegistry.get_scanner(
- "mock_scanner",
- sketch_id="test_sketch",
- scan_id="test_scan"
+ "mock_scanner", sketch_id="test_sketch", scan_id="test_scan"
)
-
+
assert isinstance(scanner, MockScanner)
assert scanner.sketch_id == "test_sketch"
assert scanner.scan_id == "test_scan"
@@ -85,15 +85,13 @@ class TestScannerRegistry:
"""Test that getting a non-existent scanner raises exception"""
with pytest.raises(Exception, match="Scanner 'nonexistent' not found"):
ScannerRegistry.get_scanner(
- "nonexistent",
- sketch_id="test_sketch",
- scan_id="test_scan"
+ "nonexistent", sketch_id="test_sketch", scan_id="test_scan"
)
def test_list_by_categories(self):
"""Test listing scanners by category"""
ScannerRegistry.register(MockScanner)
-
+
by_category = ScannerRegistry.list_by_categories()
assert "Test" in by_category
assert len(by_category["Test"]) == 1
@@ -102,11 +100,11 @@ class TestScannerRegistry:
def test_list_by_input_type(self):
"""Test listing scanners by input type"""
ScannerRegistry.register(MockScanner)
-
+
domain_scanners = ScannerRegistry.list_by_input_type("Domain")
assert len(domain_scanners) == 1
assert domain_scanners[0]["name"] == "mock_scanner"
-
+
# Test with "any" input type
any_scanners = ScannerRegistry.list_by_input_type("any")
assert len(any_scanners) == 1
@@ -116,7 +114,7 @@ class TestScannerRegistry:
"""Test registering a module path"""
ScannerRegistry.register_module("test.module.path")
assert "test.module.path" in ScannerRegistry._scanner_modules
-
+
# Registering the same module again should not duplicate
ScannerRegistry.register_module("test.module.path")
assert ScannerRegistry._scanner_modules.count("test.module.path") == 1
diff --git a/flowsint-core/src/flowsint_core/tests/test_schema_generation.py b/flowsint-core/src/flowsint_core/tests/test_schema_generation.py
index cb2bcd8b..bb09fd8b 100644
--- a/flowsint-core/src/flowsint_core/tests/test_schema_generation.py
+++ b/flowsint-core/src/flowsint_core/tests/test_schema_generation.py
@@ -1,6 +1,7 @@
"""
Test schema generation for scanners with various InputType and OutputType combinations.
"""
+
from typing import List
from flowsint_types.website import Website
from flowsint_types.domain import Domain
@@ -12,90 +13,107 @@ from flowsint_transforms.domains.to_website import DomainToWebsiteScanner
class TestSchemaGeneration:
"""Test that schema generation correctly identifies InputType and OutputType."""
-
+
def test_website_to_domain_scanner_schemas(self):
"""Test that WebsiteToDomainScanner correctly shows Website as input type."""
scanner = WebsiteToDomainScanner
-
+
# Test InputType attribute
assert scanner.InputType == List[Website]
-
+
# Test input schema generation
input_schema = scanner.input_schema()
- assert input_schema["type"] == "Website", f"Expected 'Website', got '{input_schema['type']}'"
-
- # Test output schema generation
+ assert (
+ input_schema["type"] == "Website"
+ ), f"Expected 'Website', got '{input_schema['type']}'"
+
+ # Test output schema generation
output_schema = scanner.output_schema()
- assert output_schema["type"] == "Domain", f"Expected 'Domain', got '{output_schema['type']}'"
-
+ assert (
+ output_schema["type"] == "Domain"
+ ), f"Expected 'Domain', got '{output_schema['type']}'"
+
def test_website_to_webtrackers_scanner_schemas(self):
"""Test that WebsiteToWebtrackersScanner correctly shows Website as input type."""
scanner = WebsiteToWebtrackersScanner
-
+
# Test InputType attribute
assert scanner.InputType == List[Website]
-
+
# Test input schema generation
input_schema = scanner.input_schema()
- assert input_schema["type"] == "Website", f"Expected 'Website', got '{input_schema['type']}'"
-
+ assert (
+ input_schema["type"] == "Website"
+ ), f"Expected 'Website', got '{input_schema['type']}'"
+
# Test output schema generation
output_schema = scanner.output_schema()
- assert output_schema["type"] == "WebTracker", f"Expected 'WebTracker', got '{output_schema['type']}'"
-
+ assert (
+ output_schema["type"] == "WebTracker"
+ ), f"Expected 'WebTracker', got '{output_schema['type']}'"
+
def test_website_to_crawler_scanner_schemas(self):
"""Test that WebsiteToCrawler correctly shows Website as input type."""
scanner = WebsiteToCrawler
-
+
# Test InputType attribute
assert scanner.InputType == List[Website]
-
+
# Test input schema generation
input_schema = scanner.input_schema()
- assert input_schema["type"] == "Website", f"Expected 'Website', got '{input_schema['type']}'"
-
+ assert (
+ input_schema["type"] == "Website"
+ ), f"Expected 'Website', got '{input_schema['type']}'"
+
def test_domain_to_website_scanner_schemas(self):
"""Test that DomainToWebsiteScanner correctly shows Domain as input and Website as output."""
scanner = DomainToWebsiteScanner
-
+
# Test InputType attribute
assert scanner.InputType == List[Domain]
-
+
# Test input schema generation
input_schema = scanner.input_schema()
- assert input_schema["type"] == "Domain", f"Expected 'Domain', got '{input_schema['type']}'"
-
+ assert (
+ input_schema["type"] == "Domain"
+ ), f"Expected 'Domain', got '{input_schema['type']}'"
+
# Test output schema generation
output_schema = scanner.output_schema()
- assert output_schema["type"] == "Website", f"Expected 'Website', got '{output_schema['type']}'"
-
+ assert (
+ output_schema["type"] == "Website"
+ ), f"Expected 'Website', got '{output_schema['type']}'"
+
def test_all_website_scanners_have_correct_input_types(self):
"""Test that all scanners taking Website input show Website in schema."""
website_input_scanners = [
(WebsiteToDomainScanner, "Website", "Domain"),
- (WebsiteToWebtrackersScanner, "Website", "WebTracker"),
+ (WebsiteToWebtrackersScanner, "Website", "WebTracker"),
(WebsiteToCrawler, "Website", None), # Unknown output type
]
-
+
for scanner_class, expected_input, expected_output in website_input_scanners:
input_schema = scanner_class.input_schema()
- assert input_schema["type"] == expected_input, \
- f"{scanner_class.__name__}: Expected input '{expected_input}', got '{input_schema['type']}'"
-
+ assert (
+ input_schema["type"] == expected_input
+ ), f"{scanner_class.__name__}: Expected input '{expected_input}', got '{input_schema['type']}'"
+
if expected_output:
output_schema = scanner_class.output_schema()
- assert output_schema["type"] == expected_output, \
- f"{scanner_class.__name__}: Expected output '{expected_output}', got '{output_schema['type']}'"
+ assert (
+ output_schema["type"] == expected_output
+ ), f"{scanner_class.__name__}: Expected output '{expected_output}', got '{output_schema['type']}'"
def test_schema_generation_debug_info(self):
"""Debug test to see what's actually in the schemas."""
scanner = WebsiteToDomainScanner
-
+
# Get the raw TypeAdapter schema
from pydantic import TypeAdapter
+
adapter = TypeAdapter(scanner.InputType)
raw_schema = adapter.json_schema()
-
+
print(f"\n=== Debug Info for {scanner.__name__} ===")
print(f"InputType: {scanner.InputType}")
print(f"Raw schema keys: {list(raw_schema.keys())}")
@@ -103,43 +121,45 @@ class TestSchemaGeneration:
print(f"$defs keys: {list(raw_schema['$defs'].keys())}")
print(f"Schema items: {raw_schema.get('items', 'No items')}")
print(f"Generated input schema: {scanner.input_schema()}")
-
+
# This test always passes, it's just for debugging
assert True
-
+
def test_schema_generation_follows_ref_not_first_def(self):
"""
Regression test for the schema generation bug.
-
+
Before the fix: generate_input_schema() picked the first definition in $defs
(alphabetically "Domain" came before "Website"), so Website scanners incorrectly
showed "Domain" as their input type.
-
- After the fix: generate_input_schema() follows the $ref in items to get the
+
+ After the fix: generate_input_schema() follows the $ref in items to get the
correct type name.
"""
scanner = WebsiteToDomainScanner
-
+
# Get the raw schema to understand the structure
from pydantic import TypeAdapter
+
adapter = TypeAdapter(scanner.InputType)
raw_schema = adapter.json_schema()
-
+
# Verify the raw schema structure that caused the bug
assert "$defs" in raw_schema
assert "Domain" in raw_schema["$defs"]
assert "Website" in raw_schema["$defs"]
assert raw_schema["items"]["$ref"] == "#/$defs/Website"
-
+
# Before fix: list(raw_schema["$defs"].items())[0][0] would be "Domain" (first alphabetically)
first_def_name = list(raw_schema["$defs"].items())[0][0]
assert first_def_name == "Domain" # This would have been the bug
-
+
# After fix: We follow the $ref to get "Website"
ref_type = raw_schema["items"]["$ref"].split("/")[-1]
assert ref_type == "Website" # This is what we should use
-
+
# Verify our fix works correctly
input_schema = scanner.input_schema()
- assert input_schema["type"] == "Website", \
- f"Schema generation should follow $ref, not pick first def. Got '{input_schema['type']}'"
\ No newline at end of file
+ assert (
+ input_schema["type"] == "Website"
+ ), f"Schema generation should follow $ref, not pick first def. Got '{input_schema['type']}'"
diff --git a/flowsint-core/src/flowsint_core/utils.py b/flowsint-core/src/flowsint_core/utils.py
index b1e02a8e..238eed7c 100644
--- a/flowsint-core/src/flowsint_core/utils.py
+++ b/flowsint-core/src/flowsint_core/utils.py
@@ -13,6 +13,7 @@ import inspect
from typing import Any, Dict, Type
from pydantic import BaseModel, TypeAdapter
+
def is_valid_ip(address: str) -> bool:
try:
ipaddress.ip_address(address)
@@ -20,11 +21,13 @@ def is_valid_ip(address: str) -> bool:
except ValueError:
return False
+
def is_valid_username(username: str) -> bool:
if not re.match(r"^[a-zA-Z0-9_-]{3,30}$", username):
return False
return True
+
def is_valid_email(email: str) -> bool:
if not re.match(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$", email):
return False
@@ -34,26 +37,29 @@ def is_valid_email(email: str) -> bool:
def is_valid_domain(url_or_domain: str) -> str:
try:
- parsed = urlparse(url_or_domain if "://" in url_or_domain else "http://" + url_or_domain)
+ parsed = urlparse(
+ url_or_domain if "://" in url_or_domain else "http://" + url_or_domain
+ )
hostname = parsed.hostname or url_or_domain
-
+
if not hostname or "." not in hostname:
return False
-
+
if not re.match(r"^[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$", hostname):
return False
-
+
return True
except Exception as e:
return False
+
def is_root_domain(domain: str) -> bool:
"""
Determine if a domain is a root domain or subdomain.
-
+
Args:
domain: The domain string to check
-
+
Returns:
True if it's a root domain (e.g., example.com), False if it's a subdomain (e.g., sub.example.com)
"""
@@ -62,20 +68,32 @@ def is_root_domain(domain: str) -> bool:
if "://" in domain:
parsed = urlparse(domain)
domain = parsed.hostname or domain
-
+
# Split by dots
- parts = domain.split('.')
-
+ parts = domain.split(".")
+
# Handle common country code TLDs that have 2 parts (e.g., .co.uk, .com.au, .org.uk)
- common_cc_tlds = ['.co.uk', '.com.au', '.org.uk', '.net.uk', '.gov.uk', '.ac.uk',
- '.co.nz', '.com.sg', '.co.jp', '.co.kr', '.com.br', '.com.mx']
-
+ common_cc_tlds = [
+ ".co.uk",
+ ".com.au",
+ ".org.uk",
+ ".net.uk",
+ ".gov.uk",
+ ".ac.uk",
+ ".co.nz",
+ ".com.sg",
+ ".co.jp",
+ ".co.kr",
+ ".com.br",
+ ".com.mx",
+ ]
+
# Check if the domain ends with a common country code TLD
for cc_tld in common_cc_tlds:
if domain.endswith(cc_tld):
# For country code TLDs, we need exactly 3 parts (e.g., example.co.uk)
return len(parts) == 3
-
+
# For regular TLDs, a root domain has 2 parts (e.g., example.com)
# A subdomain has 3 or more parts (e.g., sub.example.com, www.sub.example.com)
return len(parts) == 2
@@ -83,6 +101,7 @@ def is_root_domain(domain: str) -> bool:
# If we can't parse it, assume it's not a root domain
return False
+
def is_valid_number(phone: str, region: str = "FR") -> None:
"""
Validates a phone number. Raises InvalidPhoneNumberError if invalid.
@@ -94,19 +113,21 @@ def is_valid_number(phone: str, region: str = "FR") -> None:
return False
except NumberParseException:
return False
-
+
+
def parse_asn(asn: str) -> int:
if not is_valid_asn(asn):
raise ValueError(f"Invalid ASN format: {asn}")
return int(re.sub(r"(?i)^AS", "", asn))
-
+
def is_valid_asn(asn: str) -> bool:
if not re.fullmatch(r"(AS)?\d+", asn, re.IGNORECASE):
return False
asn_num = int(re.sub(r"(?i)^AS", "", asn))
return 0 <= asn_num <= 4294967295
+
def resolve_type(details: dict, schema_context: dict = None) -> str:
if "anyOf" in details:
types = []
@@ -121,22 +142,23 @@ def resolve_type(details: dict, schema_context: dict = None) -> str:
else:
types.append(option.get("type", "unknown"))
return " | ".join(types)
-
+
if "type" in details:
if details["type"] == "array":
item_type = resolve_type(details.get("items", {}), schema_context)
return f"{item_type}[]"
return details["type"]
-
+
# Handle $ref in array items or other contexts
if "$ref" in details and schema_context:
ref_path = details["$ref"]
if ref_path.startswith("#/$defs/"):
ref_name = ref_path.split("/")[-1]
return ref_name
-
+
return "any"
+
def extract_input_schema_transform(model: Type[BaseModel]) -> Dict[str, Any]:
adapter = TypeAdapter(model)
schema = adapter.json_schema()
@@ -153,37 +175,30 @@ def extract_input_schema_transform(model: Type[BaseModel]) -> Dict[str, Any]:
"outputs": {
"type": type_name,
"properties": [
- {
- "name": prop,
- "type": resolve_type(info, schema)
- }
+ {"name": prop, "type": resolve_type(info, schema)}
for prop, info in details.get("properties", {}).items()
- ]
- },
- "inputs": {
- "type": "",
- "properties": []
+ ],
},
+ "inputs": {"type": "", "properties": []},
"type": "type",
"category": model.__name__,
}
-
def get_domain_from_ssl(ip: str, port: int = 443) -> str | None:
try:
context = ssl.create_default_context()
with socket.create_connection((ip, port), timeout=3) as sock:
with context.wrap_socket(sock, server_hostname=ip) as ssock:
cert = ssock.getpeercert()
- subject = cert.get('subject', [])
+ subject = cert.get("subject", [])
for entry in subject:
- if entry[0][0] == 'commonName':
+ if entry[0][0] == "commonName":
return entry[0][1]
# Alternative: check subjectAltName
- san = cert.get('subjectAltName', [])
+ san = cert.get("subjectAltName", [])
for typ, val in san:
- if typ == 'DNS':
+ if typ == "DNS":
return val
except Exception as e:
print(f"SSL extraction failed for {ip}: {e}")
@@ -208,12 +223,14 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:
scanner_node = node_lookup.get(target_id)
if scanner_node and scanner_node["data"]["type"] == "scanner":
- scanners.append({
- "scanner_name": scanner_node["data"]["name"],
- "module": scanner_node["data"]["module"],
- "input": source_handle,
- "output": target_handle
- })
+ scanners.append(
+ {
+ "scanner_name": scanner_node["data"]["name"],
+ "module": scanner_node["data"]["module"],
+ "input": source_handle,
+ "output": target_handle,
+ }
+ )
return {
"input": {
@@ -221,25 +238,22 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:
"outputs": input_output,
},
"scanners": scanners,
- "scanner_names" : [scanner["scanner_name"] for scanner in scanners]
-
+ "scanner_names": [scanner["scanner_name"] for scanner in scanners],
}
-
+
+
def get_label_color(label: str) -> str:
- color_map = {
- 'subdomain': '#A5ABB6',
- 'domain': '#68BDF6',
- 'default': '#A5ABB6'
- }
-
+ color_map = {"subdomain": "#A5ABB6", "domain": "#68BDF6", "default": "#A5ABB6"}
+
return color_map.get(label, color_map["default"])
+
def flatten(data_dict, prefix=""):
"""
Flattens a dictionary to contain only Neo4j-compatible property values.
Neo4j supports primitive types (string, number, boolean) and arrays of those types.
Args:
- data_dict (dict): Dictionary to flatten
+ data_dict (dict): Dictionary to flatten
Returns:
dict: Flattened dictionary with only Neo4j-compatible values
"""
@@ -250,13 +264,14 @@ def flatten(data_dict, prefix=""):
if value is None:
continue
if isinstance(value, (str, int, float, bool)) or (
- isinstance(value, list) and all(isinstance(item, (str, int, float, bool)) for item in value)
+ isinstance(value, list)
+ and all(isinstance(item, (str, int, float, bool)) for item in value)
):
key = f"{prefix}{key}"
- flattened[key] = value
+ flattened[key] = value
return flattened
-
+
def get_inline_relationships(nodes: List[Any], edges: List[Any]) -> List[str]:
"""
Get the inline relationships for a list of nodes and edges.
@@ -266,14 +281,15 @@ def get_inline_relationships(nodes: List[Any], edges: List[Any]) -> List[str]:
source = next((node for node in nodes if node["id"] == edge["source"]), None)
target = next((node for node in nodes if node["id"] == edge["target"]), None)
if source and target:
- relationships.append({"source":source, "edge":edge, "target":target})
+ relationships.append({"source": source, "edge": edge, "target": target})
return relationships
+
def to_json_serializable(obj):
"""Convert any object to a JSON-serializable format."""
import json
from pydantic import BaseModel
-
+
try:
# Test if already JSON serializable
json.dumps(obj)
@@ -282,11 +298,15 @@ def to_json_serializable(obj):
# Handle common cases
if isinstance(obj, BaseModel):
# Use mode='json' to ensure all Pydantic types are properly serialized
- return obj.model_dump(mode='json') if hasattr(obj, 'model_dump') else obj.dict()
+ return (
+ obj.model_dump(mode="json")
+ if hasattr(obj, "model_dump")
+ else obj.dict()
+ )
elif isinstance(obj, (list, tuple)):
return [to_json_serializable(item) for item in obj]
elif isinstance(obj, dict):
return {key: to_json_serializable(value) for key, value in obj.items()}
else:
# Convert anything else to string
- return str(obj)
\ No newline at end of file
+ return str(obj)
diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_nfts.py b/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_nfts.py
index bbdd45b4..014cdf39 100644
--- a/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_nfts.py
+++ b/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_nfts.py
@@ -8,38 +8,39 @@ from flowsint_core.core.graph_db import Neo4jConnection
ETHERSCAN_API_URL = os.getenv("ETHERSCAN_API_URL")
+
class CryptoWalletAddressToNFTs(Scanner):
"""Resolve NFTs for a wallet address (ETH)."""
-
+
# Define types as class attributes - base class handles schema generation automatically
InputType = List[CryptoWallet]
OutputType = List[CryptoNFT]
-
+
def __init__(
- self,
- sketch_id: Optional[str] = None,
- scan_id: Optional[str] = None,
- neo4j_conn: Optional[Neo4jConnection] = None,
- vault=None,
- params: Optional[Dict[str, Any]] = None
- ):
- super().__init__(
- sketch_id=sketch_id,
- scan_id=scan_id,
- neo4j_conn=neo4j_conn,
- params_schema=self.get_params_schema(),
- vault=vault,
- params=params
- )
+ self,
+ sketch_id: Optional[str] = None,
+ scan_id: Optional[str] = None,
+ neo4j_conn: Optional[Neo4jConnection] = None,
+ vault=None,
+ params: Optional[Dict[str, Any]] = None,
+ ):
+ super().__init__(
+ sketch_id=sketch_id,
+ scan_id=scan_id,
+ neo4j_conn=neo4j_conn,
+ params_schema=self.get_params_schema(),
+ vault=vault,
+ params=params,
+ )
@classmethod
def required_params(cls) -> bool:
return True
-
+
@classmethod
def icon(cls) -> str | None:
return "cryptowallet"
-
+
@classmethod
def get_params_schema(cls) -> List[Dict[str, Any]]:
"""Declare required parameters for this scanner"""
@@ -48,17 +49,17 @@ class CryptoWalletAddressToNFTs(Scanner):
"name": "ETHERSCAN_API_KEY",
"type": "vaultSecret",
"description": "The Etherscan API key to use for the transaction lookup.",
- "required": True
+ "required": True,
},
{
"name": "ETHERSCAN_API_URL",
"type": "url",
"description": "The Etherscan API URL to use for the transaction lookup.",
"required": False,
- "default": ETHERSCAN_API_URL
- }
+ "default": ETHERSCAN_API_URL,
+ },
]
-
+
@classmethod
def name(cls) -> str:
return "wallet_to_nfts"
@@ -66,7 +67,7 @@ class CryptoWalletAddressToNFTs(Scanner):
@classmethod
def category(cls) -> str:
return "CryptoWallet"
-
+
@classmethod
def key(cls) -> str:
return "address"
@@ -101,36 +102,37 @@ class CryptoWalletAddressToNFTs(Scanner):
except Exception as e:
print(f"Error resolving nfts for {d.address}: {e}")
return results
-
+
def _get_nfts(self, address: str, api_key: str, api_url: str) -> List[CryptoNFT]:
nfts = []
"""Get nfts for a wallet address."""
params = {
- "module": "account",
- "action": "tokennfttx",
- "address": address,
- "startblock": 0,
- "endblock": 99999999,
- "page": 1,
- "offset": 10000,
- "sort": "asc",
- "apikey": api_key
- }
+ "module": "account",
+ "action": "tokennfttx",
+ "address": address,
+ "startblock": 0,
+ "endblock": 99999999,
+ "page": 1,
+ "offset": 10000,
+ "sort": "asc",
+ "apikey": api_key,
+ }
response = requests.get(api_url, params=params)
data = response.json()
results = data["result"]
for tx in results:
- nfts.append(CryptoNFT(
- wallet=CryptoWallet(address=address),
- contract_address=tx["contractAddress"],
- token_id=tx["tokenID"],
- collection_name=tx["collectionName"],
- metadata_url=tx["metadataURL"],
- image_url=tx["imageURL"],
- name=tx["name"],
- ))
+ nfts.append(
+ CryptoNFT(
+ wallet=CryptoWallet(address=address),
+ contract_address=tx["contractAddress"],
+ token_id=tx["tokenID"],
+ collection_name=tx["collectionName"],
+ metadata_url=tx["metadataURL"],
+ image_url=tx["imageURL"],
+ name=tx["name"],
+ )
+ )
return nfts
-
def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
if not self.neo4j_conn:
@@ -139,27 +141,47 @@ class CryptoWalletAddressToNFTs(Scanner):
for nfts in results:
for nft in nfts:
# Create or update wallet node
- self.create_node('cryptowallet', 'wallet', nft.wallet.address,
- caption=nft.wallet.address, type='cryptowallet')
+ self.create_node(
+ "cryptowallet",
+ "wallet",
+ nft.wallet.address,
+ caption=nft.wallet.address,
+ type="cryptowallet",
+ )
- # Create or update NFT node
+ # Create or update NFT node
nft_key = f"{nft.contract_address}_{nft.token_id}"
- self.create_node('nft', 'nft_id', nft_key,
- contract_address=nft.contract_address,
- token_id=nft.token_id,
- collection_name=nft.collection_name,
- metadata_url=nft.metadata_url,
- image_url=nft.image_url,
- name=nft.name,
- caption=nft.name, type='nft')
+ self.create_node(
+ "nft",
+ "nft_id",
+ nft_key,
+ contract_address=nft.contract_address,
+ token_id=nft.token_id,
+ collection_name=nft.collection_name,
+ metadata_url=nft.metadata_url,
+ image_url=nft.image_url,
+ name=nft.name,
+ caption=nft.name,
+ type="nft",
+ )
# Create relationship from wallet to NFT
- self.create_relationship('cryptowallet', 'wallet', nft.wallet.address,
- 'nft', 'nft_id', nft_key, 'OWNS')
- self.log_graph_message(f"Found NFT for {nft.wallet.address}: {nft.contract_address} - {nft.token_id}")
+ self.create_relationship(
+ "cryptowallet",
+ "wallet",
+ nft.wallet.address,
+ "nft",
+ "nft_id",
+ nft_key,
+ "OWNS",
+ )
+ self.log_graph_message(
+ f"Found NFT for {nft.wallet.address}: {nft.contract_address} - {nft.token_id}"
+ )
return results
+
# Make types available at module level for easy access
InputType = CryptoWalletAddressToNFTs.InputType
-OutputType = CryptoWalletAddressToNFTs.OutputType
\ No newline at end of file
+OutputType = CryptoWalletAddressToNFTs.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_transactions.py b/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_transactions.py
index 7d4b860a..2a03ad8e 100644
--- a/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_transactions.py
+++ b/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_transactions.py
@@ -10,22 +10,24 @@ from flowsint_core.core.logger import Logger
ETHERSCAN_API_URL = os.getenv("ETHERSCAN_API_URL")
+
def wei_to_eth(wei_str):
return int(wei_str) / 10**18
+
class CryptoWalletAddressToTransactions(Scanner):
-
+
# Define types as class attributes - base class handles schema generation automatically
InputType = List[CryptoWallet]
OutputType = List[CryptoWalletTransaction]
-
+
def __init__(
self,
sketch_id: Optional[str] = None,
scan_id: Optional[str] = None,
neo4j_conn: Optional[Neo4jConnection] = None,
vault=None,
- params: Optional[Dict[str, Any]] = None
+ params: Optional[Dict[str, Any]] = None,
):
super().__init__(
sketch_id=sketch_id,
@@ -33,13 +35,13 @@ class CryptoWalletAddressToTransactions(Scanner):
neo4j_conn=neo4j_conn,
params_schema=self.get_params_schema(),
vault=vault,
- params=params
+ params=params,
)
@classmethod
def required_params(cls) -> bool:
return True
-
+
@classmethod
def icon(cls) -> str | None:
return "cryptowallet"
@@ -52,15 +54,15 @@ class CryptoWalletAddressToTransactions(Scanner):
"name": "ETHERSCAN_API_KEY",
"type": "vaultSecret",
"description": "The Etherscan API key to use for the transaction lookup.",
- "required": True
+ "required": True,
},
{
"name": "ETHERSCAN_API_URL",
"type": "url",
"description": "The Etherscan API URL to use for the transaction lookup.",
"required": False,
- "default": ETHERSCAN_API_URL
- }
+ "default": ETHERSCAN_API_URL,
+ },
]
@classmethod
@@ -70,7 +72,7 @@ class CryptoWalletAddressToTransactions(Scanner):
@classmethod
def category(cls) -> str:
return "CryptoWallet"
-
+
@classmethod
def key(cls) -> str:
return "address"
@@ -102,76 +104,90 @@ class CryptoWalletAddressToTransactions(Scanner):
transactions = await self._get_transactions(d.address, api_key, api_url)
results.append(transactions)
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"Error resolving transactions for {d.address}: {e}"})
+ Logger.error(
+ self.sketch_id,
+ {"message": f"Error resolving transactions for {d.address}: {e}"},
+ )
return results
-
- async def _get_transactions(self, address: str, api_key: str, api_url: str) -> List[CryptoWalletTransaction]:
+
+ async def _get_transactions(
+ self, address: str, api_key: str, api_url: str
+ ) -> List[CryptoWalletTransaction]:
transactions = []
"""Get transactions for a wallet address."""
params = {
- "module": "account",
- "action": "txlist",
- "address": address,
- "startblock": 0,
- "endblock": 99999999,
- "page": 1,
- "offset": 100,
- "sort": "asc",
- "apikey": api_key
- }
+ "module": "account",
+ "action": "txlist",
+ "address": address,
+ "startblock": 0,
+ "endblock": 99999999,
+ "page": 1,
+ "offset": 100,
+ "sort": "asc",
+ "apikey": api_key,
+ }
try:
response = requests.get(api_url, params=params)
-
+
# Raise an exception for HTTP errors (4xx or 5xx status codes)
response.raise_for_status()
-
+
except requests.exceptions.ConnectionError as e:
- raise ValueError(f"An error occurred connecting to {api_url}: Connection failed - {str(e)}")
+ raise ValueError(
+ f"An error occurred connecting to {api_url}: Connection failed - {str(e)}"
+ )
except requests.exceptions.Timeout as e:
- raise ValueError(f"An error occurred fetching {api_url}: Request timeout - {str(e)}")
+ raise ValueError(
+ f"An error occurred fetching {api_url}: Request timeout - {str(e)}"
+ )
except requests.exceptions.RequestException as e:
raise ValueError(f"An error occurred fetching {api_url}: {str(e)}")
-
+
try:
data = response.json()
except requests.exceptions.JSONDecodeError as e:
- raise ValueError(f"An error occurred fetching {api_url}: Invalid JSON response - {str(e)}")
-
+ raise ValueError(
+ f"An error occurred fetching {api_url}: Invalid JSON response - {str(e)}"
+ )
+
# Check if the API returned an error
if data.get("status") != "1":
error_message = data.get("message", "Unknown API error")
raise ValueError(f"An error occurred fetching {api_url}: {error_message}")
-
+
results = data.get("result", [])
for tx in results:
# Properly determine source and target based on transaction data
source_address = tx["from"]
-
+
if tx["to"] is not None:
target_address = tx["to"]
else:
# Contract creation transaction
- target_address = tx["contractAddress"] if tx["contractAddress"] else address
-
- transactions.append(CryptoWalletTransaction(
- source=CryptoWallet(address=source_address),
- target=CryptoWallet(address=target_address),
- hash=tx["hash"],
- value=wei_to_eth(tx["value"]),
- timestamp=tx["timeStamp"],
- block_number=tx["blockNumber"],
- block_hash=tx["blockHash"],
- nonce=tx["nonce"],
- transaction_index=tx["transactionIndex"],
- gas=tx["gas"],
- gas_price=tx["gasPrice"],
- gas_used=tx["gasUsed"],
- cumulative_gas_used=tx["cumulativeGasUsed"],
- input=tx["input"],
- contract_address=tx["contractAddress"],
- ))
+ target_address = (
+ tx["contractAddress"] if tx["contractAddress"] else address
+ )
+
+ transactions.append(
+ CryptoWalletTransaction(
+ source=CryptoWallet(address=source_address),
+ target=CryptoWallet(address=target_address),
+ hash=tx["hash"],
+ value=wei_to_eth(tx["value"]),
+ timestamp=tx["timeStamp"],
+ block_number=tx["blockNumber"],
+ block_hash=tx["blockHash"],
+ nonce=tx["nonce"],
+ transaction_index=tx["transactionIndex"],
+ gas=tx["gas"],
+ gas_price=tx["gasPrice"],
+ gas_used=tx["gasUsed"],
+ cumulative_gas_used=tx["cumulativeGasUsed"],
+ input=tx["input"],
+ contract_address=tx["contractAddress"],
+ )
+ )
return transactions
-
def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
if not self.neo4j_conn:
@@ -180,10 +196,20 @@ class CryptoWalletAddressToTransactions(Scanner):
for transactions in results:
for tx in transactions:
# Create or update both wallet nodes
- self.create_node('cryptowallet', 'wallet', tx.source.address,
- caption=tx.source.address, type="cryptowallet")
- self.create_node('cryptowallet', 'wallet', tx.target.address,
- caption=tx.target.address, type="cryptowallet")
+ self.create_node(
+ "cryptowallet",
+ "wallet",
+ tx.source.address,
+ caption=tx.source.address,
+ type="cryptowallet",
+ )
+ self.create_node(
+ "cryptowallet",
+ "wallet",
+ tx.target.address,
+ caption=tx.target.address,
+ type="cryptowallet",
+ )
# Create transaction as an edge between wallets (keeping complex query for transaction properties)
tx_query = """
@@ -207,30 +233,42 @@ class CryptoWalletAddressToTransactions(Scanner):
tx.caption = $hash,
tx.type = "transaction"
"""
- self.neo4j_conn.query(tx_query, {
- "hash": tx.hash,
- "source": tx.source.address,
- "target": tx.target.address,
- "value": tx.value,
- "timestamp": tx.timestamp,
- "block_number": tx.block_number,
- "block_hash": tx.block_hash,
- "nonce": tx.nonce,
- "transaction_index": tx.transaction_index,
- "gas": tx.gas,
- "gas_price": tx.gas_price,
- "gas_used": tx.gas_used,
- "cumulative_gas_used": tx.cumulative_gas_used,
- "input": tx.input,
- "contract_address": tx.contract_address,
- "sketch_id": self.sketch_id
- })
-
- timestamp_str = datetime.fromtimestamp(int(tx.timestamp)).strftime('%Y-%m-%d %H:%M:%S') if tx.timestamp else 'Unknown time'
- self.log_graph_message(f"Transaction on {timestamp_str}: {tx.source.address} -> {tx.target.address}")
+ self.neo4j_conn.query(
+ tx_query,
+ {
+ "hash": tx.hash,
+ "source": tx.source.address,
+ "target": tx.target.address,
+ "value": tx.value,
+ "timestamp": tx.timestamp,
+ "block_number": tx.block_number,
+ "block_hash": tx.block_hash,
+ "nonce": tx.nonce,
+ "transaction_index": tx.transaction_index,
+ "gas": tx.gas,
+ "gas_price": tx.gas_price,
+ "gas_used": tx.gas_used,
+ "cumulative_gas_used": tx.cumulative_gas_used,
+ "input": tx.input,
+ "contract_address": tx.contract_address,
+ "sketch_id": self.sketch_id,
+ },
+ )
+
+ timestamp_str = (
+ datetime.fromtimestamp(int(tx.timestamp)).strftime(
+ "%Y-%m-%d %H:%M:%S"
+ )
+ if tx.timestamp
+ else "Unknown time"
+ )
+ self.log_graph_message(
+ f"Transaction on {timestamp_str}: {tx.source.address} -> {tx.target.address}"
+ )
return results
+
# Make types available at module level for easy access
InputType = CryptoWalletAddressToTransactions.InputType
-OutputType = CryptoWalletAddressToTransactions.OutputType
\ No newline at end of file
+OutputType = CryptoWalletAddressToTransactions.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/domains/domain_to_asn.py b/flowsint-transforms/src/flowsint_transforms/domains/domain_to_asn.py
index 0a6f920c..21d6b162 100644
--- a/flowsint-transforms/src/flowsint_transforms/domains/domain_to_asn.py
+++ b/flowsint-transforms/src/flowsint_transforms/domains/domain_to_asn.py
@@ -8,6 +8,7 @@ from flowsint_types.asn import ASN
from flowsint_core.utils import is_valid_domain
from flowsint_core.core.logger import Logger
+
class DomainToAsnScanner(Scanner):
"""Takes a domain and returns its corresponding ASN."""
@@ -22,7 +23,7 @@ class DomainToAsnScanner(Scanner):
@classmethod
def category(cls) -> str:
return "Domain"
-
+
@classmethod
def key(cls) -> str:
return "Domain"
@@ -49,37 +50,43 @@ class DomainToAsnScanner(Scanner):
try:
# First resolve domain to IP
ip = socket.gethostbyname(domain.domain)
-
+
# Use asnmap to get ASN info
result = subprocess.run(
["asnmap", "-a", ip, "-json"],
capture_output=True,
text=True,
- timeout=30
+ timeout=30,
)
-
+
if result.returncode == 0:
output = result.stdout.strip()
if output:
asn_data = json.loads(output)
- if asn_data and 'as_number' in asn_data:
+ if asn_data and "as_number" in asn_data:
asn = ASN(
- asn=str(asn_data['as_number']),
- name=asn_data.get('as_name', ''),
- org=asn_data.get('as_org', ''),
- country=asn_data.get('as_country', '')
+ asn=str(asn_data["as_number"]),
+ name=asn_data.get("as_name", ""),
+ org=asn_data.get("as_org", ""),
+ country=asn_data.get("as_country", ""),
)
results.append(asn)
-
+
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"Error getting ASN for domain {domain.domain}: {e}"})
+ Logger.error(
+ self.sketch_id,
+ {"message": f"Error getting ASN for domain {domain.domain}: {e}"},
+ )
continue
-
+
return results
- def postprocess(self, results: OutputType, input_data: InputType = None) -> OutputType:
+ def postprocess(
+ self, results: OutputType, input_data: InputType = None
+ ) -> OutputType:
return results
+
# Make types available at module level for easy access
InputType = DomainToAsnScanner.InputType
-OutputType = DomainToAsnScanner.OutputType
\ No newline at end of file
+OutputType = DomainToAsnScanner.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/domains/resolve.py b/flowsint-transforms/src/flowsint_transforms/domains/resolve.py
index 0f38d94b..a9aa53c2 100644
--- a/flowsint-transforms/src/flowsint_transforms/domains/resolve.py
+++ b/flowsint-transforms/src/flowsint_transforms/domains/resolve.py
@@ -5,9 +5,10 @@ from flowsint_types.domain import Domain
from flowsint_types.ip import Ip
from flowsint_core.utils import is_valid_domain, is_root_domain
+
class ResolveScanner(Scanner):
"""Resolve domain names to IP addresses."""
-
+
# Define the input and output types as class attributes
InputType = List[Domain]
OutputType = List[Ip]
@@ -19,7 +20,7 @@ class ResolveScanner(Scanner):
@classmethod
def category(cls) -> str:
return "Domain"
-
+
@classmethod
def key(cls) -> str:
return "domain"
@@ -263,10 +264,14 @@ Check FlowSint logs for detailed resolution information.
if isinstance(item, str):
domain_obj = Domain(domain=item, root=is_root_domain(item))
elif isinstance(item, dict) and "domain" in item:
- domain_obj = Domain(domain=item["domain"], root=is_root_domain(item["domain"]))
+ domain_obj = Domain(
+ domain=item["domain"], root=is_root_domain(item["domain"])
+ )
elif isinstance(item, Domain):
# If the Domain object already exists, update its root field
- domain_obj = Domain(domain=item.domain, root=is_root_domain(item.domain))
+ domain_obj = Domain(
+ domain=item.domain, root=is_root_domain(item.domain)
+ )
if domain_obj and is_valid_domain(domain_obj.domain):
cleaned.append(domain_obj)
return cleaned
@@ -283,12 +288,28 @@ Check FlowSint logs for detailed resolution information.
def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
for domain_obj, ip_obj in zip(original_input, results):
- self.create_node('domain', 'domain', domain_obj.domain, type='domain' if domain_obj.root else 'subdomain')
- self.create_node('ip', 'address', ip_obj.address, type='ip')
- self.create_relationship('domain', 'domain', domain_obj.domain, 'ip', 'address', ip_obj.address, 'RESOLVES_TO')
- self.log_graph_message(f"IP found for domain {domain_obj.domain} -> {ip_obj.address}")
+ self.create_node(
+ "domain",
+ "domain",
+ domain_obj.domain,
+ type="domain" if domain_obj.root else "subdomain",
+ )
+ self.create_node("ip", "address", ip_obj.address, type="ip")
+ self.create_relationship(
+ "domain",
+ "domain",
+ domain_obj.domain,
+ "ip",
+ "address",
+ ip_obj.address,
+ "RESOLVES_TO",
+ )
+ self.log_graph_message(
+ f"IP found for domain {domain_obj.domain} -> {ip_obj.address}"
+ )
return results
+
# Make types available at module level for easy access
InputType = ResolveScanner.InputType
-OutputType = ResolveScanner.OutputType
\ No newline at end of file
+OutputType = ResolveScanner.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/domains/subdomains.py b/flowsint-transforms/src/flowsint_transforms/domains/subdomains.py
index 636bc7b7..83a11a79 100644
--- a/flowsint-transforms/src/flowsint_transforms/domains/subdomains.py
+++ b/flowsint-transforms/src/flowsint_transforms/domains/subdomains.py
@@ -6,10 +6,12 @@ from flowsint_types.domain import Domain
from flowsint_core.utils import is_valid_domain
from flowsint_core.core.logger import Logger
from tools.network.subfinder import SubfinderTool
+from flowsint_core.core.logger import Logger
+
class SubdomainScanner(Scanner):
"""Scanner to find subdomains associated with a domain."""
-
+
# Define types as class attributes - base class handles schema generation automatically
InputType = List[Domain | str]
OutputType = List[Domain]
@@ -26,7 +28,6 @@ class SubdomainScanner(Scanner):
def key(cls) -> str:
return "domain"
-
def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:
cleaned: InputType = []
for item in data:
@@ -40,7 +41,6 @@ class SubdomainScanner(Scanner):
if domain_obj and is_valid_domain(domain_obj.domain):
cleaned.append(domain_obj)
return cleaned
-
async def scan(self, data: InputType) -> OutputType:
"""Find subdomains using subfinder (if available) or fallback to crt.sh."""
@@ -52,7 +52,10 @@ class SubdomainScanner(Scanner):
if use_subfinder:
subdomains = self.__get_subdomains_from_subfinder(d.domain)
if not subdomains:
- Logger.warn(self.sketch_id, f"subfinder failed for {d.domain}, falling back to crt.sh")
+ Logger.warn(
+ self.sketch_id,
+ f"subfinder failed for {d.domain}, falling back to crt.sh",
+ )
subdomains = self.__get_subdomains_from_crtsh(d.domain)
else:
Logger.info(self.sketch_id, "subfinder not found, using crt.sh only")
@@ -69,8 +72,7 @@ class SubdomainScanner(Scanner):
subdomains: set[str] = set()
try:
response = requests.get(
- f"https://crt.sh/?q=%25.{domain}&output=json",
- timeout=60
+ f"https://crt.sh/?q=%25.{domain}&output=json", timeout=60
)
if response.ok:
entries = response.json()
@@ -78,7 +80,12 @@ class SubdomainScanner(Scanner):
name_value = entry.get("name_value", "")
for sub in name_value.split("\n"):
sub = sub.strip().lower()
- if "*" not in sub and is_valid_domain(sub) and sub.endswith(domain) and sub != domain:
+ if (
+ "*" not in sub
+ and is_valid_domain(sub)
+ and sub.endswith(domain)
+ and sub != domain
+ ):
subdomains.add(sub)
elif "*" in sub:
continue
@@ -102,20 +109,31 @@ class SubdomainScanner(Scanner):
continue
for subdomain in domain_obj["subdomains"]:
output.append(Domain(domain=subdomain))
- Logger.info(self.sketch_id, {"message": f"{domain_obj['domain']} -> {subdomain}"})
-
+ Logger.info(
+ self.sketch_id,
+ {"message": f"{domain_obj['domain']} -> {subdomain}"},
+ )
+
# Create subdomain node
- self.create_node('domain', 'domain', subdomain,
- type='domain')
-
+ self.create_node("domain", "domain", subdomain, type="domain")
+
# Create relationship from parent domain to subdomain
- self.create_relationship('domain', 'domain', domain_obj["domain"],
- 'domain', 'domain', subdomain, 'HAS_SUBDOMAIN')
-
- self.log_graph_message(f"{domain_obj['domain']} -> {len(domain_obj['subdomains'])} subdomain(s) found.")
+ self.create_relationship(
+ "domain",
+ "domain",
+ domain_obj["domain"],
+ "domain",
+ "domain",
+ subdomain,
+ "HAS_SUBDOMAIN",
+ )
+
+ self.log_graph_message(
+ f"{domain_obj['domain']} -> {len(domain_obj['subdomains'])} subdomain(s) found."
+ )
return output
InputType = SubdomainScanner.InputType
-OutputType = SubdomainScanner.OutputType
\ No newline at end of file
+OutputType = SubdomainScanner.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/domains/to_website.py b/flowsint-transforms/src/flowsint_transforms/domains/to_website.py
index 8bc3f8cb..70116968 100644
--- a/flowsint-transforms/src/flowsint_transforms/domains/to_website.py
+++ b/flowsint-transforms/src/flowsint_transforms/domains/to_website.py
@@ -6,6 +6,7 @@ from flowsint_types.domain import Domain
from flowsint_types.website import Website
from flowsint_core.core.logger import Logger
+
class DomainToWebsiteScanner(Scanner):
"""From domain to website."""
@@ -20,7 +21,7 @@ class DomainToWebsiteScanner(Scanner):
@classmethod
def category(cls) -> str:
return "Domain"
-
+
@classmethod
def key(cls) -> str:
return "domain"
@@ -48,13 +49,15 @@ class DomainToWebsiteScanner(Scanner):
# Try HTTPS first
try:
https_url = f"https://{domain.domain}"
- response = requests.head(https_url, timeout=10, allow_redirects=True)
+ response = requests.head(
+ https_url, timeout=10, allow_redirects=True
+ )
if response.status_code < 400:
results.append(Website(url=https_url, domain=domain))
continue
except requests.RequestException:
pass
-
+
# Try HTTP if HTTPS fails
try:
http_url = f"http://{domain.domain}"
@@ -64,15 +67,20 @@ class DomainToWebsiteScanner(Scanner):
continue
except requests.RequestException:
pass
-
+
# If both fail, still add HTTPS URL as default
results.append(Website(url=f"https://{domain.domain}", domain=domain))
-
+
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"Error converting domain {domain.domain} to website: {e}"})
+ Logger.error(
+ self.sketch_id,
+ {
+ "message": f"Error converting domain {domain.domain} to website: {e}"
+ },
+ )
# Add HTTPS URL as fallback
results.append(Website(url=f"https://{domain.domain}", domain=domain))
-
+
return results
def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
@@ -80,33 +88,58 @@ class DomainToWebsiteScanner(Scanner):
# Log each redirect step
if website.redirects:
for i, redirect_url in enumerate(website.redirects):
- next_url = website.redirects[i + 1] if i + 1 < len(website.redirects) else str(website.url)
+ next_url = (
+ website.redirects[i + 1]
+ if i + 1 < len(website.redirects)
+ else str(website.url)
+ )
redirect_payload = {
"message": f"Redirect: {str(redirect_url)} -> {str(next_url)}"
}
Logger.info(self.sketch_id, redirect_payload)
-
+
if self.neo4j_conn:
# Create domain node
- self.create_node('domain', 'domain', website.domain.domain, type="domain")
-
+ self.create_node(
+ "domain", "domain", website.domain.domain, type="domain"
+ )
+
# Create website node
- self.create_node('website', 'url', str(website.url),
- active=website.active,
- redirects=[str(redirect) for redirect in website.redirects] if website.redirects else [],
- type="website")
-
+ self.create_node(
+ "website",
+ "url",
+ str(website.url),
+ active=website.active,
+ redirects=(
+ [str(redirect) for redirect in website.redirects]
+ if website.redirects
+ else []
+ ),
+ type="website",
+ )
+
# Create relationship
- self.create_relationship('domain', 'domain', website.domain.domain,
- 'website', 'url', str(website.url), 'HAS_WEBSITE')
-
+ self.create_relationship(
+ "domain",
+ "domain",
+ website.domain.domain,
+ "website",
+ "url",
+ str(website.url),
+ "HAS_WEBSITE",
+ )
+
is_active_str = "active" if website.active else "inactive"
- redirects_str = f" (redirects: {len(website.redirects)})" if website.redirects else ""
- self.log_graph_message(f"{website.domain.domain} -> {str(website.url)} ({is_active_str}){redirects_str}")
+ redirects_str = (
+ f" (redirects: {len(website.redirects)})" if website.redirects else ""
+ )
+ self.log_graph_message(
+ f"{website.domain.domain} -> {str(website.url)} ({is_active_str}){redirects_str}"
+ )
return results
# Make types available at module level for easy access
InputType = DomainToWebsiteScanner.InputType
-OutputType = DomainToWebsiteScanner.OutputType
\ No newline at end of file
+OutputType = DomainToWebsiteScanner.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/domains/whois.py b/flowsint-transforms/src/flowsint_transforms/domains/whois.py
index 95e5cbd8..162887c4 100644
--- a/flowsint-transforms/src/flowsint_transforms/domains/whois.py
+++ b/flowsint-transforms/src/flowsint_transforms/domains/whois.py
@@ -7,6 +7,7 @@ from flowsint_types.whois import Whois
from flowsint_types.email import Email
from flowsint_core.core.logger import Logger
+
class WhoisScanner(Scanner):
"""Scan for WHOIS information of a domain."""
@@ -21,7 +22,7 @@ class WhoisScanner(Scanner):
@classmethod
def category(cls) -> str:
return "Domain"
-
+
@classmethod
def key(cls) -> str:
return "domain"
@@ -52,92 +53,142 @@ class WhoisScanner(Scanner):
emails = []
if whois_info.emails:
if isinstance(whois_info.emails, list):
- emails = [Email(email=email) for email in whois_info.emails if email]
+ emails = [
+ Email(email=email)
+ for email in whois_info.emails
+ if email
+ ]
else:
emails = [Email(email=whois_info.emails)]
-
+
# Convert datetime objects to ISO format strings
creation_date_str = None
if whois_info.creation_date:
if isinstance(whois_info.creation_date, list):
- creation_date_str = whois_info.creation_date[0].isoformat() if whois_info.creation_date else None
+ creation_date_str = (
+ whois_info.creation_date[0].isoformat()
+ if whois_info.creation_date
+ else None
+ )
else:
creation_date_str = whois_info.creation_date.isoformat()
-
+
expiration_date_str = None
if whois_info.expiration_date:
if isinstance(whois_info.expiration_date, list):
- expiration_date_str = whois_info.expiration_date[0].isoformat() if whois_info.expiration_date else None
+ expiration_date_str = (
+ whois_info.expiration_date[0].isoformat()
+ if whois_info.expiration_date
+ else None
+ )
else:
expiration_date_str = whois_info.expiration_date.isoformat()
-
+
whois_obj = Whois(
domain=domain.domain,
- registrar=str(whois_info.registrar) if whois_info.registrar else None,
+ registrar=(
+ str(whois_info.registrar) if whois_info.registrar else None
+ ),
org=str(whois_info.org) if whois_info.org else None,
city=str(whois_info.city) if whois_info.city else None,
country=str(whois_info.country) if whois_info.country else None,
email=emails[0] if emails else None,
creation_date=creation_date_str,
- expiration_date=expiration_date_str
+ expiration_date=expiration_date_str,
)
results.append(whois_obj)
-
+
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"Error getting WHOIS for domain {domain.domain}: {e}"})
+ Logger.error(
+ self.sketch_id,
+ {"message": f"Error getting WHOIS for domain {domain.domain}: {e}"},
+ )
continue
-
+
return results
def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
for whois_obj in results:
if not self.neo4j_conn:
continue
-
+
# Create domain node
- self.create_node('domain', 'domain', whois_obj.domain, type='domain')
-
- # Create whois node
+ self.create_node("domain", "domain", whois_obj.domain, type="domain")
+
+ # Create whois node
whois_key = f"{whois_obj.domain}_{self.sketch_id}"
- self.create_node('whois', 'whois_id', whois_key,
- domain=whois_obj.domain,
- registrar=whois_obj.registrar,
- org=whois_obj.org,
- city=whois_obj.city,
- country=whois_obj.country,
- creation_date=whois_obj.creation_date,
- expiration_date=whois_obj.expiration_date,
- email=whois_obj.email.email if whois_obj.email else None,
- label="Whois", type="whois")
-
+ self.create_node(
+ "whois",
+ "whois_id",
+ whois_key,
+ domain=whois_obj.domain,
+ registrar=whois_obj.registrar,
+ org=whois_obj.org,
+ city=whois_obj.city,
+ country=whois_obj.country,
+ creation_date=whois_obj.creation_date,
+ expiration_date=whois_obj.expiration_date,
+ email=whois_obj.email.email if whois_obj.email else None,
+ label="Whois",
+ type="whois",
+ )
+
# Create relationship between domain and whois
- self.create_relationship('domain', 'domain', whois_obj.domain,
- 'whois', 'whois_id', whois_key, 'HAS_WHOIS')
+ self.create_relationship(
+ "domain",
+ "domain",
+ whois_obj.domain,
+ "whois",
+ "whois_id",
+ whois_key,
+ "HAS_WHOIS",
+ )
# Create organization node if org information is available
if whois_obj.org:
- self.create_node('organization', 'name', whois_obj.org,
- country=whois_obj.country,
- founding_date=whois_obj.creation_date,
- description=f"Organization from WHOIS data for {whois_obj.domain}",
- caption=whois_obj.org, type="organization")
-
+ self.create_node(
+ "organization",
+ "name",
+ whois_obj.org,
+ country=whois_obj.country,
+ founding_date=whois_obj.creation_date,
+ description=f"Organization from WHOIS data for {whois_obj.domain}",
+ caption=whois_obj.org,
+ type="organization",
+ )
+
# Create relationship between organization and domain
- self.create_relationship('organization', 'name', whois_obj.org,
- 'domain', 'domain', whois_obj.domain, 'HAS_DOMAIN')
-
- self.log_graph_message(f"{whois_obj.domain} -> {whois_obj.org} (organization)")
+ self.create_relationship(
+ "organization",
+ "name",
+ whois_obj.org,
+ "domain",
+ "domain",
+ whois_obj.domain,
+ "HAS_DOMAIN",
+ )
- if whois_obj.email:
- self.create_node('email', 'email', whois_obj.email.email, type="email")
- self.create_relationship('whois', 'whois_id', whois_key,
- 'email', 'email', whois_obj.email.email, 'REGISTERED_BY')
-
- self.log_graph_message(f"WHOIS for {whois_obj.domain} -> registrar: {whois_obj.registrar} org: {whois_obj.org} city: {whois_obj.city} country: {whois_obj.country} creation_date: {whois_obj.creation_date} expiration_date: {whois_obj.expiration_date}")
+ self.log_graph_message(
+ f"{whois_obj.domain} -> {whois_obj.org} (organization)"
+ )
- return results
+ if whois_obj.email:
+ self.create_node("email", "email", whois_obj.email.email, type="email")
+ self.create_relationship(
+ "whois",
+ "whois_id",
+ whois_key,
+ "email",
+ "email",
+ whois_obj.email.email,
+ "REGISTERED_BY",
+ )
+ self.log_graph_message(
+ f"WHOIS for {whois_obj.domain} -> registrar: {whois_obj.registrar} org: {whois_obj.org} city: {whois_obj.city} country: {whois_obj.country} creation_date: {whois_obj.creation_date} expiration_date: {whois_obj.expiration_date}"
+ )
+ return results
# Make types available at module level for easy access
diff --git a/flowsint-transforms/src/flowsint_transforms/emails/to_gravatar.py b/flowsint-transforms/src/flowsint_transforms/emails/to_gravatar.py
index 84fd2b9e..c8d418bc 100644
--- a/flowsint-transforms/src/flowsint_transforms/emails/to_gravatar.py
+++ b/flowsint-transforms/src/flowsint_transforms/emails/to_gravatar.py
@@ -6,6 +6,7 @@ from flowsint_core.core.logger import Logger
from flowsint_types.email import Email
from flowsint_types.gravatar import Gravatar
+
class EmailToGravatarScanner(Scanner):
"""From md5 hash of email to gravatar."""
@@ -19,7 +20,7 @@ class EmailToGravatarScanner(Scanner):
@classmethod
def category(cls) -> str:
return "Email"
-
+
@classmethod
def key(cls) -> str:
return "email"
@@ -40,75 +41,97 @@ class EmailToGravatarScanner(Scanner):
async def scan(self, data: InputType) -> OutputType:
results: OutputType = []
-
+
for email in data:
try:
# Generate MD5 hash of email
email_hash = hashlib.md5(email.email.lower().encode()).hexdigest()
-
+
# Query Gravatar API
gravatar_url = f"https://www.gravatar.com/avatar/{email_hash}?d=404"
response = requests.head(gravatar_url, timeout=10)
-
+
if response.status_code == 200:
# Gravatar found, get profile info
profile_url = f"https://www.gravatar.com/{email_hash}.json"
profile_response = requests.get(profile_url, timeout=10)
-
+
gravatar_data = {
"src": gravatar_url,
"hash": email_hash,
"profile_url": profile_url,
- "exists": True
+ "exists": True,
}
-
+
if profile_response.status_code == 200:
profile_data = profile_response.json()
if "entry" in profile_data and profile_data["entry"]:
entry = profile_data["entry"][0]
- gravatar_data.update({
- "display_name": entry.get("displayName"),
- "about_me": entry.get("aboutMe"),
- "location": entry.get("currentLocation")
- })
-
+ gravatar_data.update(
+ {
+ "display_name": entry.get("displayName"),
+ "about_me": entry.get("aboutMe"),
+ "location": entry.get("currentLocation"),
+ }
+ )
+
gravatar = Gravatar(**gravatar_data)
results.append(gravatar)
-
+
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"Error checking Gravatar for email {email.email}: {e}"})
+ Logger.error(
+ self.sketch_id,
+ {
+ "message": f"Error checking Gravatar for email {email.email}: {e}"
+ },
+ )
continue
-
+
return results
def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
for email_obj, gravatar_obj in zip(original_input, results):
if not self.neo4j_conn:
continue
-
+
# Create email node
- self.create_node('email', 'email', email_obj.email, type='email')
-
+ self.create_node("email", "email", email_obj.email, type="email")
+
# Create gravatar node
gravatar_key = f"{email_obj.email}_{self.sketch_id}"
- self.create_node('gravatar', 'gravatar_id', gravatar_key,
- email=email_obj.email,
- hash=gravatar_obj.hash,
- src=str(gravatar_obj.src),
- display_name=gravatar_obj.display_name,
- location=gravatar_obj.location,
- about_me=gravatar_obj.about_me,
- exists=gravatar_obj.exists,
- label="Gravatar", type="gravatar")
-
+ self.create_node(
+ "gravatar",
+ "gravatar_id",
+ gravatar_key,
+ email=email_obj.email,
+ hash=gravatar_obj.hash,
+ src=str(gravatar_obj.src),
+ display_name=gravatar_obj.display_name,
+ location=gravatar_obj.location,
+ about_me=gravatar_obj.about_me,
+ exists=gravatar_obj.exists,
+ label="Gravatar",
+ type="gravatar",
+ )
+
# Create relationship between email and gravatar
- self.create_relationship('email', 'email', email_obj.email,
- 'gravatar', 'gravatar_id', gravatar_key, 'HAS_GRAVATAR')
-
- self.log_graph_message(f"Gravatar found for email {email_obj.email} -> hash: {gravatar_obj.hash}")
-
+ self.create_relationship(
+ "email",
+ "email",
+ email_obj.email,
+ "gravatar",
+ "gravatar_id",
+ gravatar_key,
+ "HAS_GRAVATAR",
+ )
+
+ self.log_graph_message(
+ f"Gravatar found for email {email_obj.email} -> hash: {gravatar_obj.hash}"
+ )
+
return results
+
# Make types available at module level for easy access
InputType = EmailToGravatarScanner.InputType
-OutputType = EmailToGravatarScanner.OutputType
\ No newline at end of file
+OutputType = EmailToGravatarScanner.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/emails/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/emails/to_leaks.py
index c552ff8f..cc04f33d 100644
--- a/flowsint-transforms/src/flowsint_transforms/emails/to_leaks.py
+++ b/flowsint-transforms/src/flowsint_transforms/emails/to_leaks.py
@@ -14,28 +14,29 @@ load_dotenv()
HIBP_API_KEY = os.getenv("HIBP_API_KEY")
+
class EmailToBreachesScanner(Scanner):
"""From email to breaches using Have I Been Pwned API."""
InputType = List[Email]
OutputType = List[tuple] # List of (email, breach) tuples
-
+
def __init__(
- self,
- sketch_id: Optional[str] = None,
- scan_id: Optional[str] = None,
- neo4j_conn: Optional[Neo4jConnection] = None,
- vault=None,
- params: Optional[Dict[str, Any]] = None
- ):
- super().__init__(
- sketch_id=sketch_id,
- scan_id=scan_id,
- neo4j_conn=neo4j_conn,
- params_schema=self.get_params_schema(),
- vault=vault,
- params=params
- )
+ self,
+ sketch_id: Optional[str] = None,
+ scan_id: Optional[str] = None,
+ neo4j_conn: Optional[Neo4jConnection] = None,
+ vault=None,
+ params: Optional[Dict[str, Any]] = None,
+ ):
+ super().__init__(
+ sketch_id=sketch_id,
+ scan_id=scan_id,
+ neo4j_conn=neo4j_conn,
+ params_schema=self.get_params_schema(),
+ vault=vault,
+ params=params,
+ )
@classmethod
def name(cls) -> str:
@@ -44,15 +45,15 @@ class EmailToBreachesScanner(Scanner):
@classmethod
def category(cls) -> str:
return "Email"
-
+
@classmethod
def key(cls) -> str:
return "email"
-
+
@classmethod
def required_params(cls) -> bool:
return True
-
+
@classmethod
def get_params_schema(cls) -> List[Dict[str, Any]]:
"""Declare required parameters for this scanner"""
@@ -61,15 +62,15 @@ class EmailToBreachesScanner(Scanner):
"name": "HIBP_API_KEY",
"type": "vaultSecret",
"description": "The HIBP API key to use for breaches lookup.",
- "required": True
+ "required": True,
},
{
"name": "HIBP_API_URL",
"type": "url",
"description": "The HIBP API URL to use for breaches lookup.",
"required": False,
- "default": "https://haveibeenpwned.com/api/v3/breachedaccount/"
- }
+ "default": "https://haveibeenpwned.com/api/v3/breachedaccount/",
+ },
]
def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:
@@ -92,13 +93,15 @@ class EmailToBreachesScanner(Scanner):
api_key = self.get_params().get("HIBP_API_KEY", None)
api_url = self.get_params().get("HIBP_API_URL", None)
if not api_key:
- Logger.error(self.sketch_id, {"message": "A valid HIBP_API_KEY is required to scan for breaches."})
+ Logger.error(
+ self.sketch_id,
+ {"message": "A valid HIBP_API_KEY is required to scan for breaches."},
+ )
if not api_url:
- Logger.error(self.sketch_id, {"message": "Could not find HIBP_API_URL in params."})
- headers = {
- "hibp-api-key": api_key,
- "User-Agent": "FlowsInt-Scanner"
- }
+ Logger.error(
+ self.sketch_id, {"message": "Could not find HIBP_API_URL in params."}
+ )
+ headers = {"hibp-api-key": api_key, "User-Agent": "FlowsInt-Scanner"}
Logger.info(self.sketch_id, {"message": f"HIBP API URL: {api_url}"})
for email in data:
try:
@@ -106,10 +109,17 @@ class EmailToBreachesScanner(Scanner):
full_url = urljoin(api_url, f"{email.email}?truncateResponse=false")
Logger.error(self.sketch_id, {"message": f"full url: {full_url}"})
response = requests.get(full_url, headers=headers, timeout=10)
- Logger.info(self.sketch_id, {"message": f"HIBP API response: {response.json()}"})
+ Logger.info(
+ self.sketch_id, {"message": f"HIBP API response: {response.json()}"}
+ )
if response.status_code == 200:
breaches_data = response.json()
- Logger.info(self.sketch_id, {"message": f"Found {len(breaches_data)} breaches for {email.email}"})
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"Found {len(breaches_data)} breaches for {email.email}"
+ },
+ )
for breach_data in breaches_data:
breach = Breach(
name=breach_data.get("Name", ""),
@@ -126,60 +136,103 @@ class EmailToBreachesScanner(Scanner):
issensitive=breach_data.get("IsSensitive", False),
isretired=breach_data.get("IsRetired", False),
isspamlist=breach_data.get("IsSpamList", False),
- logopath=breach_data.get("LogoPath", "")
+ logopath=breach_data.get("LogoPath", ""),
)
# Store email and breach as a tuple
results.append((email.email, breach))
- Logger.info(self.sketch_id, {"message": f"Added breach: {breach.name} for email: {email.email}"})
-
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"Added breach: {breach.name} for email: {email.email}"
+ },
+ )
+
elif response.status_code == 404:
# No breaches found for this email
- Logger.info(self.sketch_id, {"message": f"No breaches found for email {email.email}"})
+ Logger.info(
+ self.sketch_id,
+ {"message": f"No breaches found for email {email.email}"},
+ )
continue
-
+
else:
- Logger.error(self.sketch_id, {"message": f"HIBP API error for {email.email}: {response.status_code}"})
+ Logger.error(
+ self.sketch_id,
+ {
+ "message": f"HIBP API error for {email.email}: {response.status_code}"
+ },
+ )
continue
-
+
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"Error checking breaches for email {email.email}: {e}"})
+ Logger.error(
+ self.sketch_id,
+ {
+ "message": f"Error checking breaches for email {email.email}: {e}"
+ },
+ )
continue
-
- Logger.info(self.sketch_id, {"message": f"Scan completed. Total results: {len(results)}"})
+
+ Logger.info(
+ self.sketch_id,
+ {"message": f"Scan completed. Total results: {len(results)}"},
+ )
return results
def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
- Logger.info(self.sketch_id, {"message": f"Postprocess started. Results count: {len(results)}, Original input count: {len(original_input)}"})
-
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"Postprocess started. Results count: {len(results)}, Original input count: {len(original_input)}"
+ },
+ )
+
# Create email nodes first
for email_obj in original_input:
if not self.neo4j_conn:
continue
# Create email node
- self.create_node('email', 'email', email_obj.email, type='email')
- Logger.info(self.sketch_id, {"message": f"Created email node: {email_obj.email}"})
-
+ self.create_node("email", "email", email_obj.email, type="email")
+ Logger.info(
+ self.sketch_id, {"message": f"Created email node: {email_obj.email}"}
+ )
+
# Process all breaches
for email_address, breach_obj in results:
if not self.neo4j_conn:
continue
-
+
# Create breach node
breach_key = f"{breach_obj.name}_{self.sketch_id}"
self.create_node(
- 'breach', 'breach_id', breach_key,
- **breach_obj.dict(),
- label=breach_obj.name, type="breach"
+ "breach",
+ "breach_id",
+ breach_key,
+ **breach_obj.dict(),
+ label=breach_obj.name,
+ type="breach",
)
- Logger.info(self.sketch_id, {"message": f"Created breach node: {breach_key}"})
-
+ Logger.info(
+ self.sketch_id, {"message": f"Created breach node: {breach_key}"}
+ )
+
# Create relationship between the specific email and this breach
- self.create_relationship('email', 'email', email_address,
- 'breach', 'breach_id', breach_key, 'FOUND_IN_BREACH')
- self.log_graph_message(f"Breach found for email {email_address} -> {breach_obj.name} ({breach_obj.title})")
-
+ self.create_relationship(
+ "email",
+ "email",
+ email_address,
+ "breach",
+ "breach_id",
+ breach_key,
+ "FOUND_IN_BREACH",
+ )
+ self.log_graph_message(
+ f"Breach found for email {email_address} -> {breach_obj.name} ({breach_obj.title})"
+ )
+
return results
+
# Make types available at module level for easy access
InputType = EmailToBreachesScanner.InputType
-OutputType = EmailToBreachesScanner.OutputType
\ No newline at end of file
+OutputType = EmailToBreachesScanner.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/individuals/to_org.py b/flowsint-transforms/src/flowsint_transforms/individuals/to_org.py
index ae1eb669..b4f3351c 100644
--- a/flowsint-transforms/src/flowsint_transforms/individuals/to_org.py
+++ b/flowsint-transforms/src/flowsint_transforms/individuals/to_org.py
@@ -6,6 +6,7 @@ from flowsint_types.individual import Individual
from flowsint_core.core.logger import Logger
from tools.organizations.sirene import SireneTool
+
class IndividualToOrgScanner(Scanner):
"""Find organization from a person with data from SIRENE (France only)."""
@@ -20,7 +21,7 @@ class IndividualToOrgScanner(Scanner):
@classmethod
def category(cls) -> str:
return "Individual"
-
+
@classmethod
def key(cls) -> str:
return "fullname"
@@ -35,29 +36,45 @@ class IndividualToOrgScanner(Scanner):
if len(parts) >= 2:
firstname = parts[0]
lastname = " ".join(parts[1:])
- cleaned.append(Individual(first_name=firstname, last_name=lastname, full_name=item))
- elif isinstance(item, dict) and item.get("first_name") and item.get("last_name"):
+ cleaned.append(
+ Individual(
+ first_name=firstname, last_name=lastname, full_name=item
+ )
+ )
+ elif (
+ isinstance(item, dict)
+ and item.get("first_name")
+ and item.get("last_name")
+ ):
cleaned.append(Individual(**item))
elif isinstance(item, Individual):
cleaned.append(item)
- if(len(cleaned)==0):
- Logger.error(self.sketch_id, {"message": "[INDIVIDUAL_TO_ORG] The input type did not match."})
+ if len(cleaned) == 0:
+ Logger.error(
+ self.sketch_id,
+ {"message": "[INDIVIDUAL_TO_ORG] The input type did not match."},
+ )
return cleaned
async def scan(self, data: InputType) -> OutputType:
-
+
results: OutputType = []
for individual in data:
try:
sirene = SireneTool()
raw_orgs = sirene.launch(individual.full_name, limit=25)
- if len(raw_orgs)> 0:
+ if len(raw_orgs) > 0:
for org_dict in raw_orgs:
enriched_org = self.enrich_org(org_dict)
if enriched_org is not None:
results.append(enriched_org)
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"Error finding organization for {individual.full_name}: {e}"})
+ Logger.error(
+ self.sketch_id,
+ {
+ "message": f"Error finding organization for {individual.full_name}: {e}"
+ },
+ )
return results
def enrich_org(self, company: Dict) -> Organization:
@@ -68,15 +85,16 @@ class IndividualToOrgScanner(Scanner):
siege_geo_adresse = None
if siege.get("latitude") and siege.get("longitude"):
from flowsint_types.address import PhysicalAddress
+
siege_geo_adresse = PhysicalAddress(
address=siege.get("adresse", ""),
city=siege.get("libelle_commune", ""),
country="FR", # SIRENE is French registry
zip=siege.get("code_postal", ""),
latitude=float(siege.get("latitude", 0)),
- longitude=float(siege.get("longitude", 0))
+ longitude=float(siege.get("longitude", 0)),
)
-
+
# Extract dirigeants and convert to Individual objects
dirigeants = []
for dirigeant_data in company.get("dirigeants", []):
@@ -85,17 +103,20 @@ class IndividualToOrgScanner(Scanner):
last_name=dirigeant_data.get("nom", ""),
full_name=f"{dirigeant_data.get('prenoms', '')} {dirigeant_data.get('nom', '')}".strip(),
birth_date=dirigeant_data.get("date_de_naissance"),
- gender=None # Not available in SIRENE data
+ gender=None, # Not available in SIRENE data
)
dirigeants.append(dirigeant)
-
+
name = company.get("nom_raison_sociale") or company.get("nom_complet")
-
+
# Ensure we have a valid name
if not name:
- Logger.error(self.sketch_id, {"message": f"Organization has no valid name: {company}"})
+ Logger.error(
+ self.sketch_id,
+ {"message": f"Organization has no valid name: {company}"},
+ )
return None
-
+
return Organization(
# Basic information
name=name,
@@ -103,40 +124,41 @@ class IndividualToOrgScanner(Scanner):
nom_complet=company.get("nom_complet"),
nom_raison_sociale=company.get("nom_raison_sociale"),
sigle=company.get("sigle"),
-
# Establishment information
nombre_etablissements=company.get("nombre_etablissements"),
- nombre_etablissements_ouverts=company.get("nombre_etablissements_ouverts"),
-
+ nombre_etablissements_ouverts=company.get(
+ "nombre_etablissements_ouverts"
+ ),
# Activity information
activite_principale=company.get("activite_principale"),
section_activite_principale=company.get("section_activite_principale"),
-
# Company category
categorie_entreprise=company.get("categorie_entreprise"),
annee_categorie_entreprise=company.get("annee_categorie_entreprise"),
-
# Employment information
caractere_employeur=company.get("caractere_employeur"),
tranche_effectif_salarie=company.get("tranche_effectif_salarie"),
- annee_tranche_effectif_salarie=company.get("annee_tranche_effectif_salarie"),
-
+ annee_tranche_effectif_salarie=company.get(
+ "annee_tranche_effectif_salarie"
+ ),
# Dates
date_creation=company.get("date_creation"),
date_fermeture=company.get("date_fermeture"),
date_mise_a_jour=company.get("date_mise_a_jour"),
date_mise_a_jour_insee=company.get("date_mise_a_jour_insee"),
date_mise_a_jour_rne=company.get("date_mise_a_jour_rne"),
-
# Legal information
nature_juridique=company.get("nature_juridique"),
etat_administratif=company.get("etat_administratif"),
statut_diffusion=company.get("statut_diffusion"),
-
# Siege (Headquarters) information
siege_activite_principale=siege.get("activite_principale"),
- siege_activite_principale_registre_metier=siege.get("activite_principale_registre_metier"),
- siege_annee_tranche_effectif_salarie=siege.get("annee_tranche_effectif_salarie"),
+ siege_activite_principale_registre_metier=siege.get(
+ "activite_principale_registre_metier"
+ ),
+ siege_annee_tranche_effectif_salarie=siege.get(
+ "annee_tranche_effectif_salarie"
+ ),
siege_adresse=siege.get("adresse"),
siege_caractere_employeur=siege.get("caractere_employeur"),
siege_cedex=siege.get("cedex"),
@@ -168,7 +190,9 @@ class IndividualToOrgScanner(Scanner):
siege_liste_finess=siege.get("liste_finess"),
siege_liste_id_bio=siege.get("liste_id_bio"),
siege_liste_idcc=siege.get("liste_idcc"),
- siege_liste_id_organisme_formation=siege.get("liste_id_organisme_formation"),
+ siege_liste_id_organisme_formation=siege.get(
+ "liste_id_organisme_formation"
+ ),
siege_liste_rge=siege.get("liste_rge"),
siege_liste_uai=siege.get("liste_uai"),
siege_longitude=siege.get("longitude"),
@@ -176,51 +200,85 @@ class IndividualToOrgScanner(Scanner):
siege_numero_voie=siege.get("numero_voie"),
siege_region=siege.get("region"),
siege_siret=siege.get("siret"),
- siege_statut_diffusion_etablissement=siege.get("statut_diffusion_etablissement"),
+ siege_statut_diffusion_etablissement=siege.get(
+ "statut_diffusion_etablissement"
+ ),
siege_tranche_effectif_salarie=siege.get("tranche_effectif_salarie"),
siege_type_voie=siege.get("type_voie"),
-
# Dirigeants (Leaders)
dirigeants=dirigeants if dirigeants else None,
-
# Matching establishments
matching_etablissements=company.get("matching_etablissements"),
-
# Financial information
finances=company.get("finances"),
-
# Complements (Additional information)
- complements_collectivite_territoriale=company.get("complements", {}).get("collectivite_territoriale"),
- complements_convention_collective_renseignee=company.get("complements", {}).get("convention_collective_renseignee"),
+ complements_collectivite_territoriale=company.get(
+ "complements", {}
+ ).get("collectivite_territoriale"),
+ complements_convention_collective_renseignee=company.get(
+ "complements", {}
+ ).get("convention_collective_renseignee"),
complements_liste_idcc=company.get("complements", {}).get("liste_idcc"),
- complements_egapro_renseignee=company.get("complements", {}).get("egapro_renseignee"),
- complements_est_achats_responsables=company.get("complements", {}).get("est_achats_responsables"),
- complements_est_alim_confiance=company.get("complements", {}).get("est_alim_confiance"),
- complements_est_association=company.get("complements", {}).get("est_association"),
+ complements_egapro_renseignee=company.get("complements", {}).get(
+ "egapro_renseignee"
+ ),
+ complements_est_achats_responsables=company.get("complements", {}).get(
+ "est_achats_responsables"
+ ),
+ complements_est_alim_confiance=company.get("complements", {}).get(
+ "est_alim_confiance"
+ ),
+ complements_est_association=company.get("complements", {}).get(
+ "est_association"
+ ),
complements_est_bio=company.get("complements", {}).get("est_bio"),
- complements_est_entrepreneur_individuel=company.get("complements", {}).get("est_entrepreneur_individuel"),
- complements_est_entrepreneur_spectacle=company.get("complements", {}).get("est_entrepreneur_spectacle"),
+ complements_est_entrepreneur_individuel=company.get(
+ "complements", {}
+ ).get("est_entrepreneur_individuel"),
+ complements_est_entrepreneur_spectacle=company.get(
+ "complements", {}
+ ).get("est_entrepreneur_spectacle"),
complements_est_ess=company.get("complements", {}).get("est_ess"),
complements_est_finess=company.get("complements", {}).get("est_finess"),
- complements_est_organisme_formation=company.get("complements", {}).get("est_organisme_formation"),
- complements_est_qualiopi=company.get("complements", {}).get("est_qualiopi"),
- complements_liste_id_organisme_formation=company.get("complements", {}).get("liste_id_organisme_formation"),
+ complements_est_organisme_formation=company.get("complements", {}).get(
+ "est_organisme_formation"
+ ),
+ complements_est_qualiopi=company.get("complements", {}).get(
+ "est_qualiopi"
+ ),
+ complements_liste_id_organisme_formation=company.get(
+ "complements", {}
+ ).get("liste_id_organisme_formation"),
complements_est_rge=company.get("complements", {}).get("est_rge"),
- complements_est_service_public=company.get("complements", {}).get("est_service_public"),
+ complements_est_service_public=company.get("complements", {}).get(
+ "est_service_public"
+ ),
complements_est_l100_3=company.get("complements", {}).get("est_l100_3"),
complements_est_siae=company.get("complements", {}).get("est_siae"),
- complements_est_societe_mission=company.get("complements", {}).get("est_societe_mission"),
+ complements_est_societe_mission=company.get("complements", {}).get(
+ "est_societe_mission"
+ ),
complements_est_uai=company.get("complements", {}).get("est_uai"),
- complements_est_patrimoine_vivant=company.get("complements", {}).get("est_patrimoine_vivant"),
- complements_bilan_ges_renseigne=company.get("complements", {}).get("bilan_ges_renseigne"),
- complements_identifiant_association=company.get("complements", {}).get("identifiant_association"),
- complements_statut_entrepreneur_spectacle=company.get("complements", {}).get("statut_entrepreneur_spectacle"),
- complements_type_siae=company.get("complements", {}).get("type_siae")
+ complements_est_patrimoine_vivant=company.get("complements", {}).get(
+ "est_patrimoine_vivant"
+ ),
+ complements_bilan_ges_renseigne=company.get("complements", {}).get(
+ "bilan_ges_renseigne"
+ ),
+ complements_identifiant_association=company.get("complements", {}).get(
+ "identifiant_association"
+ ),
+ complements_statut_entrepreneur_spectacle=company.get(
+ "complements", {}
+ ).get("statut_entrepreneur_spectacle"),
+ complements_type_siae=company.get("complements", {}).get("type_siae"),
)
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"Error enriching organization: {e}"})
+ Logger.error(
+ self.sketch_id, {"message": f"Error enriching organization: {e}"}
+ )
return None
-
+
def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
if not self.neo4j_conn:
return results
@@ -228,22 +286,26 @@ class IndividualToOrgScanner(Scanner):
for org in results:
# Create or update the organization node with all SIRENE properties
org_key = f"{org.name}_FR"
- self.create_node('Organization', 'org_id', org_key,
- name=org.name,
- country="FR",
- siren=org.siren,
- siege_siret=org.siege_siret,
- nom_complet=org.nom_complet,
- nom_raison_sociale=org.nom_raison_sociale,
- sigle=org.sigle,
- caption=org.name, type='organization')
+ self.create_node(
+ "Organization",
+ "org_id",
+ org_key,
+ name=org.name,
+ country="FR",
+ siren=org.siren,
+ siege_siret=org.siege_siret,
+ nom_complet=org.nom_complet,
+ nom_raison_sociale=org.nom_raison_sociale,
+ sigle=org.sigle,
+ caption=org.name,
+ type="organization",
+ )
self.log_graph_message(f"Found organization: {org.name}")
return results
+
# Make types available at module level for easy access
InputType = IndividualToOrgScanner.InputType
OutputType = IndividualToOrgScanner.OutputType
-
-
diff --git a/flowsint-transforms/src/flowsint_transforms/ips/asn_to_cidrs.py b/flowsint-transforms/src/flowsint_transforms/ips/asn_to_cidrs.py
index e14ba853..54bfb9e6 100644
--- a/flowsint-transforms/src/flowsint_transforms/ips/asn_to_cidrs.py
+++ b/flowsint-transforms/src/flowsint_transforms/ips/asn_to_cidrs.py
@@ -7,6 +7,7 @@ from flowsint_types.asn import ASN
from flowsint_core.utils import is_valid_asn, parse_asn
from flowsint_core.core.logger import Logger
+
class AsnToCidrsScanner(Scanner):
"""Takes an ASN and returns its corresponding CIDRs."""
@@ -21,12 +22,14 @@ class AsnToCidrsScanner(Scanner):
@classmethod
def category(cls) -> str:
return "Asn"
-
+
@classmethod
def key(cls) -> str:
- return 'network'
+ return "network"
- def preprocess(self, data: Union[List[str], List[int], List[dict], InputType]) -> InputType:
+ def preprocess(
+ self, data: Union[List[str], List[int], List[dict], InputType]
+ ) -> InputType:
cleaned: InputType = []
for item in data:
asn_obj = None
@@ -60,21 +63,24 @@ class AsnToCidrsScanner(Scanner):
cidrs.append(cidr)
asn_cidrs.append(cidr)
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"Failed to parse CIDR {cidr_str}: {str(e)}"})
+ Logger.error(
+ self.sketch_id,
+ {"message": f"Failed to parse CIDR {cidr_str}: {str(e)}"},
+ )
else:
- Logger.warn(self.sketch_id, {"message": f"No CIDRs found for ASN {asn.number}"})
-
+ Logger.warn(
+ self.sketch_id, {"message": f"No CIDRs found for ASN {asn.number}"}
+ )
+
if asn_cidrs: # Only add to mapping if we found valid CIDRs
self._asn_to_cidrs_map.append((asn, asn_cidrs))
return cidrs
-
+
def __get_cidrs_from_asn(self, asn: int) -> Dict[str, Any]:
try:
command = f"echo {asn} | asnmap -silent -json | jq -s '.'"
result = subprocess.run(
- command,
- shell=True,
- capture_output=True, text=True, timeout=60
+ command, shell=True, capture_output=True, text=True, timeout=60
)
if not result.stdout.strip():
Logger.info(self.sketch_id, {"message": f"No CIDRs found for {asn}."})
@@ -89,7 +95,7 @@ class AsnToCidrsScanner(Scanner):
"as_range": [],
"as_name": None,
"as_country": None,
- "as_number": None
+ "as_number": None,
}
for data in data_array:
@@ -105,35 +111,58 @@ class AsnToCidrsScanner(Scanner):
return combined_data if combined_data["as_range"] else None
except json.JSONDecodeError:
- Logger.error(self.sketch_id, {"message": f"Failed to parse JSON from asnmap output: {result.stdout}"})
+ Logger.error(
+ self.sketch_id,
+ {
+ "message": f"Failed to parse JSON from asnmap output: {result.stdout}"
+ },
+ )
return None
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"asnmap exception for {asn}: {str(e)}"})
+ Logger.error(
+ self.sketch_id, {"message": f"asnmap exception for {asn}: {str(e)}"}
+ )
return None
def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
# Create Neo4j relationships between ASNs and their corresponding CIDRs
# Use the mapping from scan if available, else fallback to zip
- asn_to_cidrs = getattr(self, '_asn_to_cidrs_map', None)
+ asn_to_cidrs = getattr(self, "_asn_to_cidrs_map", None)
if asn_to_cidrs is not None:
for asn, cidr_list in asn_to_cidrs:
for cidr in cidr_list:
if str(cidr.network) == "0.0.0.0/0":
continue # Skip default CIDR for unknown ASN
if self.neo4j_conn:
- self.create_node('asn', 'number', asn.number,
- name=asn.name or "Unknown",
- country=asn.country or "Unknown",
- label=f"AS{asn.number}",
- caption=f"AS{asn.number} - {asn.name or 'Unknown'}",
- type='asn')
-
- self.create_node('cidr', 'network', str(cidr.network),
- caption=str(cidr.network), type='cidr')
-
- self.create_relationship('asn', 'number', asn.number,
- 'cidr', 'network', str(cidr.network), 'ANNOUNCES')
+ self.create_node(
+ "asn",
+ "number",
+ asn.number,
+ name=asn.name or "Unknown",
+ country=asn.country or "Unknown",
+ label=f"AS{asn.number}",
+ caption=f"AS{asn.number} - {asn.name or 'Unknown'}",
+ type="asn",
+ )
+
+ self.create_node(
+ "cidr",
+ "network",
+ str(cidr.network),
+ caption=str(cidr.network),
+ type="cidr",
+ )
+
+ self.create_relationship(
+ "asn",
+ "number",
+ asn.number,
+ "cidr",
+ "network",
+ str(cidr.network),
+ "ANNOUNCES",
+ )
else:
# Fallback: original behavior (one-to-one zip)
for asn, cidr in zip(original_input, results):
@@ -141,20 +170,37 @@ class AsnToCidrsScanner(Scanner):
continue # Skip default CIDR for unknown ASN
self.log_graph_message(f"ASN {asn.number} -> {cidr.network}")
if self.neo4j_conn:
- self.create_node('ASN', 'number', asn.number,
- name=asn.name or "Unknown",
- country=asn.country or "Unknown",
- label=f"AS{asn.number}",
- caption=f"AS{asn.number} - {asn.name or 'Unknown'}",
- type='asn')
-
- self.create_node('CIDR', 'network', str(cidr.network),
- caption=str(cidr.network), type='cidr')
-
- self.create_relationship('ASN', 'number', asn.number,
- 'CIDR', 'network', str(cidr.network), 'ANNOUNCES')
+ self.create_node(
+ "ASN",
+ "number",
+ asn.number,
+ name=asn.name or "Unknown",
+ country=asn.country or "Unknown",
+ label=f"AS{asn.number}",
+ caption=f"AS{asn.number} - {asn.name or 'Unknown'}",
+ type="asn",
+ )
+
+ self.create_node(
+ "CIDR",
+ "network",
+ str(cidr.network),
+ caption=str(cidr.network),
+ type="cidr",
+ )
+
+ self.create_relationship(
+ "ASN",
+ "number",
+ asn.number,
+ "CIDR",
+ "network",
+ str(cidr.network),
+ "ANNOUNCES",
+ )
return results
+
# Make types available at module level for easy access
InputType = AsnToCidrsScanner.InputType
-OutputType = AsnToCidrsScanner.OutputType
\ No newline at end of file
+OutputType = AsnToCidrsScanner.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/ips/cidr_to_ips.py b/flowsint-transforms/src/flowsint_transforms/ips/cidr_to_ips.py
index 650d9cd1..130f7a6a 100644
--- a/flowsint-transforms/src/flowsint_transforms/ips/cidr_to_ips.py
+++ b/flowsint-transforms/src/flowsint_transforms/ips/cidr_to_ips.py
@@ -5,6 +5,7 @@ from flowsint_types.cidr import CIDR
from flowsint_types.ip import Ip
from flowsint_core.core.logger import Logger
+
class CidrToIpsScanner(Scanner):
"""Takes a CIDR and returns its corresponding IP addresses."""
@@ -49,31 +50,36 @@ class CidrToIpsScanner(Scanner):
ip = Ip(address=ip_str.strip())
ips.append(ip)
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"Failed to parse IP {ip_str}: {str(e)}"})
+ Logger.error(
+ self.sketch_id,
+ {"message": f"Failed to parse IP {ip_str}: {str(e)}"},
+ )
else:
- Logger.warn(self.sketch_id, {"message": f"No IPs found for CIDR {cidr.network}"})
+ Logger.warn(
+ self.sketch_id, {"message": f"No IPs found for CIDR {cidr.network}"}
+ )
return ips
-
+
def __get_ips_from_cidr(self, cidr: str) -> List[str]:
try:
command = f"echo {cidr} | dnsx -ptr"
result = subprocess.run(
- command,
- shell=True,
- capture_output=True,
- text=True,
- timeout=60
+ command, shell=True, capture_output=True, text=True, timeout=60
)
if not result.stdout.strip():
Logger.info(self.sketch_id, {"message": f"No IPs found for {cidr}."})
return []
-
+
# Split the output by newlines and filter out empty lines
- ip_addresses = [ip.strip() for ip in result.stdout.split('\n') if ip.strip()]
+ ip_addresses = [
+ ip.strip() for ip in result.stdout.split("\n") if ip.strip()
+ ]
return ip_addresses
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"dnsx exception for {cidr}: {str(e)}"})
+ Logger.error(
+ self.sketch_id, {"message": f"dnsx exception for {cidr}: {str(e)}"}
+ )
return []
def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
@@ -81,20 +87,34 @@ class CidrToIpsScanner(Scanner):
for cidr, ip in zip(original_input, results):
if self.neo4j_conn:
# Create CIDR node
- self.create_node('cidr', 'network', str(cidr.network),
- caption=str(cidr.network), type="cidr")
-
+ self.create_node(
+ "cidr",
+ "network",
+ str(cidr.network),
+ caption=str(cidr.network),
+ type="cidr",
+ )
+
# Create IP node
- self.create_node('ip', 'address', ip.address,
- caption=ip.address, type="ip")
-
+ self.create_node(
+ "ip", "address", ip.address, caption=ip.address, type="ip"
+ )
+
# Create relationship
- self.create_relationship('cidr', 'network', str(cidr.network),
- 'ip', 'address', ip.address, 'CONTAINS')
-
+ self.create_relationship(
+ "cidr",
+ "network",
+ str(cidr.network),
+ "ip",
+ "address",
+ ip.address,
+ "CONTAINS",
+ )
+
self.log_graph_message(f"Found {len(results)} IPs for CIDR {cidr.network}")
return results
+
# Make types available at module level for easy access
InputType = CidrToIpsScanner.InputType
-OutputType = CidrToIpsScanner.OutputType
\ No newline at end of file
+OutputType = CidrToIpsScanner.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/ips/geolocation.py b/flowsint-transforms/src/flowsint_transforms/ips/geolocation.py
index 368adf89..332b2060 100644
--- a/flowsint-transforms/src/flowsint_transforms/ips/geolocation.py
+++ b/flowsint-transforms/src/flowsint_transforms/ips/geolocation.py
@@ -4,13 +4,14 @@ from pydantic import TypeAdapter
from flowsint_core.core.scanner_base import Scanner
from flowsint_types.ip import Ip, Ip
from flowsint_core.utils import resolve_type, is_valid_ip
+
InputType: TypeAlias = List[Ip]
OutputType: TypeAlias = List[Ip]
+
class GeolocationScanner(Scanner):
"""Get geolocation data for IP addresses."""
-
# Define types as class attributes - base class handles schema generation automatically
InputType = List[Ip]
OutputType = List[Ip]
@@ -22,7 +23,7 @@ class GeolocationScanner(Scanner):
@classmethod
def category(cls) -> str:
return "Ip"
-
+
@classmethod
def key(cls) -> str:
return "address"
@@ -37,7 +38,7 @@ class GeolocationScanner(Scanner):
"properties": [
{"name": prop, "type": resolve_type(info, schema)}
for prop, info in details["properties"].items()
- ]
+ ],
}
@classmethod
@@ -50,7 +51,7 @@ class GeolocationScanner(Scanner):
"properties": [
{"name": prop, "type": resolve_type(info, schema)}
for prop, info in details["properties"].items()
- ]
+ ],
}
def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:
@@ -89,10 +90,20 @@ class GeolocationScanner(Scanner):
"""Update IP nodes in Neo4j with geolocation information."""
if self.neo4j_conn:
for ip in results:
- self.create_node('ip', 'address', ip.address,
- latitude=ip.latitude, longitude=ip.longitude,
- country=ip.country, city=ip.city, isp=ip.isp, type='ip')
- self.log_graph_message(f"Geolocated {ip.address} to {ip.city}, {ip.country} (lat: {ip.latitude}, lon: {ip.longitude})")
+ self.create_node(
+ "ip",
+ "address",
+ ip.address,
+ latitude=ip.latitude,
+ longitude=ip.longitude,
+ country=ip.country,
+ city=ip.city,
+ isp=ip.isp,
+ type="ip",
+ )
+ self.log_graph_message(
+ f"Geolocated {ip.address} to {ip.city}, {ip.country} (lat: {ip.latitude}, lon: {ip.longitude})"
+ )
return results
def get_location_data(self, address: str) -> Dict[str, Any]:
@@ -112,10 +123,13 @@ class GeolocationScanner(Scanner):
"isp": data.get("isp"),
}
else:
- raise ValueError(f"Geolocation failed for {address}: {data.get('message')}")
+ raise ValueError(
+ f"Geolocation failed for {address}: {data.get('message')}"
+ )
except Exception as e:
print(f"Failed to geolocate {address}: {e}")
return {}
+
InputType = GeolocationScanner.InputType
-OutputType = GeolocationScanner.OutputType
\ No newline at end of file
+OutputType = GeolocationScanner.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/ips/ip_to_asn.py b/flowsint-transforms/src/flowsint_transforms/ips/ip_to_asn.py
index 97ffd52c..85c59210 100644
--- a/flowsint-transforms/src/flowsint_transforms/ips/ip_to_asn.py
+++ b/flowsint-transforms/src/flowsint_transforms/ips/ip_to_asn.py
@@ -8,6 +8,7 @@ from flowsint_types.asn import ASN
from flowsint_core.utils import is_valid_ip
from flowsint_core.core.logger import Logger
+
class IpToAsnScanner(Scanner):
"""Takes an IP address and returns its corresponding ASN."""
@@ -22,7 +23,7 @@ class IpToAsnScanner(Scanner):
@classmethod
def category(cls) -> str:
return "Ip"
-
+
@classmethod
def key(cls) -> str:
return "address"
@@ -45,7 +46,7 @@ class IpToAsnScanner(Scanner):
async def scan(self, data: InputType) -> OutputType:
results: OutputType = []
-
+
for ip in data:
try:
# Use asnmap to get ASN info
@@ -53,48 +54,59 @@ class IpToAsnScanner(Scanner):
["asnmap", "-a", ip.address, "-json"],
capture_output=True,
text=True,
- timeout=30
+ timeout=30,
)
-
+
if result.returncode == 0:
output = result.stdout.strip()
if output:
asn_data = json.loads(output)
- if asn_data and 'as_number' in asn_data:
+ if asn_data and "as_number" in asn_data:
asn = ASN(
- asn=str(asn_data['as_number']),
- name=asn_data.get('as_name', ''),
- org=asn_data.get('as_org', ''),
- country=asn_data.get('as_country', '')
+ asn=str(asn_data["as_number"]),
+ name=asn_data.get("as_name", ""),
+ org=asn_data.get("as_org", ""),
+ country=asn_data.get("as_country", ""),
)
results.append(asn)
-
+
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"Error getting ASN for IP {ip.address}: {e}"})
+ Logger.error(
+ self.sketch_id,
+ {"message": f"Error getting ASN for IP {ip.address}: {e}"},
+ )
continue
-
+
return results
- def postprocess(self, results: OutputType, input_data: InputType = None) -> OutputType:
+ def postprocess(
+ self, results: OutputType, input_data: InputType = None
+ ) -> OutputType:
# Create Neo4j relationships between IPs and their corresponding ASNs
if input_data and self.neo4j_conn:
for ip, asn in zip(input_data, results):
# Create IP node
- self.create_node('ip', 'address', ip.address,
- caption=ip.address, type="ip")
-
+ self.create_node(
+ "ip", "address", ip.address, caption=ip.address, type="ip"
+ )
+
# Create ASN node
- self.create_node('asn', 'network', asn.asn,
- caption=f"AS{asn.asn}", type="asn")
-
+ self.create_node(
+ "asn", "network", asn.asn, caption=f"AS{asn.asn}", type="asn"
+ )
+
# Create relationship
- self.create_relationship('ip', 'address', ip.address,
- 'asn', 'network', asn.asn, 'BELONGS_TO')
-
- self.log_graph_message(f"IP {ip.address} belongs to AS{asn.asn} ({asn.name})")
-
+ self.create_relationship(
+ "ip", "address", ip.address, "asn", "network", asn.asn, "BELONGS_TO"
+ )
+
+ self.log_graph_message(
+ f"IP {ip.address} belongs to AS{asn.asn} ({asn.name})"
+ )
+
return results
+
# Make types available at module level for easy access
InputType = IpToAsnScanner.InputType
-OutputType = IpToAsnScanner.OutputType
\ No newline at end of file
+OutputType = IpToAsnScanner.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/ips/reverse_resolve.py b/flowsint-transforms/src/flowsint_transforms/ips/reverse_resolve.py
index 239ad228..daa88388 100644
--- a/flowsint-transforms/src/flowsint_transforms/ips/reverse_resolve.py
+++ b/flowsint-transforms/src/flowsint_transforms/ips/reverse_resolve.py
@@ -10,6 +10,7 @@ from flowsint_core.utils import is_valid_ip
PTR_BLACKLIST = re.compile(r"^ip\d+\.ip-\d+-\d+-\d+-\d+\.")
+
class ReverseResolveScanner(Scanner):
"""Resolve IP addresses to domain names using PTR, Certificate Transparency and optional API calls."""
@@ -47,7 +48,7 @@ class ReverseResolveScanner(Scanner):
async def scan(self, data: InputType) -> OutputType:
results: OutputType = []
-
+
for ip in data:
try:
# Try PTR lookup
@@ -59,7 +60,7 @@ class ReverseResolveScanner(Scanner):
continue
except socket.herror:
pass
-
+
# Try Certificate Transparency logs
try:
ct_url = f"https://crt.sh/?q={ip.address}&output=json"
@@ -74,16 +75,22 @@ class ReverseResolveScanner(Scanner):
break
except Exception:
pass
-
+
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"Error reverse resolving IP {ip.address}: {e}"})
+ Logger.error(
+ self.sketch_id,
+ {"message": f"Error reverse resolving IP {ip.address}: {e}"},
+ )
continue
-
+
return results
- def postprocess(self, results: OutputType, input_data: InputType = None) -> OutputType:
+ def postprocess(
+ self, results: OutputType, input_data: InputType = None
+ ) -> OutputType:
return results
+
# Make types available at module level for easy access
InputType = ReverseResolveScanner.InputType
-OutputType = ReverseResolveScanner.OutputType
\ No newline at end of file
+OutputType = ReverseResolveScanner.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/leaks/hibp.py b/flowsint-transforms/src/flowsint_transforms/leaks/hibp.py
index b5a89a89..2ac37d72 100644
--- a/flowsint-transforms/src/flowsint_transforms/leaks/hibp.py
+++ b/flowsint-transforms/src/flowsint_transforms/leaks/hibp.py
@@ -10,6 +10,7 @@ load_dotenv()
HIBP_API_KEY = os.getenv("HIBP_API_KEY")
+
class HibpScanner(Scanner):
"""Queries HaveIBeenPwned for potential leaks."""
@@ -20,11 +21,11 @@ class HibpScanner(Scanner):
@classmethod
def name(cls) -> str:
return "hibp_scanner"
-
+
@classmethod
def category(cls) -> str:
return "leaks"
-
+
@classmethod
def key(cls) -> str:
return "email"
@@ -62,12 +63,17 @@ class HibpScanner(Scanner):
}
results.append(email_result)
except Exception as e:
- results.append({
- "email": email,
- "error": f"Error during scan: {str(e)}",
- })
- Logger.error(self.sketch_id, {"message": f"Error scanning email {email}: {str(e)}"})
-
+ results.append(
+ {
+ "email": email,
+ "error": f"Error during scan: {str(e)}",
+ }
+ )
+ Logger.error(
+ self.sketch_id,
+ {"message": f"Error scanning email {email}: {str(e)}"},
+ )
+
return results
def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
@@ -78,23 +84,37 @@ class HibpScanner(Scanner):
for result in results:
if "error" not in result:
email = result["email"]
-
+
# Create email node
- self.create_node('email', 'address', email,
- caption=email, type='email')
-
+ self.create_node("email", "address", email, caption=email, type="email")
+
# Create breach relationships
for breach in result.get("breaches", []):
if breach and isinstance(breach, dict):
breach_name = breach.get("Name", "Unknown")
- self.create_node('breach', 'name', breach_name,
- caption=breach_name, type='breach')
- self.create_relationship('email', 'address', email,
- 'breach', 'name', breach_name, 'FOUND_IN_BREACH')
- self.log_graph_message(f"Email {email} found in breach: {breach_name}")
+ self.create_node(
+ "breach",
+ "name",
+ breach_name,
+ caption=breach_name,
+ type="breach",
+ )
+ self.create_relationship(
+ "email",
+ "address",
+ email,
+ "breach",
+ "name",
+ breach_name,
+ "FOUND_IN_BREACH",
+ )
+ self.log_graph_message(
+ f"Email {email} found in breach: {breach_name}"
+ )
return results
+
# Make types available at module level for easy access
InputType = HibpScanner.InputType
OutputType = HibpScanner.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/n8n/connector.py b/flowsint-transforms/src/flowsint_transforms/n8n/connector.py
index 2840caa4..36089513 100644
--- a/flowsint-transforms/src/flowsint_transforms/n8n/connector.py
+++ b/flowsint-transforms/src/flowsint_transforms/n8n/connector.py
@@ -5,31 +5,32 @@ from flowsint_core.core.scanner_base import Scanner
from flowsint_core.core.logger import Logger
from flowsint_core.core.graph_db import Neo4jConnection
+
class N8nConnector(Scanner):
"""
Connect to your custom n8n workflows to process data through webhooks.
"""
-
+
# Define types as class attributes - base class handles schema generation automatically
InputType = List[Any]
OutputType = List[Any]
-
+
def __init__(
- self,
- sketch_id: Optional[str] = None,
- scan_id: Optional[str] = None,
- neo4j_conn: Optional[Neo4jConnection] = None,
- vault=None,
- params: Optional[Dict[str, Any]] = None
- ):
- super().__init__(
- sketch_id=sketch_id,
- scan_id=scan_id,
- neo4j_conn=neo4j_conn,
- params_schema=self.get_params_schema(),
- vault=vault,
- params=params
- )
+ self,
+ sketch_id: Optional[str] = None,
+ scan_id: Optional[str] = None,
+ neo4j_conn: Optional[Neo4jConnection] = None,
+ vault=None,
+ params: Optional[Dict[str, Any]] = None,
+ ):
+ super().__init__(
+ sketch_id=sketch_id,
+ scan_id=scan_id,
+ neo4j_conn=neo4j_conn,
+ params_schema=self.get_params_schema(),
+ vault=vault,
+ params=params,
+ )
@classmethod
def documentation(cls) -> str:
@@ -223,7 +224,7 @@ Check FlowSint logs for detailed debugging information.
@classmethod
def category(cls) -> str:
return "external"
-
+
@classmethod
def required_params(cls) -> bool:
return True
@@ -245,14 +246,14 @@ Check FlowSint logs for detailed debugging information.
"name": "auth_token",
"type": "vaultSecret",
"required": False,
- "description": "Optional authentication token for the webhook"
+ "description": "Optional authentication token for the webhook",
},
{
"name": "extra_payload",
"type": "string",
"required": False,
- "description": "Optional JSON string with additional data to include in the payload"
- }
+ "description": "Optional JSON string with additional data to include in the payload",
+ },
]
async def scan(self, values: InputType) -> OutputType:
@@ -266,7 +267,7 @@ Check FlowSint logs for detailed debugging information.
payload = {
"sketch_id": self.sketch_id,
"type": values[0] if values else None,
- "inputs": values
+ "inputs": values,
}
if "extra_payload" in params and params["extra_payload"] is not None:
@@ -274,41 +275,83 @@ Check FlowSint logs for detailed debugging information.
extra = json.loads(params["extra_payload"])
payload.update(extra)
except json.JSONDecodeError:
- Logger.warn(self.sketch_id, {"message": "extra_payload is not valid JSON"})
+ Logger.warn(
+ self.sketch_id, {"message": "extra_payload is not valid JSON"}
+ )
- Logger.info(self.sketch_id, {"message": f"Sending request to n8n webhook with payload: {json.dumps(payload)}"})
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"Sending request to n8n webhook with payload: {json.dumps(payload)}"
+ },
+ )
try:
async with aiohttp.ClientSession() as session:
async with session.post(url, headers=headers, json=payload) as response:
- Logger.info(self.sketch_id, {"message": f"n8n webhook responded with status: {response.status}"})
-
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"n8n webhook responded with status: {response.status}"
+ },
+ )
+
# Log the raw response text for debugging
response_text = await response.text()
- Logger.info(self.sketch_id, {"message": f"n8n webhook raw response: {response_text}"})
-
+ Logger.info(
+ self.sketch_id,
+ {"message": f"n8n webhook raw response: {response_text}"},
+ )
+
if response.status != 200:
- Logger.warn(self.sketch_id, {"message": f"n8n responded with non-200 status: {response.status} - Response: {response_text}"})
- raise Exception(f"n8n responded with {response.status}: {response_text}")
-
+ Logger.warn(
+ self.sketch_id,
+ {
+ "message": f"n8n responded with non-200 status: {response.status} - Response: {response_text}"
+ },
+ )
+ raise Exception(
+ f"n8n responded with {response.status}: {response_text}"
+ )
+
try:
data = json.loads(response_text)
- Logger.info(self.sketch_id, {"message": f"n8n connector received response: {json.dumps(data)}"})
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"n8n connector received response: {json.dumps(data)}"
+ },
+ )
return data
except json.JSONDecodeError as e:
- Logger.warn(self.sketch_id, {"message": f"Failed to parse n8n response as JSON: {str(e)} - Raw response: {response_text}"})
+ Logger.warn(
+ self.sketch_id,
+ {
+ "message": f"Failed to parse n8n response as JSON: {str(e)} - Raw response: {response_text}"
+ },
+ )
# Return the raw text wrapped in a list of dicts as expected
- return [{"raw_response": response_text, "error": "Response was not valid JSON"}]
-
+ return [
+ {
+ "raw_response": response_text,
+ "error": "Response was not valid JSON",
+ }
+ ]
+
except Exception as e:
- Logger.warn(self.sketch_id, {"message": f"Error calling n8n webhook: {str(e)}"})
+ Logger.warn(
+ self.sketch_id, {"message": f"Error calling n8n webhook: {str(e)}"}
+ )
# Re-raise the exception so the caller knows something went wrong
raise
-
+
def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
- Logger.success(self.sketch_id, {"message": f"n8n connector results: {json.dumps(results)}"})
+ Logger.success(
+ self.sketch_id, {"message": f"n8n connector results: {json.dumps(results)}"}
+ )
return results
+
# Make types available at module level for easy access
InputType = N8nConnector.InputType
OutputType = N8nConnector.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/organizations/__init__.py b/flowsint-transforms/src/flowsint_transforms/organizations/__init__.py
index e69de29b..8b137891 100644
--- a/flowsint-transforms/src/flowsint_transforms/organizations/__init__.py
+++ b/flowsint-transforms/src/flowsint_transforms/organizations/__init__.py
@@ -0,0 +1 @@
+
diff --git a/flowsint-transforms/src/flowsint_transforms/organizations/org_to_asn.py b/flowsint-transforms/src/flowsint_transforms/organizations/org_to_asn.py
index e0f31b25..ff93f57b 100644
--- a/flowsint-transforms/src/flowsint_transforms/organizations/org_to_asn.py
+++ b/flowsint-transforms/src/flowsint_transforms/organizations/org_to_asn.py
@@ -6,6 +6,7 @@ from flowsint_types.organization import Organization
from flowsint_types.asn import ASN
from flowsint_core.core.logger import Logger
+
class OrgToAsnScanner(Scanner):
"""Takes an organization and returns its corresponding ASN."""
@@ -20,7 +21,7 @@ class OrgToAsnScanner(Scanner):
@classmethod
def category(cls) -> str:
return "Organization"
-
+
@classmethod
def key(cls) -> str:
return "name"
@@ -46,24 +47,26 @@ class OrgToAsnScanner(Scanner):
for org in data:
asn_data = self.__get_asn_from_asnmap(org.name)
if asn_data:
- asns.append(ASN(
- number=int(asn_data["as_number"].lstrip("AS")),
- name=asn_data["as_name"],
- country=asn_data["as_country"],
- cidrs=[]
- ))
+ asns.append(
+ ASN(
+ number=int(asn_data["as_number"].lstrip("AS")),
+ name=asn_data["as_name"],
+ country=asn_data["as_country"],
+ cidrs=[],
+ )
+ )
else:
- Logger.info(self.sketch_id, {"message": f"No ASN found for org {org.name}."})
+ Logger.info(
+ self.sketch_id, {"message": f"No ASN found for org {org.name}."}
+ )
return asns
-
+
def __get_asn_from_asnmap(self, name: str) -> Dict[str, Any]:
try:
# Properly run the shell pipeline using shell=True
command = f"echo {name} | asnmap -silent -json | jq -s"
result = subprocess.run(
- command,
- shell=True,
- capture_output=True, text=True, timeout=60
+ command, shell=True, capture_output=True, text=True, timeout=60
)
if not result.stdout.strip():
return None
@@ -77,7 +80,7 @@ class OrgToAsnScanner(Scanner):
"as_range": [],
"as_name": None,
"as_country": None,
- "as_number": None
+ "as_number": None,
}
for data in data_array:
@@ -93,11 +96,19 @@ class OrgToAsnScanner(Scanner):
return combined_data if combined_data["as_number"] else None
except json.JSONDecodeError as e:
- Logger.error(self.sketch_id, {"message": f"An error occurred while parsing the JSON output from asnmap: {str(e)}"})
+ Logger.error(
+ self.sketch_id,
+ {
+ "message": f"An error occurred while parsing the JSON output from asnmap: {str(e)}"
+ },
+ )
return None
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"An error occurred while running asnmap: {str(e)}"})
+ Logger.error(
+ self.sketch_id,
+ {"message": f"An error occurred while running asnmap: {str(e)}"},
+ )
return None
def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
@@ -106,28 +117,47 @@ class OrgToAsnScanner(Scanner):
# Skip if no valid ASN was found
if result_asn.number == 0:
continue
-
+
if self.neo4j_conn:
# Create organization node
- self.create_node('organization', 'name', input_org.name,
- caption=input_org.name, type="organization")
-
+ self.create_node(
+ "organization",
+ "name",
+ input_org.name,
+ caption=input_org.name,
+ type="organization",
+ )
+
# Create ASN node
- self.create_node('asn', 'number', result_asn.number,
- name=result_asn.name,
- country=result_asn.country,
- label=f"AS{result_asn.number}",
- caption=f"AS{result_asn.number} - {result_asn.name}",
- type="asn")
-
+ self.create_node(
+ "asn",
+ "number",
+ result_asn.number,
+ name=result_asn.name,
+ country=result_asn.country,
+ label=f"AS{result_asn.number}",
+ caption=f"AS{result_asn.number} - {result_asn.name}",
+ type="asn",
+ )
+
# Create relationship
- self.create_relationship('organization', 'name', input_org.name,
- 'asn', 'number', result_asn.number, 'BELONGS_TO')
-
- self.log_graph_message(f"Found for {input_org.name} -> ASN {result_asn.number}")
+ self.create_relationship(
+ "organization",
+ "name",
+ input_org.name,
+ "asn",
+ "number",
+ result_asn.number,
+ "BELONGS_TO",
+ )
+
+ self.log_graph_message(
+ f"Found for {input_org.name} -> ASN {result_asn.number}"
+ )
return results
+
# Make types available at module level for easy access
InputType = OrgToAsnScanner.InputType
-OutputType = OrgToAsnScanner.OutputType
\ No newline at end of file
+OutputType = OrgToAsnScanner.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/organizations/to_infos.py b/flowsint-transforms/src/flowsint_transforms/organizations/to_infos.py
index 9af1afd2..a9be4cf6 100644
--- a/flowsint-transforms/src/flowsint_transforms/organizations/to_infos.py
+++ b/flowsint-transforms/src/flowsint_transforms/organizations/to_infos.py
@@ -19,7 +19,7 @@ class OrgToInfosScanner(Scanner):
@classmethod
def category(cls) -> str:
return "Organization"
-
+
@classmethod
def key(cls) -> str:
return "name"
@@ -38,19 +38,22 @@ class OrgToInfosScanner(Scanner):
return cleaned
async def scan(self, data: InputType) -> OutputType:
-
+
results: OutputType = []
for org in data:
try:
sirene = SireneTool()
raw_orgs = sirene.launch(org.name, limit=25)
- if len(raw_orgs)> 0:
+ if len(raw_orgs) > 0:
for org_dict in raw_orgs:
enriched_org = self.enrich_org(org_dict)
if enriched_org is not None:
results.append(enriched_org)
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"Error enriching organization {org.name}: {e}"})
+ Logger.error(
+ self.sketch_id,
+ {"message": f"Error enriching organization {org.name}: {e}"},
+ )
return results
def enrich_org(self, company: Dict) -> Organization:
@@ -61,35 +64,40 @@ class OrgToInfosScanner(Scanner):
siege_geo_adresse = None
if siege.get("latitude") and siege.get("longitude"):
from flowsint_types.address import PhysicalAddress
+
siege_geo_adresse = PhysicalAddress(
address=siege.get("adresse", ""),
city=siege.get("libelle_commune", ""),
country="FR", # SIRENE is French registry
zip=siege.get("code_postal", ""),
latitude=float(siege.get("latitude", 0)),
- longitude=float(siege.get("longitude", 0))
+ longitude=float(siege.get("longitude", 0)),
)
-
+
# Extract dirigeants and convert to Individual objects
dirigeants = []
for dirigeant_data in company.get("dirigeants", []):
from flowsint_types.individual import Individual
+
dirigeant = Individual(
first_name=dirigeant_data.get("prenoms", ""),
last_name=dirigeant_data.get("nom", ""),
full_name=f"{dirigeant_data.get('prenoms', '')} {dirigeant_data.get('nom', '')}".strip(),
birth_date=dirigeant_data.get("date_de_naissance"),
- gender=None # Not available in SIRENE data
+ gender=None, # Not available in SIRENE data
)
dirigeants.append(dirigeant)
-
+
name = company.get("nom_raison_sociale") or company.get("nom_complet")
-
+
# Ensure we have a valid name
if not name:
- Logger.error(self.sketch_id, {"message": f"Organization has no valid name: {company}"})
+ Logger.error(
+ self.sketch_id,
+ {"message": f"Organization has no valid name: {company}"},
+ )
return None
-
+
return Organization(
# Basic information
name=name,
@@ -97,40 +105,41 @@ class OrgToInfosScanner(Scanner):
nom_complet=company.get("nom_complet"),
nom_raison_sociale=company.get("nom_raison_sociale"),
sigle=company.get("sigle"),
-
# Establishment information
nombre_etablissements=company.get("nombre_etablissements"),
- nombre_etablissements_ouverts=company.get("nombre_etablissements_ouverts"),
-
+ nombre_etablissements_ouverts=company.get(
+ "nombre_etablissements_ouverts"
+ ),
# Activity information
activite_principale=company.get("activite_principale"),
section_activite_principale=company.get("section_activite_principale"),
-
# Company category
categorie_entreprise=company.get("categorie_entreprise"),
annee_categorie_entreprise=company.get("annee_categorie_entreprise"),
-
# Employment information
caractere_employeur=company.get("caractere_employeur"),
tranche_effectif_salarie=company.get("tranche_effectif_salarie"),
- annee_tranche_effectif_salarie=company.get("annee_tranche_effectif_salarie"),
-
+ annee_tranche_effectif_salarie=company.get(
+ "annee_tranche_effectif_salarie"
+ ),
# Dates
date_creation=company.get("date_creation"),
date_fermeture=company.get("date_fermeture"),
date_mise_a_jour=company.get("date_mise_a_jour"),
date_mise_a_jour_insee=company.get("date_mise_a_jour_insee"),
date_mise_a_jour_rne=company.get("date_mise_a_jour_rne"),
-
# Legal information
nature_juridique=company.get("nature_juridique"),
etat_administratif=company.get("etat_administratif"),
statut_diffusion=company.get("statut_diffusion"),
-
# Siege (Headquarters) information
siege_activite_principale=siege.get("activite_principale"),
- siege_activite_principale_registre_metier=siege.get("activite_principale_registre_metier"),
- siege_annee_tranche_effectif_salarie=siege.get("annee_tranche_effectif_salarie"),
+ siege_activite_principale_registre_metier=siege.get(
+ "activite_principale_registre_metier"
+ ),
+ siege_annee_tranche_effectif_salarie=siege.get(
+ "annee_tranche_effectif_salarie"
+ ),
siege_adresse=siege.get("adresse"),
siege_caractere_employeur=siege.get("caractere_employeur"),
siege_cedex=siege.get("cedex"),
@@ -162,7 +171,9 @@ class OrgToInfosScanner(Scanner):
siege_liste_finess=siege.get("liste_finess"),
siege_liste_id_bio=siege.get("liste_id_bio"),
siege_liste_idcc=siege.get("liste_idcc"),
- siege_liste_id_organisme_formation=siege.get("liste_id_organisme_formation"),
+ siege_liste_id_organisme_formation=siege.get(
+ "liste_id_organisme_formation"
+ ),
siege_liste_rge=siege.get("liste_rge"),
siege_liste_uai=siege.get("liste_uai"),
siege_longitude=siege.get("longitude"),
@@ -170,51 +181,85 @@ class OrgToInfosScanner(Scanner):
siege_numero_voie=siege.get("numero_voie"),
siege_region=siege.get("region"),
siege_siret=siege.get("siret"),
- siege_statut_diffusion_etablissement=siege.get("statut_diffusion_etablissement"),
+ siege_statut_diffusion_etablissement=siege.get(
+ "statut_diffusion_etablissement"
+ ),
siege_tranche_effectif_salarie=siege.get("tranche_effectif_salarie"),
siege_type_voie=siege.get("type_voie"),
-
# Dirigeants (Leaders)
dirigeants=dirigeants if dirigeants else None,
-
# Matching establishments
matching_etablissements=company.get("matching_etablissements"),
-
# Financial information
finances=company.get("finances"),
-
# Complements (Additional information)
- complements_collectivite_territoriale=company.get("complements", {}).get("collectivite_territoriale"),
- complements_convention_collective_renseignee=company.get("complements", {}).get("convention_collective_renseignee"),
+ complements_collectivite_territoriale=company.get(
+ "complements", {}
+ ).get("collectivite_territoriale"),
+ complements_convention_collective_renseignee=company.get(
+ "complements", {}
+ ).get("convention_collective_renseignee"),
complements_liste_idcc=company.get("complements", {}).get("liste_idcc"),
- complements_egapro_renseignee=company.get("complements", {}).get("egapro_renseignee"),
- complements_est_achats_responsables=company.get("complements", {}).get("est_achats_responsables"),
- complements_est_alim_confiance=company.get("complements", {}).get("est_alim_confiance"),
- complements_est_association=company.get("complements", {}).get("est_association"),
+ complements_egapro_renseignee=company.get("complements", {}).get(
+ "egapro_renseignee"
+ ),
+ complements_est_achats_responsables=company.get("complements", {}).get(
+ "est_achats_responsables"
+ ),
+ complements_est_alim_confiance=company.get("complements", {}).get(
+ "est_alim_confiance"
+ ),
+ complements_est_association=company.get("complements", {}).get(
+ "est_association"
+ ),
complements_est_bio=company.get("complements", {}).get("est_bio"),
- complements_est_entrepreneur_individuel=company.get("complements", {}).get("est_entrepreneur_individuel"),
- complements_est_entrepreneur_spectacle=company.get("complements", {}).get("est_entrepreneur_spectacle"),
+ complements_est_entrepreneur_individuel=company.get(
+ "complements", {}
+ ).get("est_entrepreneur_individuel"),
+ complements_est_entrepreneur_spectacle=company.get(
+ "complements", {}
+ ).get("est_entrepreneur_spectacle"),
complements_est_ess=company.get("complements", {}).get("est_ess"),
complements_est_finess=company.get("complements", {}).get("est_finess"),
- complements_est_organisme_formation=company.get("complements", {}).get("est_organisme_formation"),
- complements_est_qualiopi=company.get("complements", {}).get("est_qualiopi"),
- complements_liste_id_organisme_formation=company.get("complements", {}).get("liste_id_organisme_formation"),
+ complements_est_organisme_formation=company.get("complements", {}).get(
+ "est_organisme_formation"
+ ),
+ complements_est_qualiopi=company.get("complements", {}).get(
+ "est_qualiopi"
+ ),
+ complements_liste_id_organisme_formation=company.get(
+ "complements", {}
+ ).get("liste_id_organisme_formation"),
complements_est_rge=company.get("complements", {}).get("est_rge"),
- complements_est_service_public=company.get("complements", {}).get("est_service_public"),
+ complements_est_service_public=company.get("complements", {}).get(
+ "est_service_public"
+ ),
complements_est_l100_3=company.get("complements", {}).get("est_l100_3"),
complements_est_siae=company.get("complements", {}).get("est_siae"),
- complements_est_societe_mission=company.get("complements", {}).get("est_societe_mission"),
+ complements_est_societe_mission=company.get("complements", {}).get(
+ "est_societe_mission"
+ ),
complements_est_uai=company.get("complements", {}).get("est_uai"),
- complements_est_patrimoine_vivant=company.get("complements", {}).get("est_patrimoine_vivant"),
- complements_bilan_ges_renseigne=company.get("complements", {}).get("bilan_ges_renseigne"),
- complements_identifiant_association=company.get("complements", {}).get("identifiant_association"),
- complements_statut_entrepreneur_spectacle=company.get("complements", {}).get("statut_entrepreneur_spectacle"),
- complements_type_siae=company.get("complements", {}).get("type_siae")
+ complements_est_patrimoine_vivant=company.get("complements", {}).get(
+ "est_patrimoine_vivant"
+ ),
+ complements_bilan_ges_renseigne=company.get("complements", {}).get(
+ "bilan_ges_renseigne"
+ ),
+ complements_identifiant_association=company.get("complements", {}).get(
+ "identifiant_association"
+ ),
+ complements_statut_entrepreneur_spectacle=company.get(
+ "complements", {}
+ ).get("statut_entrepreneur_spectacle"),
+ complements_type_siae=company.get("complements", {}).get("type_siae"),
)
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"Error enriching organization: {e}"})
+ Logger.error(
+ self.sketch_id, {"message": f"Error enriching organization: {e}"}
+ )
return None
-
+
def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
if not self.neo4j_conn:
return results
@@ -222,103 +267,153 @@ class OrgToInfosScanner(Scanner):
for org in results:
# Create or update the organization node with all SIRENE properties
org_key = f"{org.name}_FR"
- self.create_node('Organization', 'org_id', org_key,
- name=org.name,
- country="FR",
- siren=org.siren,
- siege_siret=org.siege_siret,
- nom_complet=org.nom_complet,
- nom_raison_sociale=org.nom_raison_sociale,
- sigle=org.sigle,
- nombre_etablissements=org.nombre_etablissements,
- nombre_etablissements_ouverts=org.nombre_etablissements_ouverts,
- activite_principale=org.activite_principale,
- section_activite_principale=org.section_activite_principale,
- categorie_entreprise=org.categorie_entreprise,
- annee_categorie_entreprise=org.annee_categorie_entreprise,
- caractere_employeur=org.caractere_employeur,
- tranche_effectif_salarie=org.tranche_effectif_salarie,
- annee_tranche_effectif_salarie=org.annee_tranche_effectif_salarie,
- date_creation=org.date_creation,
- date_fermeture=org.date_fermeture,
- date_mise_a_jour=org.date_mise_a_jour,
- date_mise_a_jour_insee=org.date_mise_a_jour_insee,
- date_mise_a_jour_rne=org.date_mise_a_jour_rne,
- nature_juridique=org.nature_juridique,
- etat_administratif=org.etat_administratif,
- statut_diffusion=org.statut_diffusion,
- caption=org.name, type='organization')
+ self.create_node(
+ "Organization",
+ "org_id",
+ org_key,
+ name=org.name,
+ country="FR",
+ siren=org.siren,
+ siege_siret=org.siege_siret,
+ nom_complet=org.nom_complet,
+ nom_raison_sociale=org.nom_raison_sociale,
+ sigle=org.sigle,
+ nombre_etablissements=org.nombre_etablissements,
+ nombre_etablissements_ouverts=org.nombre_etablissements_ouverts,
+ activite_principale=org.activite_principale,
+ section_activite_principale=org.section_activite_principale,
+ categorie_entreprise=org.categorie_entreprise,
+ annee_categorie_entreprise=org.annee_categorie_entreprise,
+ caractere_employeur=org.caractere_employeur,
+ tranche_effectif_salarie=org.tranche_effectif_salarie,
+ annee_tranche_effectif_salarie=org.annee_tranche_effectif_salarie,
+ date_creation=org.date_creation,
+ date_fermeture=org.date_fermeture,
+ date_mise_a_jour=org.date_mise_a_jour,
+ date_mise_a_jour_insee=org.date_mise_a_jour_insee,
+ date_mise_a_jour_rne=org.date_mise_a_jour_rne,
+ nature_juridique=org.nature_juridique,
+ etat_administratif=org.etat_administratif,
+ statut_diffusion=org.statut_diffusion,
+ caption=org.name,
+ type="organization",
+ )
if org.siren:
self.log_graph_message(f"{org.name}: SIREN {org.siren} -> {org.name}")
# Add SIRET as identifier if available
if org.siege_siret:
- self.log_graph_message(f"{org.name}: SIRET {org.siege_siret} -> {org.name}")
+ self.log_graph_message(
+ f"{org.name}: SIRET {org.siege_siret} -> {org.name}"
+ )
# Add dirigeants (leaders) as Individual nodes with relationships
if org.dirigeants:
for dirigeant in org.dirigeants:
- self.create_node('Individual', 'full_name', dirigeant.full_name,
- first_name=dirigeant.first_name,
- last_name=dirigeant.last_name,
- birth_date=dirigeant.birth_date,
- gender=dirigeant.gender,
- caption=dirigeant.full_name, type='individual')
-
- self.create_relationship('Organization', 'org_id', org_key,
- 'Individual', 'full_name', dirigeant.full_name, 'HAS_LEADER')
- self.log_graph_message(f"{org.name}: HAS_LEADER -> {dirigeant.full_name}")
+ self.create_node(
+ "Individual",
+ "full_name",
+ dirigeant.full_name,
+ first_name=dirigeant.first_name,
+ last_name=dirigeant.last_name,
+ birth_date=dirigeant.birth_date,
+ gender=dirigeant.gender,
+ caption=dirigeant.full_name,
+ type="individual",
+ )
+
+ self.create_relationship(
+ "Organization",
+ "org_id",
+ org_key,
+ "Individual",
+ "full_name",
+ dirigeant.full_name,
+ "HAS_LEADER",
+ )
+ self.log_graph_message(
+ f"{org.name}: HAS_LEADER -> {dirigeant.full_name}"
+ )
# Add siege address as PhysicalAddress node if available
if org.siege_geo_adresse:
address = org.siege_geo_adresse
address_key = f"{address.address}_{address.city}_{address.country}"
- self.create_node('PhysicalAddress', 'address_id', address_key,
- address=address.address,
- city=address.city,
- country=address.country,
- zip=address.zip,
- latitude=address.latitude,
- longitude=address.longitude,
- label=f"{address.address}, {address.city}",
- caption=f"{address.address}, {address.city}",
- type='location')
-
- self.create_relationship('Organization', 'org_id', org_key,
- 'PhysicalAddress', 'address_id', address_key, 'HAS_ADDRESS')
- self.log_graph_message(f"{org.name}: HAS_ADDRESS -> {address.address}, {address.city}")
+ self.create_node(
+ "PhysicalAddress",
+ "address_id",
+ address_key,
+ address=address.address,
+ city=address.city,
+ country=address.country,
+ zip=address.zip,
+ latitude=address.latitude,
+ longitude=address.longitude,
+ label=f"{address.address}, {address.city}",
+ caption=f"{address.address}, {address.city}",
+ type="location",
+ )
+
+ self.create_relationship(
+ "Organization",
+ "org_id",
+ org_key,
+ "PhysicalAddress",
+ "address_id",
+ address_key,
+ "HAS_ADDRESS",
+ )
+ self.log_graph_message(
+ f"{org.name}: HAS_ADDRESS -> {address.address}, {address.city}"
+ )
# Add siege location as Location node if coordinates are available but no PhysicalAddress
elif org.siege_latitude and org.siege_longitude:
location_key = f"{org.siege_latitude}_{org.siege_longitude}"
- self.create_node('Location', 'location_id', location_key,
- latitude=float(org.siege_latitude),
- longitude=float(org.siege_longitude),
- address=org.siege_adresse,
- city=org.siege_libelle_commune,
- country="FR",
- zip=org.siege_code_postal,
- label=f"{org.siege_adresse or 'Unknown'}, {org.siege_libelle_commune or 'Unknown'}",
- caption=f"{org.siege_adresse or 'Unknown'}, {org.siege_libelle_commune or 'Unknown'}",
- type='location')
-
- self.create_relationship('Organization', 'org_id', org_key,
- 'Location', 'location_id', location_key, 'LOCATED_AT')
- self.log_graph_message(f"{org.name}: LOCATED_AT -> {org.siege_libelle_commune or 'Unknown'}")
+ self.create_node(
+ "Location",
+ "location_id",
+ location_key,
+ latitude=float(org.siege_latitude),
+ longitude=float(org.siege_longitude),
+ address=org.siege_adresse,
+ city=org.siege_libelle_commune,
+ country="FR",
+ zip=org.siege_code_postal,
+ label=f"{org.siege_adresse or 'Unknown'}, {org.siege_libelle_commune or 'Unknown'}",
+ caption=f"{org.siege_adresse or 'Unknown'}, {org.siege_libelle_commune or 'Unknown'}",
+ type="location",
+ )
+
+ self.create_relationship(
+ "Organization",
+ "org_id",
+ org_key,
+ "Location",
+ "location_id",
+ location_key,
+ "LOCATED_AT",
+ )
+ self.log_graph_message(
+ f"{org.name}: LOCATED_AT -> {org.siege_libelle_commune or 'Unknown'}"
+ )
# Add activity codes as Activity nodes
if org.activite_principale:
- self.log_graph_message(f"{org.name}: HAS_ACTIVITY -> {org.activite_principale}")
+ self.log_graph_message(
+ f"{org.name}: HAS_ACTIVITY -> {org.activite_principale}"
+ )
# Add legal nature as LegalNature node
if org.nature_juridique:
- self.log_graph_message(f"{org.name}: HAS_LEGAL_NATURE -> {org.nature_juridique}")
+ self.log_graph_message(
+ f"{org.name}: HAS_LEGAL_NATURE -> {org.nature_juridique}"
+ )
return results
+
# Make types available at module level for easy access
InputType = OrgToInfosScanner.InputType
OutputType = OrgToInfosScanner.OutputType
-
-
diff --git a/flowsint-transforms/src/flowsint_transforms/phones/ignorant.py b/flowsint-transforms/src/flowsint_transforms/phones/ignorant.py
index 4d447f19..9c5cb65c 100644
--- a/flowsint-transforms/src/flowsint_transforms/phones/ignorant.py
+++ b/flowsint-transforms/src/flowsint_transforms/phones/ignorant.py
@@ -5,20 +5,21 @@ from flowsint_core.utils import is_valid_number
from flowsint_core.core.logger import Logger
import httpx
+
class IgnorantScanner(Scanner):
-
+
# Define types as class attributes - base class handles schema generation automatically
InputType = List[str] # Phone numbers as strings
OutputType = List[Dict[str, Any]] # Results as dictionaries
-
+
@classmethod
def name(cls) -> str:
return "ignorant_scanner"
-
+
@classmethod
def category(cls) -> str:
return "phones"
-
+
@classmethod
def key(cls) -> str:
return "number"
@@ -44,50 +45,46 @@ class IgnorantScanner(Scanner):
result = await self._perform_ignorant_research(cleaned_phone)
results.append(result)
else:
- results.append({
- "number": phone,
- "error": "Invalid phone number"
- })
+ results.append({"number": phone, "error": "Invalid phone number"})
except Exception as e:
- results.append({
- "number": phone,
- "error": f"Unexpected error in Ignorant scan: {str(e)}"
- })
- Logger.error(self.sketch_id, {"message": f"Error scanning phone {phone}: {str(e)}"})
+ results.append(
+ {
+ "number": phone,
+ "error": f"Unexpected error in Ignorant scan: {str(e)}",
+ }
+ )
+ Logger.error(
+ self.sketch_id,
+ {"message": f"Error scanning phone {phone}: {str(e)}"},
+ )
return results
-
+
async def _perform_ignorant_research(self, phone: str) -> Dict[str, Any]:
try:
# Import necessary modules for each platform
from ignorant.modules.shopping.amazon import amazon
from ignorant.modules.social_media.instagram import instagram
from ignorant.modules.social_media.snapchat import snapchat
-
+
# Create an HTTP client for asynchronous requests
async with httpx.AsyncClient() as client:
results = []
modules = [amazon, snapchat, instagram]
-
+
# Execute the modules in parallel
tasks = [module(phone, "+33", client) for module in modules]
responses = await asyncio.gather(*tasks)
-
+
# Add results from each module
for response in responses:
if response:
results.append(response)
-
- return {
- "number": phone,
- "platforms": results
- }
-
+
+ return {"number": phone, "platforms": results}
+
except Exception as e:
- return {
- "number": phone,
- "error": f"Error in Ignorant research: {str(e)}"
- }
-
+ return {"number": phone, "error": f"Error in Ignorant research: {str(e)}"}
+
def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
"""
Create Neo4j relationships for found phone accounts.
@@ -97,17 +94,25 @@ class IgnorantScanner(Scanner):
for result in results:
if "error" not in result and "platforms" in result:
- self.create_node('phone', 'number', result["number"],
- caption=result["number"], type='phone')
-
+ self.create_node(
+ "phone",
+ "number",
+ result["number"],
+ caption=result["number"],
+ type="phone",
+ )
+
# Create platform relationships
for platform_result in result["platforms"]:
if platform_result and isinstance(platform_result, dict):
platform_name = platform_result.get("platform", "unknown")
- self.log_graph_message(f"Phone {result['number']} found on {platform_name}")
+ self.log_graph_message(
+ f"Phone {result['number']} found on {platform_name}"
+ )
return results
+
# Make types available at module level for easy access
InputType = IgnorantScanner.InputType
OutputType = IgnorantScanner.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/socials/maigret.py b/flowsint-transforms/src/flowsint_transforms/socials/maigret.py
index a215ca01..ba6dbf07 100644
--- a/flowsint-transforms/src/flowsint_transforms/socials/maigret.py
+++ b/flowsint-transforms/src/flowsint_transforms/socials/maigret.py
@@ -9,6 +9,7 @@ from flowsint_core.core.logger import Logger
false_positives = ["LeagueOfLegends"]
+
class MaigretScanner(Scanner):
"""Scans usernames for associated social accounts using Maigret."""
@@ -42,7 +43,7 @@ class MaigretScanner(Scanner):
if obj and obj.username and is_valid_username(obj.username):
cleaned.append(obj)
return cleaned
-
+
def run_maigret(self, username: str) -> Path:
output_file = Path(f"/tmp/report_{username}_simple.json")
try:
@@ -50,13 +51,18 @@ class MaigretScanner(Scanner):
["maigret", username, "-J", "simple", "-fo", "/tmp"],
capture_output=True,
text=True,
- timeout=100
+ timeout=100,
)
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"Maigret execution failed for {username}: {e}"})
+ Logger.error(
+ self.sketch_id,
+ {"message": f"Maigret execution failed for {username}: {e}"},
+ )
return output_file
-
- def parse_maigret_output(self, username: str, output_file: Path) -> List[SocialProfile]:
+
+ def parse_maigret_output(
+ self, username: str, output_file: Path
+ ) -> List[SocialProfile]:
results: List[SocialProfile] = []
if not output_file.exists():
return results
@@ -65,13 +71,16 @@ class MaigretScanner(Scanner):
with open(output_file, "r") as f:
raw_data = json.load(f)
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"Failed to load output file for {username}: {e}"})
+ Logger.error(
+ self.sketch_id,
+ {"message": f"Failed to load output file for {username}: {e}"},
+ )
return results
for platform, profile in raw_data.items():
if profile.get("status", {}).get("status") != "Claimed":
continue
-
+
if any(fp in platform for fp in false_positives):
continue
@@ -82,26 +91,37 @@ class MaigretScanner(Scanner):
continue
try:
- followers = int(ids.get("follower_count", 0)) if ids.get("follower_count") else None
- following = int(ids.get("following_count", 0)) if ids.get("following_count") else None
+ followers = (
+ int(ids.get("follower_count", 0))
+ if ids.get("follower_count")
+ else None
+ )
+ following = (
+ int(ids.get("following_count", 0))
+ if ids.get("following_count")
+ else None
+ )
posts = (
- int(ids.get("public_repos_count", 0)) +
- int(ids.get("public_gists_count", 0))
- if "public_repos_count" in ids or "public_gists_count" in ids else None
+ int(ids.get("public_repos_count", 0))
+ + int(ids.get("public_gists_count", 0))
+ if "public_repos_count" in ids or "public_gists_count" in ids
+ else None
)
except ValueError:
followers = following = posts = None
- results.append(SocialProfile(
- username=username,
- profile_url=profile_url,
- platform=platform,
- profile_picture_url=ids.get("image"),
- bio=None,
- followers_count=followers,
- following_count=following,
- posts_count=posts
- ))
+ results.append(
+ SocialProfile(
+ username=username,
+ profile_url=profile_url,
+ platform=platform,
+ profile_picture_url=ids.get("image"),
+ bio=None,
+ followers_count=followers,
+ following_count=following,
+ posts_count=posts,
+ )
+ )
return results
@@ -121,30 +141,44 @@ class MaigretScanner(Scanner):
for profile in results:
# Create social profile node
- self.create_node('social_profile', 'profile_url', profile.profile_url,
- username=profile.username,
- platform=profile.platform,
- profile_picture_url=profile.profile_picture_url,
- bio=profile.bio,
- followers_count=profile.followers_count,
- following_count=profile.following_count,
- posts_count=profile.posts_count,
- label=f"{profile.platform}:{profile.username}",
- caption=f"{profile.platform}:{profile.username}",
- color="#1DA1F2",
- type="social_profile")
-
+ self.create_node(
+ "social_profile",
+ "profile_url",
+ profile.profile_url,
+ username=profile.username,
+ platform=profile.platform,
+ profile_picture_url=profile.profile_picture_url,
+ bio=profile.bio,
+ followers_count=profile.followers_count,
+ following_count=profile.following_count,
+ posts_count=profile.posts_count,
+ label=f"{profile.platform}:{profile.username}",
+ caption=f"{profile.platform}:{profile.username}",
+ color="#1DA1F2",
+ type="social_profile",
+ )
+
# Create username node
- self.create_node('username', 'username', profile.username, type='username')
-
+ self.create_node("username", "username", profile.username, type="username")
+
# Create relationship
- self.create_relationship('username', 'username', profile.username,
- 'social_profile', 'profile_url', profile.profile_url, 'HAS_SOCIAL_ACCOUNT')
-
- self.log_graph_message(f"{profile.username} -> account found on {profile.platform}")
+ self.create_relationship(
+ "username",
+ "username",
+ profile.username,
+ "social_profile",
+ "profile_url",
+ profile.profile_url,
+ "HAS_SOCIAL_ACCOUNT",
+ )
+
+ self.log_graph_message(
+ f"{profile.username} -> account found on {profile.platform}"
+ )
return results
+
# Make types available at module level for easy access
InputType = MaigretScanner.InputType
OutputType = MaigretScanner.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/socials/sherlock.py b/flowsint-transforms/src/flowsint_transforms/socials/sherlock.py
index 06f9d693..73609be7 100644
--- a/flowsint-transforms/src/flowsint_transforms/socials/sherlock.py
+++ b/flowsint-transforms/src/flowsint_transforms/socials/sherlock.py
@@ -2,7 +2,7 @@ import subprocess
from pathlib import Path
from typing import List, Union
from flowsint_core.utils import is_valid_username
-from flowsint_types.social import Social
+from flowsint_types.social import SocialProfile
from flowsint_core.core.scanner_base import Scanner
from flowsint_core.core.logger import Logger
@@ -11,30 +11,30 @@ class SherlockScanner(Scanner):
"""Scans the usernames for associated social accounts using Sherlock."""
# Define types as class attributes - base class handles schema generation automatically
- InputType = List[Social]
- OutputType = List[Social]
+ InputType = List[SocialProfile]
+ OutputType = List[SocialProfile]
@classmethod
def name(cls) -> str:
return "sherlock_scanner"
-
+
@classmethod
def category(cls) -> str:
return "social_account"
-
+
@classmethod
def key(cls) -> str:
return "username"
-
+
def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:
cleaned: InputType = []
for item in data:
obj = None
if isinstance(item, str):
- obj = Social(username=item)
+ obj = SocialProfile(username=item)
elif isinstance(item, dict) and "username" in item:
- obj = Social(username=item["username"])
- elif isinstance(item, Social):
+ obj = SocialProfile(username=item["username"])
+ elif isinstance(item, SocialProfile):
obj = item
if obj and obj.username and is_valid_username(obj.username):
@@ -44,7 +44,7 @@ class SherlockScanner(Scanner):
async def scan(self, data: InputType) -> OutputType:
"""Performs the scan using Sherlock on the list of usernames."""
results: OutputType = []
-
+
for social in data:
username = social.username
output_file = Path(f"/tmp/sherlock_{username}.txt")
@@ -54,15 +54,25 @@ class SherlockScanner(Scanner):
["sherlock", username, "-o", str(output_file)],
capture_output=True,
text=True,
- timeout=100
+ timeout=100,
)
if result.returncode != 0:
- Logger.error(self.sketch_id, {"message": f"Sherlock failed for {username}: {result.stderr.strip()}"})
+ Logger.error(
+ self.sketch_id,
+ {
+ "message": f"Sherlock failed for {username}: {result.stderr.strip()}"
+ },
+ )
continue
if not output_file.exists():
- Logger.error(self.sketch_id, {"message": f"Sherlock did not produce any output file for {username}."})
+ Logger.error(
+ self.sketch_id,
+ {
+ "message": f"Sherlock did not produce any output file for {username}."
+ },
+ )
continue
found_accounts = {}
@@ -75,16 +85,22 @@ class SherlockScanner(Scanner):
# Create Social objects for each found account
for platform, url in found_accounts.items():
- results.append(Social(
- username=username,
- platform=platform,
- url=url
- ))
+ results.append(
+ SocialProfile(username=username, platform=platform, url=url)
+ )
except subprocess.TimeoutExpired:
- Logger.error(self.sketch_id, {"message": f"Sherlock scan for {username} timed out."})
+ Logger.error(
+ self.sketch_id,
+ {"message": f"Sherlock scan for {username} timed out."},
+ )
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"Unexpected error in Sherlock scan for {username}: {str(e)}"})
+ Logger.error(
+ self.sketch_id,
+ {
+ "message": f"Unexpected error in Sherlock scan for {username}: {str(e)}"
+ },
+ )
return results
@@ -94,13 +110,22 @@ class SherlockScanner(Scanner):
return results
for social in results:
- self.create_node('social', 'username', social.username,
- platform=social.platform, url=social.url,
- caption=social.platform, type='social')
- self.log_graph_message(f"Found social account: {social.username} on {social.platform}")
+ self.create_node(
+ "social",
+ "username",
+ social.username,
+ platform=social.platform,
+ url=social.url,
+ caption=social.platform,
+ type="social",
+ )
+ self.log_graph_message(
+ f"Found social account: {social.username} on {social.platform}"
+ )
return results
+
# Make types available at module level for easy access
InputType = SherlockScanner.InputType
OutputType = SherlockScanner.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/websites/to_crawler.py b/flowsint-transforms/src/flowsint_transforms/websites/to_crawler.py
index a2e7ba19..72d35300 100644
--- a/flowsint-transforms/src/flowsint_transforms/websites/to_crawler.py
+++ b/flowsint-transforms/src/flowsint_transforms/websites/to_crawler.py
@@ -8,6 +8,7 @@ from flowsint_core.core.logger import Logger
from tools.network.reconcrawl import ReconCrawlTool
from pydantic import BaseModel
+
class ReturnType(BaseModel):
website: Website
emails: Optional[Email]
@@ -28,7 +29,7 @@ class WebsiteToCrawler(Scanner):
@classmethod
def category(cls) -> str:
return "Website"
-
+
@classmethod
def key(cls) -> str:
return "url"
@@ -55,53 +56,94 @@ class WebsiteToCrawler(Scanner):
if website_obj:
cleaned.append(website_obj)
return cleaned
-
+
async def scan(self, data: InputType) -> OutputType:
"""Crawl websites to extract emails and phone numbers."""
results = []
-
+
for website in data:
try:
- Logger.info(self.sketch_id, {"message": f"Starting comprehensive crawl of {str(website.url)}"})
+ Logger.info(
+ self.sketch_id,
+ {"message": f"Starting comprehensive crawl of {str(website.url)}"},
+ )
crawler = ReconCrawlTool()
- crawl_result = crawler.launch(website.url, {"recursive": True, "verify_ssl": True, "max_pages": 500, "timeout": 10, "delay": 1.0, "verbose": False})
+ crawl_result = crawler.launch(
+ website.url,
+ {
+ "recursive": True,
+ "verify_ssl": True,
+ "max_pages": 500,
+ "timeout": 10,
+ "delay": 1.0,
+ "verbose": False,
+ },
+ )
website_result = {
- "website": str(website.url), # Store as string instead of Website object
+ "website": str(
+ website.url
+ ), # Store as string instead of Website object
"emails": [],
"phones": [],
}
for item in crawl_result:
- Logger.info(self.sketch_id, {"message": f"{item.type}: {item.value}"})
+ Logger.info(
+ self.sketch_id, {"message": f"{item.type}: {item.value}"}
+ )
if item.source_url:
- Logger.info(self.sketch_id, {"message": f" Found on: {item.source_url}"})
+ Logger.info(
+ self.sketch_id,
+ {"message": f" Found on: {item.source_url}"},
+ )
if item.type == "email":
website_result["emails"].append(Email(email=item.value))
if item.type == "phone":
website_result["phones"].append(Phone(number=item.value))
# Log results
- Logger.info(self.sketch_id, {
- "message": f"Crawl completed for {str(website.url)}: {len(website_result['emails'])} emails, {len(website_result['phones'])} phones found."
- })
-
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"Crawl completed for {str(website.url)}: {len(website_result['emails'])} emails, {len(website_result['phones'])} phones found."
+ },
+ )
+
if not website_result["emails"] and not website_result["phones"]:
- Logger.info(self.sketch_id, {"message": f"No emails or phones found for website {str(website.url)}."})
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"No emails or phones found for website {str(website.url)}."
+ },
+ )
elif not website_result["emails"]:
- Logger.info(self.sketch_id, {"message": f"No emails found for website {str(website.url)}"})
+ Logger.info(
+ self.sketch_id,
+ {"message": f"No emails found for website {str(website.url)}"},
+ )
elif not website_result["phones"]:
- Logger.info(self.sketch_id, {"message": f"No phones found for website {str(website.url)}"})
-
+ Logger.info(
+ self.sketch_id,
+ {"message": f"No phones found for website {str(website.url)}"},
+ )
+
results.append(website_result)
-
+
except Exception as e:
# Log error but continue with other websites
- Logger.error(self.sketch_id, {"message": f"Error crawling {str(website.url)}: {str(e)}"})
+ Logger.error(
+ self.sketch_id,
+ {"message": f"Error crawling {str(website.url)}: {str(e)}"},
+ )
# Add empty result for failed website
- results.append({
- "website": str(website.url), # Store as string instead of Website object
- "emails": [],
- "phones": [],
- })
+ results.append(
+ {
+ "website": str(
+ website.url
+ ), # Store as string instead of Website object
+ "emails": [],
+ "phones": [],
+ }
+ )
continue
return results
@@ -110,29 +152,55 @@ class WebsiteToCrawler(Scanner):
# Create Neo4j relationships between websites and their corresponding emails and phones
for input_website, result in zip(original_input, results):
website_url = str(input_website.url)
-
+
# Create website node
if self.neo4j_conn:
- self.create_node('website', 'url', website_url,
- caption=website_url, type='website')
-
+ self.create_node(
+ "website", "url", website_url, caption=website_url, type="website"
+ )
+
# Create email nodes and relationships
for email in result["emails"]:
- self.create_node('email', 'email', email.email,
- caption=email.email, type='email')
- self.create_relationship('website', 'url', website_url,
- 'email', 'email', email.email, 'HAS_EMAIL')
- self.log_graph_message(f"Found email {email.email} for website {website_url}")
-
+ self.create_node(
+ "email", "email", email.email, caption=email.email, type="email"
+ )
+ self.create_relationship(
+ "website",
+ "url",
+ website_url,
+ "email",
+ "email",
+ email.email,
+ "HAS_EMAIL",
+ )
+ self.log_graph_message(
+ f"Found email {email.email} for website {website_url}"
+ )
+
# Create phone nodes and relationships
for phone in result["phones"]:
- self.create_node('phone', 'number', phone.number,
- caption=phone.number, type='phone')
- self.create_relationship('website', 'url', website_url,
- 'phone', 'number', phone.number, 'HAS_PHONE')
- self.log_graph_message(f"Found phone {phone.number} for website {website_url}")
+ self.create_node(
+ "phone",
+ "number",
+ phone.number,
+ caption=phone.number,
+ type="phone",
+ )
+ self.create_relationship(
+ "website",
+ "url",
+ website_url,
+ "phone",
+ "number",
+ phone.number,
+ "HAS_PHONE",
+ )
+ self.log_graph_message(
+ f"Found phone {phone.number} for website {website_url}"
+ )
return results
-
+
+
InputType = WebsiteToCrawler.InputType
-OutputType = WebsiteToCrawler.OutputType
\ No newline at end of file
+OutputType = WebsiteToCrawler.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/websites/to_domain.py b/flowsint-transforms/src/flowsint_transforms/websites/to_domain.py
index a847c8ff..0d1403e5 100644
--- a/flowsint-transforms/src/flowsint_transforms/websites/to_domain.py
+++ b/flowsint-transforms/src/flowsint_transforms/websites/to_domain.py
@@ -5,6 +5,7 @@ from flowsint_types.website import Website
from flowsint_types.domain import Domain
from flowsint_core.core.logger import Logger
+
class WebsiteToDomainScanner(Scanner):
"""From website to domain."""
@@ -19,7 +20,7 @@ class WebsiteToDomainScanner(Scanner):
@classmethod
def category(cls) -> str:
return "Website"
-
+
@classmethod
def key(cls) -> str:
return "website"
@@ -44,28 +45,36 @@ class WebsiteToDomainScanner(Scanner):
try:
parsed_url = urlparse(website.url)
domain_name = parsed_url.netloc
-
+
# Remove port if present
- if ':' in domain_name:
- domain_name = domain_name.split(':')[0]
-
+ if ":" in domain_name:
+ domain_name = domain_name.split(":")[0]
+
# Remove www. prefix if present
- if domain_name.startswith('www.'):
+ if domain_name.startswith("www."):
domain_name = domain_name[4:]
-
+
if domain_name:
domain_obj = Domain(domain=domain_name)
results.append(domain_obj)
-
+
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"Error extracting domain from website {website.url}: {e}"})
+ Logger.error(
+ self.sketch_id,
+ {
+ "message": f"Error extracting domain from website {website.url}: {e}"
+ },
+ )
continue
-
+
return results
- def postprocess(self, results: OutputType, input_data: InputType = None) -> OutputType:
+ def postprocess(
+ self, results: OutputType, input_data: InputType = None
+ ) -> OutputType:
return results
+
# Make types available at module level for easy access
InputType = WebsiteToDomainScanner.InputType
-OutputType = WebsiteToDomainScanner.OutputType
\ No newline at end of file
+OutputType = WebsiteToDomainScanner.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/websites/to_links.py b/flowsint-transforms/src/flowsint_transforms/websites/to_links.py
index 324712b2..fa465eaf 100644
--- a/flowsint-transforms/src/flowsint_transforms/websites/to_links.py
+++ b/flowsint-transforms/src/flowsint_transforms/websites/to_links.py
@@ -5,6 +5,7 @@ from flowsint_types.website import Website
from flowsint_core.core.logger import Logger
from reconspread import Crawler
+
class WebsiteToLinks(Scanner):
"""From website to spread crawler that extracts domains and internal/external links."""
@@ -19,7 +20,7 @@ class WebsiteToLinks(Scanner):
@classmethod
def category(cls) -> str:
return "Website"
-
+
@classmethod
def key(cls) -> str:
return "url"
@@ -45,34 +46,56 @@ class WebsiteToLinks(Scanner):
if website_obj:
cleaned.append(website_obj)
return cleaned
-
+
async def scan(self, data: InputType) -> OutputType:
"""Crawl websites using reconspread to extract internal and external links."""
results = []
-
+
for website in data:
try:
- Logger.info(self.sketch_id, {"message": f"Starting reconspread crawl of {str(website.url)}"})
-
+ Logger.info(
+ self.sketch_id,
+ {"message": f"Starting reconspread crawl of {str(website.url)}"},
+ )
+
# Extract main domain from input website (needed in callback)
main_domain = self.extract_domain(str(website.url))
-
+
# Create main website and domain nodes upfront
if self.neo4j_conn:
- self.create_node('website', 'url', str(website.url),
- caption=str(website.url), type='website')
+ self.create_node(
+ "website",
+ "url",
+ str(website.url),
+ caption=str(website.url),
+ type="website",
+ )
if main_domain:
- self.create_node('domain', 'name', main_domain,
- caption=main_domain, type='domain')
- self.create_relationship('website', 'url', str(website.url),
- 'domain', 'name', main_domain, 'BELONGS_TO_DOMAIN')
- self.log_graph_message(f"Website {str(website.url)} belongs to domain {main_domain}")
-
+ self.create_node(
+ "domain",
+ "name",
+ main_domain,
+ caption=main_domain,
+ type="domain",
+ )
+ self.create_relationship(
+ "website",
+ "url",
+ str(website.url),
+ "domain",
+ "name",
+ main_domain,
+ "BELONGS_TO_DOMAIN",
+ )
+ self.log_graph_message(
+ f"Website {str(website.url)} belongs to domain {main_domain}"
+ )
+
# Store discovered URLs
internal_urls = []
external_urls = []
external_domains = set()
-
+
def url_handler(url, is_external=False):
"""Custom callback to handle URLs as they're discovered."""
if is_external:
@@ -82,38 +105,95 @@ class WebsiteToLinks(Scanner):
external_domains.add(domain)
# Create external website node immediately
if self.neo4j_conn:
- self.create_node('website', 'url', url,
- caption=url, type='website')
- self.create_relationship('website', 'url', str(website.url),
- 'website', 'url', url, 'LINKS_TO')
- self.log_graph_message(f"Website {str(website.url)} links to external website {url}")
-
+ self.create_node(
+ "website", "url", url, caption=url, type="website"
+ )
+ self.create_relationship(
+ "website",
+ "url",
+ str(website.url),
+ "website",
+ "url",
+ url,
+ "LINKS_TO",
+ )
+ self.log_graph_message(
+ f"Website {str(website.url)} links to external website {url}"
+ )
+
# Create external domain node and link external website to its domain
if domain != main_domain:
- self.create_node('domain', 'name', domain,
- caption=domain, type='domain')
- self.create_relationship('website', 'url', url,
- 'domain', 'name', domain, 'BELONGS_TO_DOMAIN')
- self.create_relationship('website', 'url', str(website.url),
- 'domain', 'name', domain, 'LINKS_TO_DOMAIN')
- self.log_graph_message(f"External website {url} belongs to domain {domain}")
- self.log_graph_message(f"Website {str(website.url)} links to external domain {domain}")
- Logger.info(self.sketch_id, {"message": f"[EXTERNAL] Found: {url} -> Domain: {domain}"})
+ self.create_node(
+ "domain",
+ "name",
+ domain,
+ caption=domain,
+ type="domain",
+ )
+ self.create_relationship(
+ "website",
+ "url",
+ url,
+ "domain",
+ "name",
+ domain,
+ "BELONGS_TO_DOMAIN",
+ )
+ self.create_relationship(
+ "website",
+ "url",
+ str(website.url),
+ "domain",
+ "name",
+ domain,
+ "LINKS_TO_DOMAIN",
+ )
+ self.log_graph_message(
+ f"External website {url} belongs to domain {domain}"
+ )
+ self.log_graph_message(
+ f"Website {str(website.url)} links to external domain {domain}"
+ )
+ Logger.info(
+ self.sketch_id,
+ {"message": f"[EXTERNAL] Found: {url} -> Domain: {domain}"},
+ )
else:
internal_urls.append(url)
# Create internal website node immediately
- if self.neo4j_conn and url != str(website.url): # Don't create duplicate of main website
- self.create_node('website', 'url', url,
- caption=url, type='website')
- self.create_relationship('website', 'url', str(website.url),
- 'website', 'url', url, 'LINKS_TO')
- self.log_graph_message(f"Website {str(website.url)} links to internal website {url}")
-
+ if self.neo4j_conn and url != str(
+ website.url
+ ): # Don't create duplicate of main website
+ self.create_node(
+ "website", "url", url, caption=url, type="website"
+ )
+ self.create_relationship(
+ "website",
+ "url",
+ str(website.url),
+ "website",
+ "url",
+ url,
+ "LINKS_TO",
+ )
+ self.log_graph_message(
+ f"Website {str(website.url)} links to internal website {url}"
+ )
+
# Also link internal websites to main domain
if main_domain:
- self.create_relationship('website', 'url', url,
- 'domain', 'name', main_domain, 'BELONGS_TO_DOMAIN')
- Logger.info(self.sketch_id, {"message": f"[INTERNAL] Found: {url}"})
+ self.create_relationship(
+ "website",
+ "url",
+ url,
+ "domain",
+ "name",
+ main_domain,
+ "BELONGS_TO_DOMAIN",
+ )
+ Logger.info(
+ self.sketch_id, {"message": f"[INTERNAL] Found: {url}"}
+ )
# Create crawler with custom callback
crawler = Crawler(
@@ -121,7 +201,7 @@ class WebsiteToLinks(Scanner):
recursive=True,
same_domain_only=False,
verbose=False, # Disable default verbose output
- _on_result_callback=url_handler # Use our custom handler
+ _on_result_callback=url_handler, # Use our custom handler
)
# Perform the crawl
@@ -130,12 +210,12 @@ class WebsiteToLinks(Scanner):
# Get final results (backup in case callback missed anything)
crawl_results = crawler.get_results()
-
+
# Ensure we have all internal URLs
for url in crawl_results.internal:
if url not in internal_urls:
internal_urls.append(url)
-
+
# Ensure we have all external URLs and domains
for url in crawl_results.external:
if url not in external_urls:
@@ -149,44 +229,71 @@ class WebsiteToLinks(Scanner):
"main_domain": main_domain,
"internal_urls": internal_urls,
"external_urls": external_urls,
- "external_domains": list(external_domains)
+ "external_domains": list(external_domains),
}
# Log results
- Logger.info(self.sketch_id, {
- "message": f"Spread crawl completed for {str(website.url)}: "
- f"Main domain: {main_domain}, "
- f"{len(internal_urls)} internal URLs, "
- f"{len(external_urls)} external URLs, "
- f"{len(external_domains)} external domains found."
- })
-
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"Spread crawl completed for {str(website.url)}: "
+ f"Main domain: {main_domain}, "
+ f"{len(internal_urls)} internal URLs, "
+ f"{len(external_urls)} external URLs, "
+ f"{len(external_domains)} external domains found."
+ },
+ )
+
results.append(website_result)
-
+
except Exception as e:
# Log error but continue with other websites
- Logger.error(self.sketch_id, {"message": f"Error crawling {str(website.url)}: {str(e)}"})
-
+ Logger.error(
+ self.sketch_id,
+ {"message": f"Error crawling {str(website.url)}: {str(e)}"},
+ )
+
# Still create main website and domain nodes even on error
main_domain = self.extract_domain(str(website.url))
if self.neo4j_conn:
- self.create_node('website', 'url', str(website.url),
- caption=str(website.url), type='website')
+ self.create_node(
+ "website",
+ "url",
+ str(website.url),
+ caption=str(website.url),
+ type="website",
+ )
if main_domain:
- self.create_node('domain', 'name', main_domain,
- caption=main_domain, type='domain')
- self.create_relationship('website', 'url', str(website.url),
- 'domain', 'name', main_domain, 'BELONGS_TO_DOMAIN')
- self.log_graph_message(f"Website {str(website.url)} belongs to domain {main_domain}")
-
+ self.create_node(
+ "domain",
+ "name",
+ main_domain,
+ caption=main_domain,
+ type="domain",
+ )
+ self.create_relationship(
+ "website",
+ "url",
+ str(website.url),
+ "domain",
+ "name",
+ main_domain,
+ "BELONGS_TO_DOMAIN",
+ )
+ self.log_graph_message(
+ f"Website {str(website.url)} belongs to domain {main_domain}"
+ )
+
# Add empty result for failed website
- results.append({
- "website": str(website.url),
- "main_domain": main_domain,
- "internal_urls": [],
- "external_urls": [],
- "external_domains": []
- })
+ results.append(
+ {
+ "website": str(website.url),
+ "main_domain": main_domain,
+ "internal_urls": [],
+ "external_urls": [],
+ "external_domains": [],
+ }
+ )
continue
return results
@@ -195,6 +302,7 @@ class WebsiteToLinks(Scanner):
# Neo4j nodes and relationships are created in real-time during the callback
# No additional processing needed here
return results
-
+
+
InputType = WebsiteToLinks.InputType
-OutputType = WebsiteToLinks.OutputType
\ No newline at end of file
+OutputType = WebsiteToLinks.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/websites/to_webtrackers.py b/flowsint-transforms/src/flowsint_transforms/websites/to_webtrackers.py
index 0675f3de..ba4e8dc7 100644
--- a/flowsint-transforms/src/flowsint_transforms/websites/to_webtrackers.py
+++ b/flowsint-transforms/src/flowsint_transforms/websites/to_webtrackers.py
@@ -7,6 +7,7 @@ from flowsint_core.core.graph_db import Neo4jConnection
from flowsint_core.core.vault import VaultProtocol
from recontrack import TrackingCodeExtractor
+
class WebsiteToWebtrackersScanner(Scanner):
"""From website to webtrackers."""
@@ -21,7 +22,7 @@ class WebsiteToWebtrackersScanner(Scanner):
neo4j_conn: Optional[Neo4jConnection] = None,
params_schema: Optional[List[Dict[str, Any]]] = None,
vault: Optional[VaultProtocol] = None,
- params: Optional[Dict[str, Any]] = None
+ params: Optional[Dict[str, Any]] = None,
):
super().__init__(sketch_id, scan_id, neo4j_conn, params_schema, vault, params)
self.tracker_website_mapping: List[tuple[WebTracker, Website]] = []
@@ -33,7 +34,7 @@ class WebsiteToWebtrackersScanner(Scanner):
@classmethod
def category(cls) -> str:
return "Website"
-
+
@classmethod
def key(cls) -> str:
return "website"
@@ -55,31 +56,39 @@ class WebsiteToWebtrackersScanner(Scanner):
async def scan(self, data: InputType) -> OutputType:
results: OutputType = []
extractor = TrackingCodeExtractor()
-
+
for website in data:
try:
# Extract tracking codes from the website
tracking_data = extractor.extract(str(website.url))
-
+
for tracker_info in tracking_data:
tracker = WebTracker(
name=tracker_info.get("name", ""),
tracker_id=tracker_info.get("id", ""),
category=tracker_info.get("category", ""),
- website_url=str(website.url)
+ website_url=str(website.url),
)
results.append(tracker)
self.tracker_website_mapping.append((tracker, website))
-
+
except Exception as e:
- Logger.error(self.sketch_id, {"message": f"Error extracting web trackers from {website.url}: {e}"})
+ Logger.error(
+ self.sketch_id,
+ {
+ "message": f"Error extracting web trackers from {website.url}: {e}"
+ },
+ )
continue
-
+
return results
- def postprocess(self, results: OutputType, input_data: InputType = None) -> OutputType:
+ def postprocess(
+ self, results: OutputType, input_data: InputType = None
+ ) -> OutputType:
return results
+
# Make types available at module level for easy access
InputType = WebsiteToWebtrackersScanner.InputType
-OutputType = WebsiteToWebtrackersScanner.OutputType
\ No newline at end of file
+OutputType = WebsiteToWebtrackersScanner.OutputType
diff --git a/flowsint-transforms/src/tools/base.py b/flowsint-transforms/src/tools/base.py
index 6ee7f8fb..8f49e643 100644
--- a/flowsint-transforms/src/tools/base.py
+++ b/flowsint-transforms/src/tools/base.py
@@ -1,6 +1,7 @@
from abc import ABC, abstractmethod
from typing import Any
+
class Tool(ABC):
@classmethod
@abstractmethod
@@ -11,12 +12,12 @@ class Tool(ABC):
@abstractmethod
def category(cls) -> str:
pass
-
+
@classmethod
@abstractmethod
def description(cls) -> str:
pass
-
+
@classmethod
@abstractmethod
def version(cls) -> str:
diff --git a/flowsint-transforms/src/tools/dockertool.py b/flowsint-transforms/src/tools/dockertool.py
index 8ad24aac..e1a652d7 100644
--- a/flowsint-transforms/src/tools/dockertool.py
+++ b/flowsint-transforms/src/tools/dockertool.py
@@ -2,11 +2,12 @@ from docker import from_env
from docker.errors import ImageNotFound, APIError, DockerException
from .base import Tool
+
class DockerTool(Tool):
def __init__(self, image: str, default_tag: str = "latest"):
self.image = f"{image}:{default_tag}"
self.client = from_env()
-
+
@classmethod
def get_image(cls) -> str:
return cls.image
@@ -17,7 +18,7 @@ class DockerTool(Tool):
self.client.images.pull(self.image)
except APIError as e:
raise RuntimeError(f"Failed to pull image {self.image}: {e.explanation}")
-
+
def version(self) -> str:
try:
output = self.client.containers.run(
@@ -27,19 +28,18 @@ class DockerTool(Tool):
stdout=True,
stderr=True,
detach=False,
- tty=False
+ tty=False,
)
return output.decode().strip()
except Exception as e:
return f"unknown (error: {str(e)})"
-
+
def is_installed(self) -> bool:
try:
self.client.images.get(self.image)
return True
except ImageNotFound:
return False
-
def launch(self, command: str, volumes: dict = None, timeout: int = 30):
self.install()
@@ -63,13 +63,13 @@ class DockerTool(Tool):
except DockerException as e:
# Try to get more detailed error information
error_detail = str(e)
- if hasattr(e, 'response') and hasattr(e.response, 'json'):
+ if hasattr(e, "response") and hasattr(e.response, "json"):
try:
error_json = e.response.json()
error_detail = f"{str(e)} - Details: {error_json}"
except:
pass
-
+
# Check if it's a container exit error
if "returned non-zero exit status" in str(e):
# Try to run the command with stderr capture to see what went wrong
@@ -90,5 +90,7 @@ class DockerTool(Tool):
return test_result.decode()
except DockerException as test_e:
error_detail = f"{str(e)} - Test run also failed: {str(test_e)}"
-
- raise RuntimeError(f"Docker error while running {self.image}: {error_detail}")
+
+ raise RuntimeError(
+ f"Docker error while running {self.image}: {error_detail}"
+ )
diff --git a/flowsint-transforms/src/tools/network/asnmap.py b/flowsint-transforms/src/tools/network/asnmap.py
index c92b11a8..757445c2 100644
--- a/flowsint-transforms/src/tools/network/asnmap.py
+++ b/flowsint-transforms/src/tools/network/asnmap.py
@@ -2,17 +2,18 @@ import json
from typing import Any, Literal
from ..dockertool import DockerTool
+
class AsnmapTool(DockerTool):
image = "projectdiscovery/asnmap"
default_tag = "latest"
-
+
def __init__(self):
super().__init__(self.image, self.default_tag)
-
+
@classmethod
def name(cls) -> str:
return "asnmap"
-
+
@classmethod
def description(cls) -> str:
return "ASN mapping and network reconnaissance tool."
@@ -31,16 +32,17 @@ class AsnmapTool(DockerTool):
command="--version",
remove=True,
stderr=True,
- stdout=True
+ stdout=True,
)
output_str = output.decode()
import re
- match = re.search(r'(v[\d\.]+)', output_str)
+
+ match = re.search(r"(v[\d\.]+)", output_str)
version = match.group(1) if match else "unknown"
return version
except Exception as e:
return f"unknown (error: {str(e)})"
-
+
def update(self) -> None:
# Pull the latest image
self.install()
@@ -48,15 +50,14 @@ class AsnmapTool(DockerTool):
def is_installed(self) -> bool:
return super().is_installed()
- def launch(self, item: str, type: Literal["domain", "organization", "ip", "asn"] = "domain") -> Any:
- flags = {
- "domain" : '-d',
- 'org': '-org',
- 'ip' : '-i',
- 'asn': '-a'
- }
+ def launch(
+ self, item: str, type: Literal["domain", "organization", "ip", "asn"] = "domain"
+ ) -> Any:
+ flags = {"domain": "-d", "org": "-org", "ip": "-i", "asn": "-a"}
if type not in flags:
- raise ValueError(f"Invalid type: '{type}'. Valid types are: {list(flags.keys())}")
+ raise ValueError(
+ f"Invalid type: '{type}'. Valid types are: {list(flags.keys())}"
+ )
flag = flags[type]
try:
# Use the -target argument as asnmap expects
@@ -69,5 +70,6 @@ class AsnmapTool(DockerTool):
raise RuntimeError(f"Failed to parse JSON output from asnmap: {str(e)}")
except Exception as e:
# Try to get more info from the container logs
- raise RuntimeError(f"Error running asnmap: {str(e)}. Output: {getattr(e, 'output', 'No output')}")
-
+ raise RuntimeError(
+ f"Error running asnmap: {str(e)}. Output: {getattr(e, 'output', 'No output')}"
+ )
diff --git a/flowsint-transforms/src/tools/network/httpx.py b/flowsint-transforms/src/tools/network/httpx.py
index 2d1e89eb..ed13c88a 100644
--- a/flowsint-transforms/src/tools/network/httpx.py
+++ b/flowsint-transforms/src/tools/network/httpx.py
@@ -2,17 +2,18 @@ import json
from typing import Any, List
from ..dockertool import DockerTool
+
class HttpxTool(DockerTool):
image = "projectdiscovery/httpx"
default_tag = "latest"
-
+
def __init__(self):
super().__init__(self.image, self.default_tag)
-
+
@classmethod
def name(cls) -> str:
return "httpx"
-
+
@classmethod
def description(cls) -> str:
return "An HTTP toolkit that probes services, web servers, and other valuable metadata."
@@ -31,17 +32,17 @@ class HttpxTool(DockerTool):
command="--version",
remove=True,
stderr=True,
- stdout=True
+ stdout=True,
)
output_str = output.decode()
import re
- match = re.search(r'(v[\d\.]+)', output_str)
+
+ match = re.search(r"(v[\d\.]+)", output_str)
version = match.group(1) if match else "unknown"
return version
except Exception as e:
return f"unknown (error: {str(e)})"
-
-
+
def update(self) -> None:
# Pull the latest image
self.install()
@@ -49,21 +50,21 @@ class HttpxTool(DockerTool):
def is_installed(self) -> bool:
return super().is_installed()
- def launch(self, target: str, args: List[str]| None = None) -> Any:
+ def launch(self, target: str, args: List[str] | None = None) -> Any:
if args is None:
args = []
- args_str = ' '.join(args) if args else ''
+ args_str = " ".join(args) if args else ""
command = f"-u {target} {args_str} -json -silent"
result = super().launch(command)
-
+
# Handle empty result
if not result or result.strip() == "":
return []
-
+
# Handle multiple JSON lines (one per result)
- lines = result.strip().split('\n')
+ lines = result.strip().split("\n")
results = []
-
+
for line in lines:
line = line.strip()
if line:
@@ -72,4 +73,3 @@ class HttpxTool(DockerTool):
except json.JSONDecodeError as e:
raise e
return results
-
diff --git a/flowsint-transforms/src/tools/network/reconcrawl.py b/flowsint-transforms/src/tools/network/reconcrawl.py
index abd35129..8183b4a5 100644
--- a/flowsint-transforms/src/tools/network/reconcrawl.py
+++ b/flowsint-transforms/src/tools/network/reconcrawl.py
@@ -2,15 +2,16 @@ from typing import Any, Dict
from ..base import Tool
+
class ReconCrawlTool(Tool):
-
+
def __init__(self):
super().__init__()
-
+
@classmethod
def name(cls) -> str:
return "reconcrawl"
-
+
@classmethod
def description(cls) -> str:
return "Emails and phone numbers crawler from websites by analyzing their HTML and embedded scripts."
@@ -21,31 +22,34 @@ class ReconCrawlTool(Tool):
def install(self) -> None:
pass
+
def version(self) -> str:
pass
-
def is_installed(self) -> bool:
try:
import reconcrawl
+
return True
except ImportError:
return False
def launch(self, url: str, args: Dict[str, Any] = None) -> Any:
from reconcrawl import Crawler
+
crawler = Crawler(
- url=str(url),
- max_pages=args.get("max_pages", 500),
- timeout=args.get("timeout", 30),
- delay=args.get("delay", 1.0),
- verbose=args.get("verbose", False),
- recursive=args.get("recursive", True),
- verify_ssl=args.get("verify_ssl", False) # Default to False for compatibility
- )
+ url=str(url),
+ max_pages=args.get("max_pages", 500),
+ timeout=args.get("timeout", 30),
+ delay=args.get("delay", 1.0),
+ verbose=args.get("verbose", False),
+ recursive=args.get("recursive", True),
+ verify_ssl=args.get(
+ "verify_ssl", False
+ ), # Default to False for compatibility
+ )
crawler.fetch()
crawler.extract_emails()
crawler.extract_phones()
crawl_result = crawler.get_results()
return crawl_result
-
diff --git a/flowsint-transforms/src/tools/network/subfinder.py b/flowsint-transforms/src/tools/network/subfinder.py
index fce9b9e7..2c8c5e81 100644
--- a/flowsint-transforms/src/tools/network/subfinder.py
+++ b/flowsint-transforms/src/tools/network/subfinder.py
@@ -2,17 +2,18 @@ from typing import Any, List
from ..dockertool import DockerTool
from flowsint_core.utils import is_valid_domain
+
class SubfinderTool(DockerTool):
image = "projectdiscovery/subfinder"
default_tag = "latest"
-
+
def __init__(self):
super().__init__(self.image, self.default_tag)
-
+
@classmethod
def name(cls) -> str:
return "subfinder"
-
+
@classmethod
def description(cls) -> str:
return "Fast passive subdomain enumeration tool."
@@ -32,17 +33,17 @@ class SubfinderTool(DockerTool):
command="--version",
remove=True,
stderr=True,
- stdout=True
+ stdout=True,
)
output_str = output.decode()
import re
- match = re.search(r'(v[\d\.]+)', output_str)
+
+ match = re.search(r"(v[\d\.]+)", output_str)
version = match.group(1) if match else "unknown"
return version
except Exception as e:
return f"unknown (error: {str(e)})"
-
-
+
def update(self) -> None:
# Pull the latest image
self.install()
@@ -55,9 +56,13 @@ class SubfinderTool(DockerTool):
if args is None:
args = []
command = f"-d {domain} {' '.join(args)}"
- result = super().launch(command)
+ result = super().launch(command)
for sub in result.split("\n"):
- if is_valid_domain(sub) and sub.endswith(domain) and sub != domain and not sub.startswith("."):
+ if (
+ is_valid_domain(sub)
+ and sub.endswith(domain)
+ and sub != domain
+ and not sub.startswith(".")
+ ):
subdomains.add(sub)
return list(subdomains)
-
diff --git a/flowsint-transforms/src/tools/organizations/sirene.py b/flowsint-transforms/src/tools/organizations/sirene.py
index fc9bded9..a19ad8bb 100644
--- a/flowsint-transforms/src/tools/organizations/sirene.py
+++ b/flowsint-transforms/src/tools/organizations/sirene.py
@@ -3,16 +3,17 @@ from typing import Dict
import requests
from ..base import Tool
+
class SireneTool(Tool):
-
+
@classmethod
def name(cls) -> str:
return "sirene"
-
+
@classmethod
def version(cls) -> str:
return "1.0.0"
-
+
@classmethod
def description(cls) -> str:
return "The Sirene API allows you to query the Sirene directory of businesses and establishments, managed by Insee."
@@ -21,18 +22,22 @@ class SireneTool(Tool):
def category(cls) -> str:
return "Business intelligence"
- def launch(self, query: str, limit: int=25) -> list[Dict]:
- # Query can be multiple types of entries: full names ("<first> + <last>"), name, geolocation, etc.
+ def launch(self, query: str, limit: int = 25) -> list[Dict]:
+ # Query can be multiple types of entries: full names ("<first> + <last>"), name, geolocation, etc.
try:
query = query.replace(" ", "+")
- params = {"q": query, "per_page": limit }
- resp = requests.get("https://recherche-entreprises.api.gouv.fr/search", params=params, timeout=10)
+ params = {"q": query, "per_page": limit}
+ resp = requests.get(
+ "https://recherche-entreprises.api.gouv.fr/search",
+ params=params,
+ timeout=10,
+ )
resp.raise_for_status()
data = resp.json()
if len(data.get("results")) == 0:
raise ValueError(f"No match found for {query}.")
return data["results"]
except Exception as e:
- raise RuntimeError(f"Error querying Sirene API: {str(e)}. Output: {getattr(e, 'output', 'No output')}")
-
-
+ raise RuntimeError(
+ f"Error querying Sirene API: {str(e)}. Output: {getattr(e, 'output', 'No output')}"
+ )
diff --git a/flowsint-transforms/tests/conftest.py b/flowsint-transforms/tests/conftest.py
index c713514f..be95bac0 100644
--- a/flowsint-transforms/tests/conftest.py
+++ b/flowsint-transforms/tests/conftest.py
@@ -1,9 +1,12 @@
import pytest
from tests.logger import TestLogger
+
@pytest.fixture(autouse=True)
def mock_logger(monkeypatch):
"""Automatically replace the production Logger with TestLogger for all tests."""
monkeypatch.setattr("flowsint_core.core.logger.Logger", TestLogger)
# Mock the emit_event_task to do nothing
- monkeypatch.setattr("flowsint_core.core.logger.emit_event_task.delay", lambda *args, **kwargs: None)
\ No newline at end of file
+ monkeypatch.setattr(
+ "flowsint_core.core.logger.emit_event_task.delay", lambda *args, **kwargs: None
+ )
diff --git a/flowsint-transforms/tests/insert_command.py b/flowsint-transforms/tests/insert_command.py
index 103db359..8e6cc2de 100644
--- a/flowsint-transforms/tests/insert_command.py
+++ b/flowsint-transforms/tests/insert_command.py
@@ -1,11 +1,10 @@
-
import sys
import os
import asyncio
if __name__ == "__main__":
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
-
+
from flowsint_types.domain import Domain
from flowsint_types.ip import Ip
from flowsint_transforms.domains.resolve import ResolveScanner
@@ -14,15 +13,16 @@ from flowsint_transforms.domains.resolve import ResolveScanner
async def main():
# Create test data
domains = [Domain(domain="adaltas.com")]
- ips = [Ip(address='12.23.34.45'), Ip(address='56.67.78.89')]
-
+ ips = [Ip(address="12.23.34.45"), Ip(address="56.67.78.89")]
+
# Test the scanner
scanner = ResolveScanner("sketch_123", "scan_123")
-
+
# Test the new KISS postprocess method
scanner.postprocess(ips[:1], domains) # Only use first IP to match domains length
-
+
print("Postprocess test completed successfully!")
+
if __name__ == "__main__":
- asyncio.run(main())
\ No newline at end of file
+ asyncio.run(main())
diff --git a/flowsint-transforms/tests/logger.py b/flowsint-transforms/tests/logger.py
index 87242d03..b00a2f02 100644
--- a/flowsint-transforms/tests/logger.py
+++ b/flowsint-transforms/tests/logger.py
@@ -5,7 +5,7 @@ EventLevel = Literal["info", "warn", "error", "success", "debug"]
LEVEL_MAP = {
"info": "INFO",
- "warn": "WARN",
+ "warn": "WARN",
"error": "FAILED",
"success": "SUCCESS",
"debug": "DEBUG",
@@ -17,13 +17,15 @@ class TestLogger:
def _format_message(type: str, message: str) -> str:
"""Format the log message with type prefix"""
return f"[{type.upper()}] {message}"
-
+
@staticmethod
def _create_log(sketch_id: Union[str, UUID], log_type: str, content: str) -> Any:
"""Create a dummy log object for testing"""
+
class DummyLog:
def __init__(self):
- self.id = 'dummy_id'
+ self.id = "dummy_id"
+
return DummyLog()
@staticmethod
diff --git a/flowsint-transforms/tests/scanners/crypto/wallet_to_nfts.py b/flowsint-transforms/tests/scanners/crypto/wallet_to_nfts.py
index 4c5864b6..9e7778e6 100644
--- a/flowsint-transforms/tests/scanners/crypto/wallet_to_nfts.py
+++ b/flowsint-transforms/tests/scanners/crypto/wallet_to_nfts.py
@@ -1,17 +1,22 @@
from flowsint_transforms.crypto.wallet_to_nfts import CryptoWalletAddressToNFTs
from flowsint_types.wallet import CryptoWallet, CryptoNFT
from pydantic import HttpUrl
+
scanner = CryptoWalletAddressToNFTs("sketch_123", "scan_123")
+
def test_wallet_address_to_transactions_name():
assert scanner.name() == "wallet_to_nfts"
+
def test_wallet_address_to_transactions_category():
assert scanner.category() == "crypto"
+
def test_wallet_address_to_transactions_key():
assert scanner.key() == "address"
+
def test_preprocess_with_string():
input_data = ["0x742d35Cc6634C0532925a3b844Bc454e4438f44e"]
result = scanner.preprocess(input_data)
@@ -19,6 +24,7 @@ def test_preprocess_with_string():
assert isinstance(result[0], CryptoWallet)
assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"
+
def test_preprocess_with_dict():
input_data = [{"address": "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"}]
result = scanner.preprocess(input_data)
@@ -26,6 +32,7 @@ def test_preprocess_with_dict():
assert isinstance(result[0], CryptoWallet)
assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"
+
def test_preprocess_with_wallet_object():
wallet = CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")
input_data = [wallet]
@@ -34,26 +41,29 @@ def test_preprocess_with_wallet_object():
assert isinstance(result[0], CryptoWallet)
assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"
+
def test_scan_mocked_transactions(monkeypatch):
# Mock the _get_transactions method
def mock_get_nfts(address):
return [
CryptoNFT(
- wallet=CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e"),
+ wallet=CryptoWallet(
+ address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e"
+ ),
contract_address="0x123",
token_id="1",
collection_name="Test Collection",
metadata_url="https://example.com/metadata.json",
image_url="https://example.com/image.png",
- name="Test NFT"
+ name="Test NFT",
)
]
-
+
monkeypatch.setattr(scanner, "_get_nfts", mock_get_nfts)
-
+
input_data = [CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")]
result = scanner.scan(input_data)
-
+
assert len(result) == 1
assert len(result[0]) == 1
assert result[0][0].contract_address == "0x123"
@@ -61,4 +71,3 @@ def test_scan_mocked_transactions(monkeypatch):
assert result[0][0].metadata_url == HttpUrl("https://example.com/metadata.json")
assert result[0][0].image_url == HttpUrl("https://example.com/image.png")
assert result[0][0].name == "Test NFT"
-
\ No newline at end of file
diff --git a/flowsint-transforms/tests/scanners/crypto/wallet_to_transactions.py b/flowsint-transforms/tests/scanners/crypto/wallet_to_transactions.py
index 50475221..af0deb74 100644
--- a/flowsint-transforms/tests/scanners/crypto/wallet_to_transactions.py
+++ b/flowsint-transforms/tests/scanners/crypto/wallet_to_transactions.py
@@ -1,18 +1,28 @@
import pytest
-from flowsint_transforms.crypto.wallet_to_transactions import CryptoWalletAddressToTransactions
+from flowsint_transforms.crypto.wallet_to_transactions import (
+ CryptoWalletAddressToTransactions,
+)
from flowsint_types.wallet import CryptoWallet, CryptoWalletTransaction
-scanner = CryptoWalletAddressToTransactions("sketch_123", "scan_123", params={"ETHERSCAN_API_KEY": "ta-clef-api"},)
+scanner = CryptoWalletAddressToTransactions(
+ "sketch_123",
+ "scan_123",
+ params={"ETHERSCAN_API_KEY": "ta-clef-api"},
+)
+
def test_wallet_address_to_transactions_name():
assert scanner.name() == "wallet_to_transactions"
+
def test_wallet_address_to_transactions_category():
assert scanner.category() == "CryptoCryptoWallet"
+
def test_wallet_address_to_transactions_key():
assert scanner.key() == "address"
+
def test_preprocess_with_string():
input_data = ["0x742d35Cc6634C0532925a3b844Bc454e4438f44e"]
result = scanner.preprocess(input_data)
@@ -20,6 +30,7 @@ def test_preprocess_with_string():
assert isinstance(result[0], CryptoWallet)
assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"
+
def test_preprocess_with_dict():
input_data = [{"address": "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"}]
result = scanner.preprocess(input_data)
@@ -27,6 +38,7 @@ def test_preprocess_with_dict():
assert isinstance(result[0], CryptoWallet)
assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"
+
def test_preprocess_with_wallet_object():
wallet = CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")
input_data = [wallet]
@@ -35,6 +47,7 @@ def test_preprocess_with_wallet_object():
assert isinstance(result[0], CryptoWallet)
assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"
+
@pytest.mark.asyncio
async def test_scan_mocked_transactions(monkeypatch):
# Mock the _get_transactions method - note it takes address and api_key parameters
@@ -42,7 +55,9 @@ async def test_scan_mocked_transactions(monkeypatch):
return [
CryptoWalletTransaction(
hash="0x123",
- source=CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e"),
+ source=CryptoWallet(
+ address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e"
+ ),
target=CryptoWallet(address="0x456"),
value=1.0, # 1 ETH
timestamp="1234567890",
@@ -55,15 +70,15 @@ async def test_scan_mocked_transactions(monkeypatch):
gas_used="21000",
cumulative_gas_used="21000",
input="0x",
- contract_address=None
+ contract_address=None,
)
]
-
+
monkeypatch.setattr(scanner, "_get_transactions", mock_get_transactions)
-
+
input_data = [CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")]
result = await scanner.scan(input_data)
-
+
assert len(result) == 1
assert len(result[0]) == 1
assert result[0][0].hash == "0x123"
@@ -72,13 +87,20 @@ async def test_scan_mocked_transactions(monkeypatch):
assert result[0][0].value == 1.0
assert result[0][0].timestamp == "1234567890"
+
def test_scanner_requires_api_key():
"""Test that the scanner validates required ETHERSCAN_API_KEY parameter at construction"""
- with pytest.raises(ValueError, match="Scanner wallet_to_transactions received invalid params"):
+ with pytest.raises(
+ ValueError, match="Scanner wallet_to_transactions received invalid params"
+ ):
CryptoWalletAddressToTransactions("sketch_123", "scan_123", params={})
+
def test_scanner_with_invalid_api_key_type():
"""Test that the scanner validates parameter types"""
- with pytest.raises(ValueError, match="Scanner wallet_to_transactions received invalid params"):
- CryptoWalletAddressToTransactions("sketch_123", "scan_123", params={"ETHERSCAN_API_KEY": 123})
-
\ No newline at end of file
+ with pytest.raises(
+ ValueError, match="Scanner wallet_to_transactions received invalid params"
+ ):
+ CryptoWalletAddressToTransactions(
+ "sketch_123", "scan_123", params={"ETHERSCAN_API_KEY": 123}
+ )
diff --git a/flowsint-transforms/tests/scanners/domains/resolve.py b/flowsint-transforms/tests/scanners/domains/resolve.py
index bccf4c69..de72d91a 100644
--- a/flowsint-transforms/tests/scanners/domains/resolve.py
+++ b/flowsint-transforms/tests/scanners/domains/resolve.py
@@ -6,18 +6,20 @@ import pytest
scanner = ResolveScanner("sketch_123", "scan_123")
+
def test_preprocess_valid_domains():
domains = [
Domain(domain="example.com"),
Domain(domain="example2.com"),
]
result = scanner.preprocess(domains)
-
+
result_domains = [d.domain for d in result]
expected_domains = [d.domain for d in domains]
assert result_domains == expected_domains
-
+
+
def test_unprocessed_valid_domains():
domains = [
"example.com",
@@ -26,8 +28,9 @@ def test_unprocessed_valid_domains():
result = scanner.preprocess(domains)
result_domains = [d for d in result]
expected_domains = [Domain(domain=d) for d in domains]
- assert result_domains == expected_domains
-
+ assert result_domains == expected_domains
+
+
def test_preprocess_invalid_domains():
domains = [
Domain(domain="example.com"),
@@ -41,6 +44,7 @@ def test_preprocess_invalid_domains():
assert "example.org" in result_domains
assert "invalid_domain" not in result_domains
+
def test_preprocess_multiple_formats():
domains = [
{"domain": "example.com"},
@@ -56,6 +60,7 @@ def test_preprocess_multiple_formats():
assert "invalid_domain" not in result_domains
assert "example.io" not in result_domains
+
@pytest.mark.asyncio
async def test_scan_returns_ip(monkeypatch):
# on crée une fonction mock qui retourne une IP
@@ -70,78 +75,79 @@ async def test_scan_returns_ip(monkeypatch):
assert isinstance(output, list)
assert output[0].address == "12.23.34.45"
+
def test_schemas():
input_schema = scanner.input_schema()
output_schema = scanner.output_schema()
-
+
# Test the structure and key properties rather than exact match
- assert input_schema['type'] == 'Domain'
- assert isinstance(input_schema['properties'], list)
- input_property_names = [prop['name'] for prop in input_schema['properties']]
- assert 'domain' in input_property_names
-
- assert output_schema['type'] == 'Ip'
- assert isinstance(output_schema['properties'], list)
- output_property_names = [prop['name'] for prop in output_schema['properties']]
- assert 'address' in output_property_names
+ assert input_schema["type"] == "Domain"
+ assert isinstance(input_schema["properties"], list)
+ input_property_names = [prop["name"] for prop in input_schema["properties"]]
+ assert "domain" in input_property_names
+
+ assert output_schema["type"] == "Ip"
+ assert isinstance(output_schema["properties"], list)
+ output_property_names = [prop["name"] for prop in output_schema["properties"]]
+ assert "address" in output_property_names
class TestResolveInputOutputTypes:
"""Test the InputType/OutputType functionality for ResolveScanner"""
-
+
def test_input_output_types_are_defined(self):
"""Test that InputType and OutputType are properly defined"""
- assert hasattr(ResolveScanner, 'InputType')
- assert hasattr(ResolveScanner, 'OutputType')
+ assert hasattr(ResolveScanner, "InputType")
+ assert hasattr(ResolveScanner, "OutputType")
assert ResolveScanner.InputType == List[Domain]
assert ResolveScanner.OutputType == List[Ip]
-
+
def test_schemas_use_generate_methods(self):
"""Test that schema methods use the new generate methods"""
# These should work without error
input_schema = ResolveScanner.generate_input_schema()
output_schema = ResolveScanner.generate_output_schema()
-
+
assert isinstance(input_schema, dict)
assert isinstance(output_schema, dict)
assert input_schema["type"] == "Domain"
assert output_schema["type"] == "Ip"
-
+
def test_schema_methods_return_same_as_generate_methods(self):
"""Test that input_schema() and output_schema() return the same as generate methods"""
assert ResolveScanner.input_schema() == ResolveScanner.generate_input_schema()
assert ResolveScanner.output_schema() == ResolveScanner.generate_output_schema()
-
+
def test_input_schema_properties(self):
"""Test input schema has expected properties"""
schema = ResolveScanner.input_schema()
-
+
properties = schema["properties"]
property_names = [p["name"] for p in properties]
-
+
# Domain should have these properties
assert "domain" in property_names
-
+
def test_output_schema_properties(self):
"""Test output schema has expected properties"""
schema = ResolveScanner.output_schema()
-
+
properties = schema["properties"]
property_names = [p["name"] for p in properties]
-
+
# Ip should have these properties
assert "address" in property_names
-
+
def test_type_accessibility_from_instance(self):
"""Test that types are accessible from scanner instance"""
scanner_instance = ResolveScanner("test", "test")
-
+
assert scanner_instance.InputType == List[Domain]
assert scanner_instance.OutputType == List[Ip]
-
+
# Should be able to generate schemas from instance
input_schema = scanner_instance.generate_input_schema()
output_schema = scanner_instance.generate_output_schema()
-
+
assert input_schema["type"] == "Domain"
assert output_schema["type"] == "Ip"
diff --git a/flowsint-transforms/tests/scanners/domains/subdomains.py b/flowsint-transforms/tests/scanners/domains/subdomains.py
index 65712f43..d323721c 100644
--- a/flowsint-transforms/tests/scanners/domains/subdomains.py
+++ b/flowsint-transforms/tests/scanners/domains/subdomains.py
@@ -3,18 +3,20 @@ from flowsint_types.domain import Domain, Domain
scanner = SubdomainScanner("sketch_123", "scan_123")
+
def test_preprocess_valid_domains():
domains = [
Domain(domain="example.com"),
Domain(domain="example2.com"),
]
result = scanner.preprocess(domains)
-
+
result_domains = [d.domain for d in result]
expected_domains = [d.domain for d in domains]
assert result_domains == expected_domains
-
+
+
def test_unprocessed_valid_domains():
domains = [
"example.com",
@@ -23,8 +25,9 @@ def test_unprocessed_valid_domains():
result = scanner.preprocess(domains)
result_domains = [d for d in result]
expected_domains = [Domain(domain=d) for d in domains]
- assert result_domains == expected_domains
-
+ assert result_domains == expected_domains
+
+
def test_preprocess_invalid_domains():
domains = [
Domain(domain="example.com"),
@@ -38,6 +41,7 @@ def test_preprocess_invalid_domains():
assert "example.org" in result_domains
assert "invalid_domain" not in result_domains
+
def test_preprocess_multiple_formats():
domains = [
{"domain": "example.com"},
@@ -83,13 +87,9 @@ def test_scan_extracts_subdomains(monkeypatch):
input_data = [Domain(domain="example.com")]
domains = scanner.execute(input_data)
assert isinstance(domains, list)
- for sub in domains:
+ for sub in domains:
print(sub)
assert isinstance(sub, Domain)
- expected = sorted([
- "mail.example.com",
- "www.example.com",
- "api.example.com"
- ])
+ expected = sorted(["mail.example.com", "www.example.com", "api.example.com"])
print(domains)
# assert domains[0].subdomains == expected
diff --git a/flowsint-transforms/tests/scanners/domains/whois.py b/flowsint-transforms/tests/scanners/domains/whois.py
index 8146dfc4..c03eac81 100644
--- a/flowsint-transforms/tests/scanners/domains/whois.py
+++ b/flowsint-transforms/tests/scanners/domains/whois.py
@@ -3,18 +3,20 @@ from flowsint_types.domain import Domain
scanner = WhoisScanner("sketch_123", "scan_123")
+
def test_preprocess_valid_domains():
domains = [
Domain(domain="example.com"),
Domain(domain="example2.com"),
]
result = scanner.preprocess(domains)
-
+
result_domains = [d.domain for d in result]
expected_domains = [d.domain for d in domains]
assert result_domains == expected_domains
-
+
+
def test_unprocessed_valid_domains():
domains = [
"example.com",
@@ -23,8 +25,9 @@ def test_unprocessed_valid_domains():
result = scanner.preprocess(domains)
result_domains = [d for d in result]
expected_domains = [Domain(domain=d) for d in domains]
- assert result_domains == expected_domains
-
+ assert result_domains == expected_domains
+
+
def test_preprocess_invalid_domains():
domains = [
Domain(domain="example.com"),
@@ -38,6 +41,7 @@ def test_preprocess_invalid_domains():
assert "example.org" in result_domains
assert "invalid_domain" not in result_domains
+
def test_preprocess_multiple_formats():
domains = [
{"domain": "example.com"},
@@ -53,6 +57,7 @@ def test_preprocess_multiple_formats():
assert "invalid_domain" not in result_domains
assert "example.io" not in result_domains
+
def test_scan_returns_whois_objects(monkeypatch):
# Patch `whois.whois` to avoid real network call
mock_whois = lambda domain: {
@@ -62,7 +67,7 @@ def test_scan_returns_whois_objects(monkeypatch):
"country": "MockCountry",
"emails": ["[email protected]"],
"creation_date": "2020-01-01",
- "expiration_date": "2030-01-01"
+ "expiration_date": "2030-01-01",
}
monkeypatch.setattr("whois.whois", mock_whois)
@@ -74,8 +79,25 @@ def test_scan_returns_whois_objects(monkeypatch):
assert output[0].whois.org == "MockOrg"
assert output[0].whois.email.email == "[email protected]"
+
def test_schemas():
input_schema = scanner.input_schema()
output_schema = scanner.output_schema()
- assert input_schema == {'type': 'Domain', 'properties': [{'name': 'domain', 'type': 'string'}, {'name': 'subdomains', 'type': 'array | null'}, {'name': 'ips', 'type': 'array | null'}, {'name': 'whois', 'type': 'Whois | null'}]}
- assert output_schema == {'type': 'Domain', 'properties': [{'name': 'domain', 'type': 'string'}, {'name': 'subdomains', 'type': 'array | null'}, {'name': 'ips', 'type': 'array | null'}, {'name': 'whois', 'type': 'Whois | null'}]}
+ assert input_schema == {
+ "type": "Domain",
+ "properties": [
+ {"name": "domain", "type": "string"},
+ {"name": "subdomains", "type": "array | null"},
+ {"name": "ips", "type": "array | null"},
+ {"name": "whois", "type": "Whois | null"},
+ ],
+ }
+ assert output_schema == {
+ "type": "Domain",
+ "properties": [
+ {"name": "domain", "type": "string"},
+ {"name": "subdomains", "type": "array | null"},
+ {"name": "ips", "type": "array | null"},
+ {"name": "whois", "type": "Whois | null"},
+ ],
+ }
diff --git a/flowsint-transforms/tests/scanners/emails/to_gravatar.py b/flowsint-transforms/tests/scanners/emails/to_gravatar.py
index 13824bcc..871b5b4f 100644
--- a/flowsint-transforms/tests/scanners/emails/to_gravatar.py
+++ b/flowsint-transforms/tests/scanners/emails/to_gravatar.py
@@ -28,7 +28,9 @@ class TestEmailToGravatarScanner:
assert schema["type"] == "Email"
assert "properties" in schema
# Check that email property is present
- email_prop = next((prop for prop in schema["properties"] if prop["name"] == "email"), None)
+ email_prop = next(
+ (prop for prop in schema["properties"] if prop["name"] == "email"), None
+ )
assert email_prop is not None
assert email_prop["type"] == "string"
@@ -38,8 +40,12 @@ class TestEmailToGravatarScanner:
assert schema["type"] == "Gravatar"
assert "properties" in schema
# Check that required properties are present
- src_prop = next((prop for prop in schema["properties"] if prop["name"] == "src"), None)
- hash_prop = next((prop for prop in schema["properties"] if prop["name"] == "hash"), None)
+ src_prop = next(
+ (prop for prop in schema["properties"] if prop["name"] == "src"), None
+ )
+ hash_prop = next(
+ (prop for prop in schema["properties"] if prop["name"] == "hash"), None
+ )
assert src_prop is not None
assert hash_prop is not None
@@ -114,7 +120,7 @@ class TestEmailToGravatarScanner:
result = scanner.preprocess([])
assert result == []
- @patch('requests.get')
+ @patch("requests.get")
def test_scan_successful_gravatar(self, mock_get):
"""Test successful gravatar retrieval"""
# Mock successful response
@@ -130,7 +136,7 @@ class TestEmailToGravatarScanner:
assert result[0].hash == hashlib.md5("[email protected]".encode()).hexdigest()
assert "gravatar.com/avatar/" in str(result[0].src)
- @patch('requests.get')
+ @patch("requests.get")
def test_scan_failed_request(self, mock_get):
"""Test handling of failed HTTP requests"""
# Mock failed response
@@ -143,7 +149,7 @@ class TestEmailToGravatarScanner:
assert len(result) == 0
- @patch('requests.get')
+ @patch("requests.get")
def test_scan_request_exception(self, mock_get):
"""Test handling of request exceptions"""
# Mock exception
@@ -154,7 +160,7 @@ class TestEmailToGravatarScanner:
assert len(result) == 0
- @patch('requests.get')
+ @patch("requests.get")
def test_scan_multiple_emails(self, mock_get):
"""Test scanning multiple emails"""
# Mock successful responses
@@ -173,9 +179,10 @@ class TestEmailToGravatarScanner:
assert all(isinstance(gravatar, Gravatar) for gravatar in result)
assert mock_get.call_count == 3
- @patch('requests.get')
+ @patch("requests.get")
def test_scan_mixed_success_failure(self, mock_get):
"""Test scanning with mixed success and failure"""
+
# Mock mixed responses - check the actual URL being called
def side_effect(url, *args, **kwargs):
mock_response = Mock()
@@ -203,7 +210,9 @@ class TestEmailToGravatarScanner:
"""Test postprocessing with Neo4j connection"""
# Mock Neo4j connection
mock_neo4j = Mock()
- scanner_with_neo4j = EmailToGravatarScanner("sketch_123", "scan_123", neo4j_conn=mock_neo4j)
+ scanner_with_neo4j = EmailToGravatarScanner(
+ "sketch_123", "scan_123", neo4j_conn=mock_neo4j
+ )
gravatars = [
Gravatar(src="https://www.gravatar.com/avatar/hash1", hash="hash1"),
@@ -218,7 +227,7 @@ class TestEmailToGravatarScanner:
# Verify Neo4j queries were executed
assert mock_neo4j.query.call_count == 2
-
+
# Check that results are returned unchanged
assert result == gravatars
@@ -261,7 +270,7 @@ class TestEmailToGravatarScanner:
def test_execute_full_workflow(self):
"""Test the complete execute workflow"""
- with patch('requests.get') as mock_get:
+ with patch("requests.get") as mock_get:
# Mock successful response
mock_response = Mock()
mock_response.status_code = 200
@@ -272,18 +281,20 @@ class TestEmailToGravatarScanner:
assert len(result) == 1
assert isinstance(result[0], Gravatar)
- assert result[0].hash == hashlib.md5("[email protected]".encode()).hexdigest()
+ assert (
+ result[0].hash == hashlib.md5("[email protected]".encode()).hexdigest()
+ )
def test_execute_with_invalid_input(self):
"""Test execute with invalid input"""
emails = ["not-an-email", "also-invalid"]
-
- with patch('requests.get') as mock_get:
+
+ with patch("requests.get") as mock_get:
# Mock successful response for any request
mock_response = Mock()
mock_response.status_code = 200
mock_get.return_value = mock_response
-
+
result = scanner.execute(emails)
# The scanner processes any string as an email, so it will create Email objects
@@ -295,8 +306,8 @@ class TestEmailToGravatarScanner:
"""Test that gravatar hash is calculated correctly"""
expected_hash = hashlib.md5(email.encode()).hexdigest()
-
- with patch('requests.get') as mock_get:
+
+ with patch("requests.get") as mock_get:
mock_response = Mock()
mock_response.status_code = 200
mock_get.return_value = mock_response
@@ -312,8 +323,8 @@ class TestEmailToGravatarScanner:
expected_hash = hashlib.md5(email.encode()).hexdigest()
expected_url = f"https://www.gravatar.com/avatar/{expected_hash}"
-
- with patch('requests.get') as mock_get:
+
+ with patch("requests.get") as mock_get:
mock_response = Mock()
mock_response.status_code = 200
mock_get.return_value = mock_response
@@ -322,4 +333,4 @@ class TestEmailToGravatarScanner:
result = scanner.scan(emails)
assert len(result) == 1
- assert str(result[0].src) == expected_url
\ No newline at end of file
+ assert str(result[0].src) == expected_url
diff --git a/flowsint-transforms/tests/scanners/emails/to_leaks.py b/flowsint-transforms/tests/scanners/emails/to_leaks.py
index ce099e18..0880682e 100644
--- a/flowsint-transforms/tests/scanners/emails/to_leaks.py
+++ b/flowsint-transforms/tests/scanners/emails/to_leaks.py
@@ -6,15 +6,19 @@ from flowsint_types.breach import Breach
scanner = EmailToBreachesScanner("sketch_123", "scan_123")
+
def test_scanner_name():
assert EmailToBreachesScanner.name() == "to_leaks"
+
def test_scanner_category():
assert EmailToBreachesScanner.category() == "Email"
+
def test_scanner_key():
assert EmailToBreachesScanner.key() == "email"
+
def test_preprocess_string_emails():
emails = [
@@ -24,6 +28,7 @@ def test_preprocess_string_emails():
expected_emails = [Email(email=email) for email in emails]
assert result == expected_emails
+
def test_preprocess_dict_emails():
emails = [
{"email": "[email protected]"},
@@ -33,6 +38,7 @@ def test_preprocess_dict_emails():
expected_emails = [Email(email=email["email"]) for email in emails]
assert result == expected_emails
+
def test_preprocess_email_objects():
emails = [
Email(email="[email protected]"),
@@ -41,6 +47,7 @@ def test_preprocess_email_objects():
result = scanner.preprocess(emails)
assert result == emails
+
def test_preprocess_mixed_formats():
emails = [
@@ -49,27 +56,28 @@ def test_preprocess_mixed_formats():
{"invalid_key": "[email protected]"},
]
result = scanner.preprocess(emails)
-
+
result_emails = [email.email for email in result]
assert "[email protected]" in result_emails
assert "[email protected]" in result_emails
assert "[email protected]" in result_emails
assert "[email protected]" not in result_emails
-@patch('src.transforms.emails.to_leaks.requests.get')
+
+@patch("src.transforms.emails.to_leaks.requests.get")
def test_scan_successful_response(mock_get):
# Mock successful API response
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = [
{"Name": "TestBreach", "Title": "Test Breach", "Domain": "test.com"},
- {"Name": "AnotherBreach", "Title": "Another Breach", "Domain": "another.com"}
+ {"Name": "AnotherBreach", "Title": "Another Breach", "Domain": "another.com"},
]
mock_get.return_value = mock_response
-
+
emails = [Email(email="[email protected]")]
result = scanner.scan(emails)
-
+
assert len(result) == 2
assert isinstance(result[0], Breach)
assert isinstance(result[1], Breach)
@@ -78,85 +86,90 @@ def test_scan_successful_response(mock_get):
assert result[0].breach["name"] == "testbreach"
assert result[1].breach["name"] == "anotherbreach"
-@patch('src.transforms.emails.to_leaks.requests.get')
+
+@patch("src.transforms.emails.to_leaks.requests.get")
def test_scan_no_breaches_found(mock_get):
# Mock 404 response (no breaches found)
mock_response = MagicMock()
mock_response.status_code = 404
mock_get.return_value = mock_response
-
+
emails = [Email(email="[email protected]")]
result = scanner.scan(emails)
-
+
assert len(result) == 0
-@patch('src.transforms.emails.to_leaks.requests.get')
+
+@patch("src.transforms.emails.to_leaks.requests.get")
def test_scan_api_error(mock_get):
# Mock API error
mock_get.side_effect = Exception("API Error")
-
+
emails = [Email(email="[email protected]")]
result = scanner.scan(emails)
-
+
assert len(result) == 0
-@patch('src.transforms.emails.to_leaks.requests.get')
+
+@patch("src.transforms.emails.to_leaks.requests.get")
def test_scan_missing_name_field(mock_get):
# Mock API response with missing "Name" field
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = [
{"Title": "Test Breach", "Domain": "test.com"}, # Missing "Name" field
- {"Name": "ValidBreach", "Title": "Valid Breach", "Domain": "valid.com"}
+ {"Name": "ValidBreach", "Title": "Valid Breach", "Domain": "valid.com"},
]
mock_get.return_value = mock_response
-
+
emails = [Email(email="[email protected]")]
result = scanner.scan(emails)
-
+
assert len(result) == 2
assert result[0].name == "unknown" # Should default to "unknown"
assert result[1].name == "validbreach" # Should use the provided name
assert result[0].breach["title"] == "Test Breach"
assert result[1].breach["name"] == "validbreach"
-@patch('src.transforms.emails.to_leaks.HIBP_API_KEY', None)
+
+@patch("src.transforms.emails.to_leaks.HIBP_API_KEY", None)
def test_scan_no_api_key():
"""Test that scanner raises ValueError when HIBP_API_KEY is not set."""
emails = [Email(email="[email protected]")]
-
+
with pytest.raises(ValueError, match="HIBP_API_KEY not set"):
scanner.scan(emails)
+
def test_postprocess():
# Test postprocess method with mocked neo4j connection
scanner.neo4j_conn = MagicMock()
-
+
# Create breach objects with the new structure
breach1 = Breach(
name="testbreach",
title="Test Breach",
domain="test.com",
pwncount=1000,
- breach={"name": "testbreach", "title": "Test Breach"}
+ breach={"name": "testbreach", "title": "Test Breach"},
)
breach2 = Breach(
- name="anotherbreach",
+ name="anotherbreach",
title="Another Breach",
domain="another.com",
pwncount=2000,
- breach={"name": "anotherbreach", "title": "Another Breach"}
+ breach={"name": "anotherbreach", "title": "Another Breach"},
)
-
+
breaches = [breach1, breach2]
original_input = [Email(email="[email protected]")]
-
+
result = scanner.postprocess(breaches, original_input)
-
+
assert result == breaches
# Verify that neo4j queries were called:
# - 2 breach node creation queries
- # - 1 email node creation query
+ # - 1 email node creation query
# - 2 relationship creation queries
# Total: 5 queries
- assert scanner.neo4j_conn.query.call_count == 5
\ No newline at end of file
+ assert scanner.neo4j_conn.query.call_count == 5
diff --git a/flowsint-transforms/tests/scanners/ips/asn_to_cidrs.py b/flowsint-transforms/tests/scanners/ips/asn_to_cidrs.py
index 234ae660..fc40c860 100644
--- a/flowsint-transforms/tests/scanners/ips/asn_to_cidrs.py
+++ b/flowsint-transforms/tests/scanners/ips/asn_to_cidrs.py
@@ -3,18 +3,20 @@ from flowsint_types.asn import ASN
scanner = AsnToCidrsScanner("sketch_123", "scan_123")
+
def test_preprocess_valid_asns():
asns = [
ASN(number=15169),
ASN(number=13335),
]
result = scanner.preprocess(asns)
-
+
result_numbers = [asn.number for asn in result]
expected_numbers = [asn.number for asn in asns]
assert result_numbers == expected_numbers
-
+
+
def test_unprocessed_valid_asns():
asns = [
"15169",
@@ -23,8 +25,9 @@ def test_unprocessed_valid_asns():
result = scanner.preprocess(asns)
result_asns = [asn for asn in result]
expected_asns = [ASN(number=int(asn)) for asn in asns]
- assert result_asns == expected_asns
-
+ assert result_asns == expected_asns
+
+
def test_preprocess_invalid_asns():
asns = [
ASN(number=15169),
@@ -38,6 +41,7 @@ def test_preprocess_invalid_asns():
assert 13335 in result_numbers
assert 999999999999 not in result_numbers
+
def test_preprocess_multiple_formats():
asns = [
{"number": 15169},
@@ -50,19 +54,23 @@ def test_preprocess_multiple_formats():
result_numbers = [asn.number for asn in result]
assert 15169 in result_numbers
assert 13335 in result_numbers
- assert "invalid_key" not in result_numbers # Should be filtered out due to invalid key
+ assert (
+ "invalid_key" not in result_numbers
+ ) # Should be filtered out due to invalid key
def test_schemas():
input_schema = scanner.input_schema()
output_schema = scanner.output_schema()
-
+
# Input schema should have number field
assert "properties" in input_schema
- number_prop = next((prop for prop in input_schema["properties"] if prop["name"] == "number"), None)
+ number_prop = next(
+ (prop for prop in input_schema["properties"] if prop["name"] == "number"), None
+ )
assert number_prop is not None
assert number_prop["type"] == "integer"
-
+
# Output schema should have network field
assert "properties" in output_schema
prop_names = [prop["name"] for prop in output_schema["properties"]]
diff --git a/flowsint-transforms/tests/scanners/ips/cidr_to_ips.py b/flowsint-transforms/tests/scanners/ips/cidr_to_ips.py
index 76a0f036..0803c9da 100644
--- a/flowsint-transforms/tests/scanners/ips/cidr_to_ips.py
+++ b/flowsint-transforms/tests/scanners/ips/cidr_to_ips.py
@@ -6,18 +6,20 @@ from tests.logger import TestLogger
logger = TestLogger()
scanner = CidrToIpsScanner("sketch_123", "scan_123", logger)
+
def test_preprocess_valid_cidrs():
cidrs = [
CIDR(network="8.8.8.0/24"),
CIDR(network="1.1.1.0/24"),
]
result = scanner.preprocess(cidrs)
-
+
result_networks = [cidr.network for cidr in result]
expected_networks = [cidr.network for cidr in cidrs]
assert result_networks == expected_networks
+
def test_preprocess_unprocessed_valid_cidrs():
cidrs = [
"8.8.8.0/24",
@@ -28,6 +30,7 @@ def test_preprocess_unprocessed_valid_cidrs():
expected_cidrs = [CIDR(network=c) for c in cidrs]
assert result_cidrs == expected_cidrs
+
def test_preprocess_invalid_cidrs():
cidrs = [
CIDR(network="8.8.8.0/24"),
@@ -40,6 +43,7 @@ def test_preprocess_invalid_cidrs():
assert "invalid-cidr" not in result_networks
assert "not-a-cidr" not in result_networks
+
def test_preprocess_multiple_formats():
cidrs = [
{"network": "8.8.8.0/24"},
@@ -54,6 +58,7 @@ def test_preprocess_multiple_formats():
assert "1.1.1.0/24" not in result_networks
assert "InvalidCIDR" not in result_networks
+
def test_scan_extracts_ips(monkeypatch):
mock_dnsx_output = """8.35.200.12
8.35.200.112
@@ -75,21 +80,17 @@ def test_scan_extracts_ips(monkeypatch):
input_data = [CIDR(network="8.35.200.0/24")]
ips = scanner.scan(input_data)
-
+
assert isinstance(ips, list)
assert len(ips) == 4
-
- expected_ips = [
- "8.35.200.12",
- "8.35.200.112",
- "8.35.200.16",
- "8.35.200.170"
- ]
-
+
+ expected_ips = ["8.35.200.12", "8.35.200.112", "8.35.200.16", "8.35.200.170"]
+
for ip in ips:
assert isinstance(ip, Ip)
assert ip.address in expected_ips
+
def test_scan_handles_empty_output(monkeypatch):
class MockSubprocessResult:
def __init__(self):
@@ -103,10 +104,11 @@ def test_scan_handles_empty_output(monkeypatch):
input_data = [CIDR(network="8.8.8.0/24")]
ips = scanner.scan(input_data)
-
+
assert isinstance(ips, list)
assert len(ips) == 0
+
def test_scan_handles_subprocess_exception(monkeypatch):
def mock_subprocess_run(cmd, shell, capture_output, text, timeout):
raise Exception("Subprocess failed")
@@ -115,6 +117,6 @@ def test_scan_handles_subprocess_exception(monkeypatch):
input_data = [CIDR(network="8.8.8.0/24")]
ips = scanner.scan(input_data)
-
+
assert isinstance(ips, list)
- assert len(ips) == 0
\ No newline at end of file
+ assert len(ips) == 0
diff --git a/flowsint-transforms/tests/scanners/ips/geolocation.py b/flowsint-transforms/tests/scanners/ips/geolocation.py
index 378beba8..607ef8b6 100644
--- a/flowsint-transforms/tests/scanners/ips/geolocation.py
+++ b/flowsint-transforms/tests/scanners/ips/geolocation.py
@@ -3,6 +3,7 @@ from flowsint_types.ip import Ip, Ip
scanner = GeolocationScanner("sketch_123", "scan_123")
+
def test_preprocess_valid_ips():
ips = [
Ip(address="8.8.8.8"),
@@ -13,6 +14,7 @@ def test_preprocess_valid_ips():
expected_ips = [d.address for d in ips]
assert result_ips == expected_ips
+
def test_preprocess_string_ips():
ips = [
"8.8.8.8",
@@ -23,6 +25,7 @@ def test_preprocess_string_ips():
expected_ips = [d for d in ips]
assert [ip.address for ip in result] == expected_ips
+
def test_preprocess_invalid_ips():
ips = [
Ip(address="8.8.8.8"),
@@ -35,6 +38,7 @@ def test_preprocess_invalid_ips():
assert "1.1.1.1" in result_ips
assert "invalid_ip" not in result_ips
+
def test_preprocess_multiple_formats():
ips = [
{"address": "8.8.8.8"},
@@ -48,6 +52,7 @@ def test_preprocess_multiple_formats():
assert "1.1.1.1" in result_ips
assert "1.2.3.4" not in result_ips
+
def test_scan_returns_ip(monkeypatch):
# Mock of get_location_data
def mock_get_location_data(address):
@@ -56,7 +61,7 @@ def test_scan_returns_ip(monkeypatch):
"longitude": -122.0838,
"country": "US",
"city": "Mountain View",
- "isp": "Google LLC"
+ "isp": "Google LLC",
}
monkeypatch.setattr(scanner, "get_location_data", mock_get_location_data)
@@ -70,9 +75,29 @@ def test_scan_returns_ip(monkeypatch):
assert output[0].country == "US"
assert output[0].isp == "Google LLC"
+
def test_schemas():
input_schema = scanner.input_schema()
output_schema = scanner.output_schema()
- assert input_schema == {'type': 'Ip', 'properties': [{'name': 'address', 'type': 'string'}, {'name': 'latitude', 'type': 'number | null'}, {'name': 'longitude', 'type': 'number | null'}, {'name': 'country', 'type': 'string | null'}, {'name': 'city', 'type': 'string | null'}, {'name': 'isp', 'type': 'string | null'}]}
- assert output_schema == {'type': 'Ip', 'properties': [{'name': 'address', 'type': 'string'}, {'name': 'latitude', 'type': 'number | null'}, {'name': 'longitude', 'type': 'number | null'}, {'name': 'country', 'type': 'string | null'}, {'name': 'city', 'type': 'string | null'}, {'name': 'isp', 'type': 'string | null'}]}
-
+ assert input_schema == {
+ "type": "Ip",
+ "properties": [
+ {"name": "address", "type": "string"},
+ {"name": "latitude", "type": "number | null"},
+ {"name": "longitude", "type": "number | null"},
+ {"name": "country", "type": "string | null"},
+ {"name": "city", "type": "string | null"},
+ {"name": "isp", "type": "string | null"},
+ ],
+ }
+ assert output_schema == {
+ "type": "Ip",
+ "properties": [
+ {"name": "address", "type": "string"},
+ {"name": "latitude", "type": "number | null"},
+ {"name": "longitude", "type": "number | null"},
+ {"name": "country", "type": "string | null"},
+ {"name": "city", "type": "string | null"},
+ {"name": "isp", "type": "string | null"},
+ ],
+ }
diff --git a/flowsint-transforms/tests/scanners/ips/ip_to_asn.py b/flowsint-transforms/tests/scanners/ips/ip_to_asn.py
index 769f81d7..091017e7 100644
--- a/flowsint-transforms/tests/scanners/ips/ip_to_asn.py
+++ b/flowsint-transforms/tests/scanners/ips/ip_to_asn.py
@@ -10,18 +10,20 @@ logger = TestLogger()
# The scanner will get a mock logger from conftest.py automatically
scanner = IpToAsnScanner("sketch_123", "scan_123", logger)
+
def test_preprocess_valid_ips():
ips = [
Ip(address="8.8.8.8"),
Ip(address="1.1.1.1"),
]
result = scanner.preprocess(ips)
-
+
result_addresses = [ip.address for ip in result]
expected_addresses = [ip.address for ip in ips]
assert result_addresses == expected_addresses
-
+
+
def test_unprocessed_valid_ips():
ips = [
"8.8.8.8",
@@ -30,8 +32,9 @@ def test_unprocessed_valid_ips():
result = scanner.preprocess(ips)
result_ips = [ip for ip in result]
expected_ips = [Ip(address=ip) for ip in ips]
- assert result_ips == expected_ips
-
+ assert result_ips == expected_ips
+
+
def test_preprocess_invalid_ips():
ips = [
Ip(address="8.8.8.8"),
@@ -45,6 +48,7 @@ def test_preprocess_invalid_ips():
assert "192.168.1.1" in result_addresses
assert "invalid_ip" not in result_addresses
+
def test_preprocess_multiple_formats():
ips = [
{"address": "8.8.8.8"},
@@ -58,7 +62,9 @@ def test_preprocess_multiple_formats():
assert "8.8.8.8" in result_addresses
assert "192.168.1.1" in result_addresses
assert "10.0.0.1" in result_addresses
- assert "1.1.1.1" not in result_addresses # Should be filtered out due to invalid key
+ assert (
+ "1.1.1.1" not in result_addresses
+ ) # Should be filtered out due to invalid key
def test_scan_extracts_asn_info(monkeypatch):
@@ -67,7 +73,7 @@ def test_scan_extracts_asn_info(monkeypatch):
"as_number": "AS15169",
"as_name": "GOOGLE",
"as_country": "US",
- "as_range": ["8.8.8.0/24", "8.8.4.0/24"]
+ "as_range": ["8.8.8.0/24", "8.8.4.0/24"],
}
class MockSubprocessResult:
@@ -85,10 +91,10 @@ def test_scan_extracts_asn_info(monkeypatch):
input_data = [Ip(address="8.8.8.8")]
asns = scanner.scan(input_data)
-
+
assert isinstance(asns, list)
assert len(asns) == 1
-
+
asn = asns[0]
assert isinstance(asn, ASN)
assert asn.number == 15169 # AS15169 -> 15169
@@ -113,10 +119,10 @@ def test_scan_handles_no_asn_found(monkeypatch):
input_data = [Ip(address="192.168.1.1")]
asns = scanner.scan(input_data)
-
+
assert isinstance(asns, list)
assert len(asns) == 1
-
+
asn = asns[0]
assert isinstance(asn, ASN)
assert asn.number == 0
@@ -133,10 +139,10 @@ def test_scan_handles_subprocess_exception(monkeypatch):
input_data = [Ip(address="8.8.8.8")]
asns = scanner.scan(input_data)
-
+
assert isinstance(asns, list)
assert len(asns) == 1
-
+
asn = asns[0]
assert isinstance(asn, ASN)
assert asn.number == 0
@@ -151,15 +157,15 @@ def test_scan_multiple_ips(monkeypatch):
"as_number": "AS15169",
"as_name": "GOOGLE",
"as_country": "US",
- "as_range": ["8.8.8.0/24"]
+ "as_range": ["8.8.8.0/24"],
},
"1.1.1.1": {
"input": "1.1.1.1",
"as_number": "AS13335",
"as_name": "CLOUDFLARE",
"as_country": "US",
- "as_range": ["1.1.1.0/24"]
- }
+ "as_range": ["1.1.1.0/24"],
+ },
}
class MockSubprocessResult:
@@ -176,13 +182,13 @@ def test_scan_multiple_ips(monkeypatch):
input_data = [Ip(address="8.8.8.8"), Ip(address="1.1.1.1")]
asns = scanner.scan(input_data)
-
+
assert len(asns) == 2
-
+
# Check first ASN
assert asns[0].number == 15169
assert asns[0].name == "GOOGLE"
-
+
# Check second ASN
assert asns[1].number == 13335
assert asns[1].name == "CLOUDFLARE"
@@ -191,13 +197,15 @@ def test_scan_multiple_ips(monkeypatch):
def test_schemas():
input_schema = scanner.input_schema()
output_schema = scanner.output_schema()
-
+
# Input schema should have address field
assert "properties" in input_schema
- address_prop = next((prop for prop in input_schema["properties"] if prop["name"] == "address"), None)
+ address_prop = next(
+ (prop for prop in input_schema["properties"] if prop["name"] == "address"), None
+ )
assert address_prop is not None
assert address_prop["type"] == "string"
-
+
# Output schema should have ASN fields
assert "properties" in output_schema
prop_names = [prop["name"] for prop in output_schema["properties"]]
@@ -211,20 +219,22 @@ def test_postprocess_creates_neo4j_relationships(monkeypatch):
# Mock Neo4j connection
mock_neo4j = Mock()
scanner.neo4j_conn = mock_neo4j
-
+
input_data = [Ip(address="8.8.8.8")]
- asn_results = [ASN(
- number=15169,
- name="GOOGLE",
- country="US",
- cidrs=[CIDR(network="8.8.8.0/24")]
- )]
-
+ asn_results = [
+ ASN(
+ number=15169,
+ name="GOOGLE",
+ country="US",
+ cidrs=[CIDR(network="8.8.8.0/24")],
+ )
+ ]
+
result = scanner.postprocess(asn_results, input_data)
-
+
# Verify Neo4j query was called
mock_neo4j.query.assert_called_once()
-
+
# Check the query parameters
call_args = mock_neo4j.query.call_args
params = call_args[0][1]
@@ -233,7 +243,7 @@ def test_postprocess_creates_neo4j_relationships(monkeypatch):
assert params["asn_name"] == "GOOGLE"
assert params["asn_country"] == "US"
assert params["sketch_id"] == "sketch_123"
-
+
# Should return the same results
assert result == asn_results
@@ -242,19 +252,16 @@ def test_postprocess_skips_unknown_asns(monkeypatch):
# Mock Neo4j connection
mock_neo4j = Mock()
scanner.neo4j_conn = mock_neo4j
-
+
input_data = [Ip(address="192.168.1.1")]
- asn_results = [ASN(
- number=0, # Unknown ASN
- name="Unknown",
- country="Unknown",
- cidrs=[]
- )]
-
+ asn_results = [
+ ASN(number=0, name="Unknown", country="Unknown", cidrs=[]) # Unknown ASN
+ ]
+
result = scanner.postprocess(asn_results, input_data)
-
+
# Verify Neo4j query was NOT called for unknown ASN
mock_neo4j.query.assert_not_called()
-
+
# Should return the same results
- assert result == asn_results
\ No newline at end of file
+ assert result == asn_results
diff --git a/flowsint-transforms/tests/scanners/organizations/to_infos.py b/flowsint-transforms/tests/scanners/organizations/to_infos.py
index c61720cb..26eb180f 100644
--- a/flowsint-transforms/tests/scanners/organizations/to_infos.py
+++ b/flowsint-transforms/tests/scanners/organizations/to_infos.py
@@ -3,17 +3,15 @@ from flowsint_types.organization import Organization
scanner = OrgToInfosScanner("sketch_123", "scan_123")
+
def test_preprocess_valid_names():
- data = [
- Organization(name="OpenAI"),
- {"name": "Inria"},
- "OVH"
- ]
+ data = [Organization(name="OpenAI"), {"name": "Inria"}, "OVH"]
result = scanner.preprocess(data)
result_names = [org.name for org in result]
assert result_names == ["OpenAI", "Inria", "OVH"]
+
# def test_preprocess_invalid_entries():
# data = [
# {"wrong_key": "value"},
@@ -25,6 +23,7 @@ def test_preprocess_valid_names():
# result = scanner.preprocess(data)
# assert result == []
+
def test_execute():
scanner.execute(["Karim Terrache"])
- assert True
\ No newline at end of file
+ assert True
diff --git a/flowsint-transforms/tests/scanners/socials/maigret.py b/flowsint-transforms/tests/scanners/socials/maigret.py
index 289b739d..e0c368a7 100644
--- a/flowsint-transforms/tests/scanners/socials/maigret.py
+++ b/flowsint-transforms/tests/scanners/socials/maigret.py
@@ -1,9 +1,10 @@
from pathlib import Path
from flowsint_transforms.socials.maigret import MaigretScanner
-from flowsint_types.social import Social
+from flowsint_types.social import SocialProfile
scanner = MaigretScanner("sketch_123", "scan_123")
+
def test_unprocessed_valid_usernames():
usernames = [
"toto123",
@@ -11,14 +12,15 @@ def test_unprocessed_valid_usernames():
]
result = scanner.preprocess(usernames)
result_usernames = [d for d in result]
- expected_usernames = [Social(username=d) for d in usernames]
- assert result_usernames == expected_usernames
-
+ expected_usernames = [SocialProfile(username=d) for d in usernames]
+ assert result_usernames == expected_usernames
+
+
def test_preprocess_invalid_usernames():
usernames = [
- Social(username="toto123"),
- Social(username="DorianXd78_Official"),
- Social(username="This is not a username"),
+ SocialProfile(username="toto123"),
+ SocialProfile(username="DorianXd78_Official"),
+ SocialProfile(username="This is not a username"),
]
result = scanner.preprocess(usernames)
@@ -27,11 +29,12 @@ def test_preprocess_invalid_usernames():
assert "DorianXd78_Official" in result_usernames
assert "This is not a username" not in result_usernames
+
def test_preprocess_multiple_formats():
usernames = [
{"username": "toto123"},
{"invalid_key": "ValId_UseRnAme"},
- Social(username="DorianXd78_Official"),
+ SocialProfile(username="DorianXd78_Official"),
"MySimpleUsername",
]
result = scanner.preprocess(usernames)
@@ -41,13 +44,14 @@ def test_preprocess_multiple_formats():
assert "DorianXd78_Official" in result_usernames
assert "ValId_UseRnAme" not in result_usernames
assert "MySimpleUsername" in result_usernames
-
+
+
def test_parsing_invalid_output_file():
results = scanner.parse_maigret_output("toto123", Path("/this/path/does/not/exist"))
assert results == []
-
+
+
def test_parsing():
results = scanner.parse_maigret_output("toto123", Path("/tmp/maigret_test.json"))
print(results)
assert len(results) == 2
-
diff --git a/flowsint-transforms/tests/scanners/websites/to_links.py b/flowsint-transforms/tests/scanners/websites/to_links.py
index 2a8dff58..02c668ab 100644
--- a/flowsint-transforms/tests/scanners/websites/to_links.py
+++ b/flowsint-transforms/tests/scanners/websites/to_links.py
@@ -1,5 +1,5 @@
import pytest
-from unittest.mock import Mock, patch, call
+from unittest.mock import Mock, patch
from flowsint_transforms.websites.to_links import WebsiteToLinks
from flowsint_types.website import Website
@@ -14,10 +14,10 @@ class MockCrawler:
def __init__(self, url, recursive=True, verbose=False, _on_result_callback=None):
self.url = url
self.callback = _on_result_callback
-
+
def fetch(self):
pass
-
+
def extract_urls(self):
# Simulate callback calls
if self.callback:
@@ -25,11 +25,14 @@ class MockCrawler:
self.callback("https://example.com/page2", is_external=False)
self.callback("https://external.com/page", is_external=True)
self.callback("https://another-external.org/resource", is_external=True)
-
+
def get_results(self):
return MockCrawlResults(
internal=["https://example.com/page1", "https://example.com/page2"],
- external=["https://external.com/page", "https://another-external.org/resource"]
+ external=[
+ "https://external.com/page",
+ "https://another-external.org/resource",
+ ],
)
@@ -37,102 +40,210 @@ class MockCrawler:
async def test_website_to_links_real_time_neo4j_creation():
"""Test that Neo4j nodes are created in real-time during the callback."""
scanner = WebsiteToLinks(sketch_id="test", scan_id="test")
-
+
# Mock neo4j connection and methods
scanner.neo4j_conn = Mock()
scanner.create_node = Mock()
scanner.create_relationship = Mock()
scanner.log_graph_message = Mock()
-
+
# Test input
websites = [Website(url="https://example.com")]
-
- with patch('src.transforms.websites.to_links.Crawler', MockCrawler):
+
+ with patch("src.transforms.websites.to_links.Crawler", MockCrawler):
results = await scanner.scan(websites)
-
+
# Verify main website and domain nodes were created upfront
- scanner.create_node.assert_any_call('website', 'url', 'https://example.com',
- caption='https://example.com', type='website')
- scanner.create_node.assert_any_call('domain', 'name', 'example.com',
- caption='example.com', type='domain')
-
+ scanner.create_node.assert_any_call(
+ "website",
+ "url",
+ "https://example.com",
+ caption="https://example.com",
+ type="website",
+ )
+ scanner.create_node.assert_any_call(
+ "domain", "name", "example.com", caption="example.com", type="domain"
+ )
+
# Verify main website to domain relationship
- scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com',
- 'domain', 'name', 'example.com', 'BELONGS_TO_DOMAIN')
-
+ scanner.create_relationship.assert_any_call(
+ "website",
+ "url",
+ "https://example.com",
+ "domain",
+ "name",
+ "example.com",
+ "BELONGS_TO_DOMAIN",
+ )
+
# Verify internal website nodes were created in callback
- scanner.create_node.assert_any_call('website', 'url', 'https://example.com/page1',
- caption='https://example.com/page1', type='website')
- scanner.create_node.assert_any_call('website', 'url', 'https://example.com/page2',
- caption='https://example.com/page2', type='website')
-
+ scanner.create_node.assert_any_call(
+ "website",
+ "url",
+ "https://example.com/page1",
+ caption="https://example.com/page1",
+ type="website",
+ )
+ scanner.create_node.assert_any_call(
+ "website",
+ "url",
+ "https://example.com/page2",
+ caption="https://example.com/page2",
+ type="website",
+ )
+
# Verify internal website relationships
- scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com',
- 'website', 'url', 'https://example.com/page1', 'LINKS_TO')
- scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com',
- 'website', 'url', 'https://example.com/page2', 'LINKS_TO')
-
+ scanner.create_relationship.assert_any_call(
+ "website",
+ "url",
+ "https://example.com",
+ "website",
+ "url",
+ "https://example.com/page1",
+ "LINKS_TO",
+ )
+ scanner.create_relationship.assert_any_call(
+ "website",
+ "url",
+ "https://example.com",
+ "website",
+ "url",
+ "https://example.com/page2",
+ "LINKS_TO",
+ )
+
# Verify external website nodes were created in callback
- scanner.create_node.assert_any_call('website', 'url', 'https://external.com/page',
- caption='https://external.com/page', type='website')
- scanner.create_node.assert_any_call('website', 'url', 'https://another-external.org/resource',
- caption='https://another-external.org/resource', type='website')
-
+ scanner.create_node.assert_any_call(
+ "website",
+ "url",
+ "https://external.com/page",
+ caption="https://external.com/page",
+ type="website",
+ )
+ scanner.create_node.assert_any_call(
+ "website",
+ "url",
+ "https://another-external.org/resource",
+ caption="https://another-external.org/resource",
+ type="website",
+ )
+
# Verify external domain nodes were created in callback
- scanner.create_node.assert_any_call('domain', 'name', 'external.com',
- caption='external.com', type='domain')
- scanner.create_node.assert_any_call('domain', 'name', 'another-external.org',
- caption='another-external.org', type='domain')
-
+ scanner.create_node.assert_any_call(
+ "domain", "name", "external.com", caption="external.com", type="domain"
+ )
+ scanner.create_node.assert_any_call(
+ "domain",
+ "name",
+ "another-external.org",
+ caption="another-external.org",
+ type="domain",
+ )
+
# Verify main website to external website relationships
- scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com',
- 'website', 'url', 'https://external.com/page', 'LINKS_TO')
- scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com',
- 'website', 'url', 'https://another-external.org/resource', 'LINKS_TO')
-
+ scanner.create_relationship.assert_any_call(
+ "website",
+ "url",
+ "https://example.com",
+ "website",
+ "url",
+ "https://external.com/page",
+ "LINKS_TO",
+ )
+ scanner.create_relationship.assert_any_call(
+ "website",
+ "url",
+ "https://example.com",
+ "website",
+ "url",
+ "https://another-external.org/resource",
+ "LINKS_TO",
+ )
+
# Verify external website to domain relationships
- scanner.create_relationship.assert_any_call('website', 'url', 'https://external.com/page',
- 'domain', 'name', 'external.com', 'BELONGS_TO_DOMAIN')
- scanner.create_relationship.assert_any_call('website', 'url', 'https://another-external.org/resource',
- 'domain', 'name', 'another-external.org', 'BELONGS_TO_DOMAIN')
-
+ scanner.create_relationship.assert_any_call(
+ "website",
+ "url",
+ "https://external.com/page",
+ "domain",
+ "name",
+ "external.com",
+ "BELONGS_TO_DOMAIN",
+ )
+ scanner.create_relationship.assert_any_call(
+ "website",
+ "url",
+ "https://another-external.org/resource",
+ "domain",
+ "name",
+ "another-external.org",
+ "BELONGS_TO_DOMAIN",
+ )
+
# Verify main website to external domain relationships
- scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com',
- 'domain', 'name', 'external.com', 'LINKS_TO_DOMAIN')
- scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com',
- 'domain', 'name', 'another-external.org', 'LINKS_TO_DOMAIN')
+ scanner.create_relationship.assert_any_call(
+ "website",
+ "url",
+ "https://example.com",
+ "domain",
+ "name",
+ "external.com",
+ "LINKS_TO_DOMAIN",
+ )
+ scanner.create_relationship.assert_any_call(
+ "website",
+ "url",
+ "https://example.com",
+ "domain",
+ "name",
+ "another-external.org",
+ "LINKS_TO_DOMAIN",
+ )
@pytest.mark.asyncio
async def test_website_to_links_error_handling_with_neo4j():
"""Test that main nodes are still created even when crawling fails."""
scanner = WebsiteToLinks(sketch_id="test", scan_id="test")
-
+
# Mock neo4j connection and methods
scanner.neo4j_conn = Mock()
scanner.create_node = Mock()
scanner.create_relationship = Mock()
scanner.log_graph_message = Mock()
-
+
# Mock crawler that raises an exception
def mock_crawler_error(*args, **kwargs):
raise Exception("Test error")
-
+
websites = [Website(url="https://example.com")]
-
- with patch('src.transforms.websites.to_links.Crawler', mock_crawler_error):
+
+ with patch("src.transforms.websites.to_links.Crawler", mock_crawler_error):
results = await scanner.scan(websites)
-
+
# Verify main website and domain nodes were still created despite error
- scanner.create_node.assert_any_call('website', 'url', 'https://example.com',
- caption='https://example.com', type='website')
- scanner.create_node.assert_any_call('domain', 'name', 'example.com',
- caption='example.com', type='domain')
-
+ scanner.create_node.assert_any_call(
+ "website",
+ "url",
+ "https://example.com",
+ caption="https://example.com",
+ type="website",
+ )
+ scanner.create_node.assert_any_call(
+ "domain", "name", "example.com", caption="example.com", type="domain"
+ )
+
# Verify main website to domain relationship was created
- scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com',
- 'domain', 'name', 'example.com', 'BELONGS_TO_DOMAIN')
-
+ scanner.create_relationship.assert_any_call(
+ "website",
+ "url",
+ "https://example.com",
+ "domain",
+ "name",
+ "example.com",
+ "BELONGS_TO_DOMAIN",
+ )
+
# Verify result structure
assert len(results) == 1
result = results[0]
@@ -146,17 +257,19 @@ async def test_website_to_links_error_handling_with_neo4j():
def test_postprocess_simplified():
"""Test that postprocess now just returns results as-is."""
scanner = WebsiteToLinks(sketch_id="test", scan_id="test")
-
+
original_input = [Website(url="https://example.com")]
- results = [{
- "website": "https://example.com",
- "main_domain": "example.com",
- "internal_urls": ["https://example.com/page1"],
- "external_urls": ["https://external.com/page"],
- "external_domains": ["external.com"]
- }]
-
+ results = [
+ {
+ "website": "https://example.com",
+ "main_domain": "example.com",
+ "internal_urls": ["https://example.com/page1"],
+ "external_urls": ["https://external.com/page"],
+ "external_domains": ["external.com"],
+ }
+ ]
+
processed_results = scanner.postprocess(results, original_input)
-
+
# Should just return the same results since Neo4j work is done in real-time
- assert processed_results == results
\ No newline at end of file
+ assert processed_results == results
diff --git a/flowsint-transforms/tests/tools/network/asnmap.py b/flowsint-transforms/tests/tools/network/asnmap.py
index f21f1736..12f83fbe 100644
--- a/flowsint-transforms/tests/tools/network/asnmap.py
+++ b/flowsint-transforms/tests/tools/network/asnmap.py
@@ -1,42 +1,52 @@
import re
from typing import Dict
-from app.tools.network.asnmap import AsnmapTool
+from tools.network.asnmap import AsnmapTool
tool = AsnmapTool()
+
def test_name():
assert tool.name() == "asnmap"
+
def test_description():
assert tool.description() == "ASN mapping and network reconnaissance tool."
-
+
+
def test_category():
assert tool.category() == "ASN discovery"
+
def test_image():
assert tool.get_image() == "projectdiscovery/asnmap"
-
+
+
def test_install():
tool.install()
assert tool.is_installed() == True
-
+
+
def test_version():
tool.install()
version = tool.version()
# Check that version follows the expected format: v followed by digits and dots
- assert re.match(r'^v[\d\.]+$', version)
-
+ assert re.match(r"^v[\d\.]+$", version)
+
+
def test_launch_no_api_key():
import pytest
+
with pytest.raises(KeyError, match="Missing key"):
- tool.launch("alliage.io", 'domain')
+ tool.launch("alliage.io", "domain")
+
def test_launch_wrong_type():
import pytest
+
with pytest.raises(ValueError, match="Invalid type: 'domains'"):
- tool.launch("alliage.io", 'domains')
-
+ tool.launch("alliage.io", "domains")
+
+
def test_launch():
- results = tool.launch("alliage.io", 'domain')
+ results = tool.launch("alliage.io", "domain")
assert isinstance(results, Dict)
-
\ No newline at end of file
diff --git a/flowsint-transforms/tests/tools/network/httpx.py b/flowsint-transforms/tests/tools/network/httpx.py
index 5e746acb..63af13a6 100644
--- a/flowsint-transforms/tests/tools/network/httpx.py
+++ b/flowsint-transforms/tests/tools/network/httpx.py
@@ -1,40 +1,50 @@
import re
from typing import List
-from app.tools.network.httpx import HttpxTool
+from tools.network.httpx import HttpxTool
tool = HttpxTool()
+
def test_name():
assert tool.name() == "httpx"
+
def test_description():
- assert tool.description() == "An HTTP toolkit that probes services, web servers, and other valuable metadata."
-
+ assert (
+ tool.description()
+ == "An HTTP toolkit that probes services, web servers, and other valuable metadata."
+ )
+
+
def test_category():
assert tool.category() == "Web technologies enumeration"
+
def test_image():
assert tool.get_image() == "projectdiscovery/httpx"
-
+
+
def test_install():
tool.install()
assert tool.is_installed() == True
-
+
+
def test_version():
tool.install()
version = tool.version()
# Check that version follows the expected format: v followed by digits and dots
- assert re.match(r'^v[\d\.]+$', version)
-
+ assert re.match(r"^v[\d\.]+$", version)
+
+
def test_launch():
assert True
results = tool.launch("https://alliage.io")
print(results)
assert isinstance(results, List)
-
+
+
def test_launch_unreached_host():
assert True
results = tool.launch("https://this-is-not-a-valid-domain.local")
assert isinstance(results, List)
assert len(results) == 0
-
\ No newline at end of file
diff --git a/flowsint-transforms/tests/tools/network/reconcrawl.py b/flowsint-transforms/tests/tools/network/reconcrawl.py
index 1abbf1e4..0bffd125 100644
--- a/flowsint-transforms/tests/tools/network/reconcrawl.py
+++ b/flowsint-transforms/tests/tools/network/reconcrawl.py
@@ -1,20 +1,23 @@
-import re
-from typing import Dict
-from app.tools.network.reconcrawl import ReconCrawlTool
+from tools.network.reconcrawl import ReconCrawlTool
tool = ReconCrawlTool()
+
def test_name():
assert tool.name() == "reconcrawl"
+
def test_description():
- assert tool.description() == "Emails and phone numbers crawler from websites by analyzing their HTML and embedded scripts."
-
+ assert (
+ tool.description()
+ == "Emails and phone numbers crawler from websites by analyzing their HTML and embedded scripts."
+ )
+
+
def test_category():
assert tool.category() == "Crawler"
-
+
+
def test_install():
tool.install()
assert tool.is_installed() == True
-
-
\ No newline at end of file
diff --git a/flowsint-transforms/tests/tools/network/subfinder.py b/flowsint-transforms/tests/tools/network/subfinder.py
index 444c3444..3219e214 100644
--- a/flowsint-transforms/tests/tools/network/subfinder.py
+++ b/flowsint-transforms/tests/tools/network/subfinder.py
@@ -1,32 +1,38 @@
import re
-from app.tools.network.subfinder import SubfinderTool
+from tools.network.subfinder import SubfinderTool
tool = SubfinderTool()
+
def test_name():
assert tool.name() == "subfinder"
+
def test_description():
assert tool.description() == "Fast passive subdomain enumeration tool."
-
+
+
def test_category():
assert tool.category() == "Subdomain enumeration"
+
def test_image():
assert tool.get_image() == "projectdiscovery/subfinder"
-
+
+
def test_install():
tool.install()
assert tool.is_installed() == True
-
+
+
def test_version():
tool.install()
version = tool.version()
# Check that version follows the expected format: v followed by digits and dots
- assert re.match(r'^v[\d\.]+$', version)
+ assert re.match(r"^v[\d\.]+$", version)
+
def test_launch():
results = tool.launch("alliage.io")
assert isinstance(results, list)
assert all(isinstance(item, str) for item in results)
-
\ No newline at end of file
diff --git a/flowsint-transforms/tests/tools/organizations/sirene.py b/flowsint-transforms/tests/tools/organizations/sirene.py
index eb3f79be..762b3b98 100644
--- a/flowsint-transforms/tests/tools/organizations/sirene.py
+++ b/flowsint-transforms/tests/tools/organizations/sirene.py
@@ -1,28 +1,36 @@
-import re
from typing import Dict
-from app.tools.organizations.sirene import SireneTool
+from tools.organizations.sirene import SireneTool
tool = SireneTool()
+
def test_name():
assert tool.name() == "sirene"
+
def test_description():
- assert tool.description() == "The Sirene API allows you to query the Sirene directory of businesses and establishments, managed by Insee."
-
+ assert (
+ tool.description()
+ == "The Sirene API allows you to query the Sirene directory of businesses and establishments, managed by Insee."
+ )
+
+
def test_category():
assert tool.category() == "Business intelligence"
+
def test_launch_org():
results = tool.launch("blablacar", 1)
assert isinstance(results, list)
assert all(isinstance(item, Dict) for item in results)
+
def test_launch_person():
results = tool.launch("Karim+Terrache", 1)
assert isinstance(results, list)
assert all(isinstance(item, Dict) for item in results)
-
+
+
def test_launch_person_space_format():
results = tool.launch("Karim Terrache", 1)
assert isinstance(results, list)
diff --git a/flowsint-transforms/tests/utils.py b/flowsint-transforms/tests/utils.py
index d1576d96..d9affb08 100644
--- a/flowsint-transforms/tests/utils.py
+++ b/flowsint-transforms/tests/utils.py
@@ -1,41 +1,40 @@
from flowsint_core.core.scanner_base import build_params_model
+
def test_build_params_model_valid():
param_schema = [
- {
- "name": "ETHERSCAN_API_KEY",
- "type": "string",
- "description": "The Etherscan API key to use for the transaction lookup.",
- "required": True
- },
- {
- "name": "url",
- "type": "string",
- "description": "Base URL for API",
- "required": False,
- "default": "https://api.etherscan.io/api"
- }
-]
+ {
+ "name": "ETHERSCAN_API_KEY",
+ "type": "string",
+ "description": "The Etherscan API key to use for the transaction lookup.",
+ "required": True,
+ },
+ {
+ "name": "url",
+ "type": "string",
+ "description": "Base URL for API",
+ "required": False,
+ "default": "https://api.etherscan.io/api",
+ },
+ ]
ParamsModel = build_params_model(param_schema)
validated_params = ParamsModel(ETHERSCAN_API_KEY="clef-123")
assert validated_params.ETHERSCAN_API_KEY == "clef-123"
assert validated_params.url == "https://api.etherscan.io/api"
-
+
+
def test_build_params_model_invalid():
param_schema = [
- {
-
- },
- {
- "name": "url",
- "type": "string",
- "description": "Base URL for API",
- "required": False,
- "default": "https://api.etherscan.io/api"
- }
-]
+ {},
+ {
+ "name": "url",
+ "type": "string",
+ "description": "Base URL for API",
+ "required": False,
+ "default": "https://api.etherscan.io/api",
+ },
+ ]
ParamsModel = build_params_model(param_schema)
validated_params = ParamsModel(ETHERSCAN_API_KEY="clef-123")
assert validated_params.ETHERSCAN_API_KEY == "clef-123"
assert validated_params.url == "https://api.etherscan.io/api"
-
\ No newline at end of file
diff --git a/flowsint-transforms/utils.py b/flowsint-transforms/utils.py
index b1e02a8e..238eed7c 100644
--- a/flowsint-transforms/utils.py
+++ b/flowsint-transforms/utils.py
@@ -13,6 +13,7 @@ import inspect
from typing import Any, Dict, Type
from pydantic import BaseModel, TypeAdapter
+
def is_valid_ip(address: str) -> bool:
try:
ipaddress.ip_address(address)
@@ -20,11 +21,13 @@ def is_valid_ip(address: str) -> bool:
except ValueError:
return False
+
def is_valid_username(username: str) -> bool:
if not re.match(r"^[a-zA-Z0-9_-]{3,30}$", username):
return False
return True
+
def is_valid_email(email: str) -> bool:
if not re.match(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$", email):
return False
@@ -34,26 +37,29 @@ def is_valid_email(email: str) -> bool:
def is_valid_domain(url_or_domain: str) -> str:
try:
- parsed = urlparse(url_or_domain if "://" in url_or_domain else "http://" + url_or_domain)
+ parsed = urlparse(
+ url_or_domain if "://" in url_or_domain else "http://" + url_or_domain
+ )
hostname = parsed.hostname or url_or_domain
-
+
if not hostname or "." not in hostname:
return False
-
+
if not re.match(r"^[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$", hostname):
return False
-
+
return True
except Exception as e:
return False
+
def is_root_domain(domain: str) -> bool:
"""
Determine if a domain is a root domain or subdomain.
-
+
Args:
domain: The domain string to check
-
+
Returns:
True if it's a root domain (e.g., example.com), False if it's a subdomain (e.g., sub.example.com)
"""
@@ -62,20 +68,32 @@ def is_root_domain(domain: str) -> bool:
if "://" in domain:
parsed = urlparse(domain)
domain = parsed.hostname or domain
-
+
# Split by dots
- parts = domain.split('.')
-
+ parts = domain.split(".")
+
# Handle common country code TLDs that have 2 parts (e.g., .co.uk, .com.au, .org.uk)
- common_cc_tlds = ['.co.uk', '.com.au', '.org.uk', '.net.uk', '.gov.uk', '.ac.uk',
- '.co.nz', '.com.sg', '.co.jp', '.co.kr', '.com.br', '.com.mx']
-
+ common_cc_tlds = [
+ ".co.uk",
+ ".com.au",
+ ".org.uk",
+ ".net.uk",
+ ".gov.uk",
+ ".ac.uk",
+ ".co.nz",
+ ".com.sg",
+ ".co.jp",
+ ".co.kr",
+ ".com.br",
+ ".com.mx",
+ ]
+
# Check if the domain ends with a common country code TLD
for cc_tld in common_cc_tlds:
if domain.endswith(cc_tld):
# For country code TLDs, we need exactly 3 parts (e.g., example.co.uk)
return len(parts) == 3
-
+
# For regular TLDs, a root domain has 2 parts (e.g., example.com)
# A subdomain has 3 or more parts (e.g., sub.example.com, www.sub.example.com)
return len(parts) == 2
@@ -83,6 +101,7 @@ def is_root_domain(domain: str) -> bool:
# If we can't parse it, assume it's not a root domain
return False
+
def is_valid_number(phone: str, region: str = "FR") -> None:
"""
Validates a phone number. Raises InvalidPhoneNumberError if invalid.
@@ -94,19 +113,21 @@ def is_valid_number(phone: str, region: str = "FR") -> None:
return False
except NumberParseException:
return False
-
+
+
def parse_asn(asn: str) -> int:
if not is_valid_asn(asn):
raise ValueError(f"Invalid ASN format: {asn}")
return int(re.sub(r"(?i)^AS", "", asn))
-
+
def is_valid_asn(asn: str) -> bool:
if not re.fullmatch(r"(AS)?\d+", asn, re.IGNORECASE):
return False
asn_num = int(re.sub(r"(?i)^AS", "", asn))
return 0 <= asn_num <= 4294967295
+
def resolve_type(details: dict, schema_context: dict = None) -> str:
if "anyOf" in details:
types = []
@@ -121,22 +142,23 @@ def resolve_type(details: dict, schema_context: dict = None) -> str:
else:
types.append(option.get("type", "unknown"))
return " | ".join(types)
-
+
if "type" in details:
if details["type"] == "array":
item_type = resolve_type(details.get("items", {}), schema_context)
return f"{item_type}[]"
return details["type"]
-
+
# Handle $ref in array items or other contexts
if "$ref" in details and schema_context:
ref_path = details["$ref"]
if ref_path.startswith("#/$defs/"):
ref_name = ref_path.split("/")[-1]
return ref_name
-
+
return "any"
+
def extract_input_schema_transform(model: Type[BaseModel]) -> Dict[str, Any]:
adapter = TypeAdapter(model)
schema = adapter.json_schema()
@@ -153,37 +175,30 @@ def extract_input_schema_transform(model: Type[BaseModel]) -> Dict[str, Any]:
"outputs": {
"type": type_name,
"properties": [
- {
- "name": prop,
- "type": resolve_type(info, schema)
- }
+ {"name": prop, "type": resolve_type(info, schema)}
for prop, info in details.get("properties", {}).items()
- ]
- },
- "inputs": {
- "type": "",
- "properties": []
+ ],
},
+ "inputs": {"type": "", "properties": []},
"type": "type",
"category": model.__name__,
}
-
def get_domain_from_ssl(ip: str, port: int = 443) -> str | None:
try:
context = ssl.create_default_context()
with socket.create_connection((ip, port), timeout=3) as sock:
with context.wrap_socket(sock, server_hostname=ip) as ssock:
cert = ssock.getpeercert()
- subject = cert.get('subject', [])
+ subject = cert.get("subject", [])
for entry in subject:
- if entry[0][0] == 'commonName':
+ if entry[0][0] == "commonName":
return entry[0][1]
# Alternative: check subjectAltName
- san = cert.get('subjectAltName', [])
+ san = cert.get("subjectAltName", [])
for typ, val in san:
- if typ == 'DNS':
+ if typ == "DNS":
return val
except Exception as e:
print(f"SSL extraction failed for {ip}: {e}")
@@ -208,12 +223,14 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:
scanner_node = node_lookup.get(target_id)
if scanner_node and scanner_node["data"]["type"] == "scanner":
- scanners.append({
- "scanner_name": scanner_node["data"]["name"],
- "module": scanner_node["data"]["module"],
- "input": source_handle,
- "output": target_handle
- })
+ scanners.append(
+ {
+ "scanner_name": scanner_node["data"]["name"],
+ "module": scanner_node["data"]["module"],
+ "input": source_handle,
+ "output": target_handle,
+ }
+ )
return {
"input": {
@@ -221,25 +238,22 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:
"outputs": input_output,
},
"scanners": scanners,
- "scanner_names" : [scanner["scanner_name"] for scanner in scanners]
-
+ "scanner_names": [scanner["scanner_name"] for scanner in scanners],
}
-
+
+
def get_label_color(label: str) -> str:
- color_map = {
- 'subdomain': '#A5ABB6',
- 'domain': '#68BDF6',
- 'default': '#A5ABB6'
- }
-
+ color_map = {"subdomain": "#A5ABB6", "domain": "#68BDF6", "default": "#A5ABB6"}
+
return color_map.get(label, color_map["default"])
+
def flatten(data_dict, prefix=""):
"""
Flattens a dictionary to contain only Neo4j-compatible property values.
Neo4j supports primitive types (string, number, boolean) and arrays of those types.
Args:
- data_dict (dict): Dictionary to flatten
+ data_dict (dict): Dictionary to flatten
Returns:
dict: Flattened dictionary with only Neo4j-compatible values
"""
@@ -250,13 +264,14 @@ def flatten(data_dict, prefix=""):
if value is None:
continue
if isinstance(value, (str, int, float, bool)) or (
- isinstance(value, list) and all(isinstance(item, (str, int, float, bool)) for item in value)
+ isinstance(value, list)
+ and all(isinstance(item, (str, int, float, bool)) for item in value)
):
key = f"{prefix}{key}"
- flattened[key] = value
+ flattened[key] = value
return flattened
-
+
def get_inline_relationships(nodes: List[Any], edges: List[Any]) -> List[str]:
"""
Get the inline relationships for a list of nodes and edges.
@@ -266,14 +281,15 @@ def get_inline_relationships(nodes: List[Any], edges: List[Any]) -> List[str]:
source = next((node for node in nodes if node["id"] == edge["source"]), None)
target = next((node for node in nodes if node["id"] == edge["target"]), None)
if source and target:
- relationships.append({"source":source, "edge":edge, "target":target})
+ relationships.append({"source": source, "edge": edge, "target": target})
return relationships
+
def to_json_serializable(obj):
"""Convert any object to a JSON-serializable format."""
import json
from pydantic import BaseModel
-
+
try:
# Test if already JSON serializable
json.dumps(obj)
@@ -282,11 +298,15 @@ def to_json_serializable(obj):
# Handle common cases
if isinstance(obj, BaseModel):
# Use mode='json' to ensure all Pydantic types are properly serialized
- return obj.model_dump(mode='json') if hasattr(obj, 'model_dump') else obj.dict()
+ return (
+ obj.model_dump(mode="json")
+ if hasattr(obj, "model_dump")
+ else obj.dict()
+ )
elif isinstance(obj, (list, tuple)):
return [to_json_serializable(item) for item in obj]
elif isinstance(obj, dict):
return {key: to_json_serializable(value) for key, value in obj.items()}
else:
# Convert anything else to string
- return str(obj)
\ No newline at end of file
+ return str(obj)
diff --git a/flowsint-types/src/flowsint_types/__init__.py b/flowsint-types/src/flowsint_types/__init__.py
index 7bf9047c..6d7cf3d2 100644
--- a/flowsint-types/src/flowsint_types/__init__.py
+++ b/flowsint-types/src/flowsint_types/__init__.py
@@ -44,7 +44,7 @@ __author__ = "EliottElek <[email protected]>"
__all__ = [
"PhysicalAddress",
- "Affiliation",
+ "Affiliation",
"Alias",
"ASN",
"BankAccount",
diff --git a/flowsint-types/src/flowsint_types/address.py b/flowsint-types/src/flowsint_types/address.py
index e7e84602..91621f27 100644
--- a/flowsint-types/src/flowsint_types/address.py
+++ b/flowsint-types/src/flowsint_types/address.py
@@ -1,12 +1,17 @@
from pydantic import BaseModel, Field
from typing import Optional
+
class PhysicalAddress(BaseModel):
"""Represents a physical address with geographical coordinates."""
+
address: str = Field(..., description="Street address", title="Street Address")
city: str = Field(..., description="City name", title="City")
country: str = Field(..., description="Country name", title="Country")
zip: str = Field(..., description="ZIP or postal code", title="ZIP/Postal Code")
- latitude: Optional[float] = Field(None, description="Latitude coordinate of the address", title="Latitude")
- longitude: Optional[float] = Field(None, description="Longitude coordinate of the address", title="Longitude")
-
+ latitude: Optional[float] = Field(
+ None, description="Latitude coordinate of the address", title="Latitude"
+ )
+ longitude: Optional[float] = Field(
+ None, description="Longitude coordinate of the address", title="Longitude"
+ )
diff --git a/flowsint-types/src/flowsint_types/affiliation.py b/flowsint-types/src/flowsint_types/affiliation.py
index 0c0527f3..9b594f11 100644
--- a/flowsint-types/src/flowsint_types/affiliation.py
+++ b/flowsint-types/src/flowsint_types/affiliation.py
@@ -1,19 +1,55 @@
from pydantic import BaseModel, Field
from typing import Optional, List
+
class Affiliation(BaseModel):
"""Represents an organizational affiliation or employment relationship."""
- organization: str = Field(..., description="Organization or group name", title="Organization")
- role: Optional[str] = Field(None, description="Role or position within organization", title="Role")
- start_date: Optional[str] = Field(None, description="Start date of affiliation", title="Start Date")
- end_date: Optional[str] = Field(None, description="End date of affiliation", title="End Date")
- is_current: Optional[bool] = Field(None, description="Whether affiliation is currently active", title="Is Current")
- department: Optional[str] = Field(None, description="Department or division", title="Department")
- location: Optional[str] = Field(None, description="Geographic location of affiliation", title="Location")
- industry: Optional[str] = Field(None, description="Industry or sector", title="Industry")
- description: Optional[str] = Field(None, description="Description of affiliation", title="Description")
- source: Optional[str] = Field(None, description="Source of affiliation information", title="Source")
- confidence: Optional[float] = Field(None, description="Confidence score for affiliation", title="Confidence")
- associated_individuals: Optional[List[str]] = Field(None, description="Individuals with similar affiliations", title="Associated Individuals")
- organization_type: Optional[str] = Field(None, description="Type of organization (company, NGO, government, etc.)", title="Organization Type")
- hierarchy_level: Optional[str] = Field(None, description="Hierarchical level within organization", title="Hierarchy Level")
\ No newline at end of file
+
+ organization: str = Field(
+ ..., description="Organization or group name", title="Organization"
+ )
+ role: Optional[str] = Field(
+ None, description="Role or position within organization", title="Role"
+ )
+ start_date: Optional[str] = Field(
+ None, description="Start date of affiliation", title="Start Date"
+ )
+ end_date: Optional[str] = Field(
+ None, description="End date of affiliation", title="End Date"
+ )
+ is_current: Optional[bool] = Field(
+ None, description="Whether affiliation is currently active", title="Is Current"
+ )
+ department: Optional[str] = Field(
+ None, description="Department or division", title="Department"
+ )
+ location: Optional[str] = Field(
+ None, description="Geographic location of affiliation", title="Location"
+ )
+ industry: Optional[str] = Field(
+ None, description="Industry or sector", title="Industry"
+ )
+ description: Optional[str] = Field(
+ None, description="Description of affiliation", title="Description"
+ )
+ source: Optional[str] = Field(
+ None, description="Source of affiliation information", title="Source"
+ )
+ confidence: Optional[float] = Field(
+ None, description="Confidence score for affiliation", title="Confidence"
+ )
+ associated_individuals: Optional[List[str]] = Field(
+ None,
+ description="Individuals with similar affiliations",
+ title="Associated Individuals",
+ )
+ organization_type: Optional[str] = Field(
+ None,
+ description="Type of organization (company, NGO, government, etc.)",
+ title="Organization Type",
+ )
+ hierarchy_level: Optional[str] = Field(
+ None,
+ description="Hierarchical level within organization",
+ title="Hierarchy Level",
+ )
diff --git a/flowsint-types/src/flowsint_types/alias.py b/flowsint-types/src/flowsint_types/alias.py
index aab7f7b2..2e766125 100644
--- a/flowsint-types/src/flowsint_types/alias.py
+++ b/flowsint-types/src/flowsint_types/alias.py
@@ -1,17 +1,45 @@
from pydantic import BaseModel, Field
from typing import Optional, List
+
class Alias(BaseModel):
"""Represents an alias or alternative name used by an entity."""
+
alias: str = Field(..., description="Alias or alternative name", title="Alias")
- type: Optional[str] = Field(None, description="Type of alias (nickname, code name, etc.)", title="Alias Type")
- context: Optional[str] = Field(None, description="Context where alias is used", title="Context")
- first_seen: Optional[str] = Field(None, description="First time alias was observed", title="First Seen")
- last_seen: Optional[str] = Field(None, description="Last time alias was observed", title="Last Seen")
- confidence: Optional[float] = Field(None, description="Confidence score for alias association", title="Confidence")
- source: Optional[str] = Field(None, description="Source of alias information", title="Source")
- associated_entities: Optional[List[str]] = Field(None, description="Entities associated with this alias", title="Associated Entities")
- description: Optional[str] = Field(None, description="Description or notes about the alias", title="Description")
- is_active: Optional[bool] = Field(None, description="Whether alias is currently active", title="Is Active")
- language: Optional[str] = Field(None, description="Language of the alias", title="Language")
- region: Optional[str] = Field(None, description="Geographic region where alias is used", title="Region")
\ No newline at end of file
+ type: Optional[str] = Field(
+ None,
+ description="Type of alias (nickname, code name, etc.)",
+ title="Alias Type",
+ )
+ context: Optional[str] = Field(
+ None, description="Context where alias is used", title="Context"
+ )
+ first_seen: Optional[str] = Field(
+ None, description="First time alias was observed", title="First Seen"
+ )
+ last_seen: Optional[str] = Field(
+ None, description="Last time alias was observed", title="Last Seen"
+ )
+ confidence: Optional[float] = Field(
+ None, description="Confidence score for alias association", title="Confidence"
+ )
+ source: Optional[str] = Field(
+ None, description="Source of alias information", title="Source"
+ )
+ associated_entities: Optional[List[str]] = Field(
+ None,
+ description="Entities associated with this alias",
+ title="Associated Entities",
+ )
+ description: Optional[str] = Field(
+ None, description="Description or notes about the alias", title="Description"
+ )
+ is_active: Optional[bool] = Field(
+ None, description="Whether alias is currently active", title="Is Active"
+ )
+ language: Optional[str] = Field(
+ None, description="Language of the alias", title="Language"
+ )
+ region: Optional[str] = Field(
+ None, description="Geographic region where alias is used", title="Region"
+ )
diff --git a/flowsint-types/src/flowsint_types/asn.py b/flowsint-types/src/flowsint_types/asn.py
index 61a29015..df8db510 100644
--- a/flowsint-types/src/flowsint_types/asn.py
+++ b/flowsint-types/src/flowsint_types/asn.py
@@ -2,12 +2,29 @@ from typing import List, Optional
from pydantic import BaseModel, Field
from .cidr import CIDR
+
class ASN(BaseModel):
"""Represents an Autonomous System Number with associated network information."""
- number: int = Field(..., description="Autonomous System Number (e.g., 15169)", title="ASN Number")
- name: Optional[str] = Field(None, description="Name of the organization owning the ASN", title="Organization Name")
- country: Optional[str] = Field(None, description="ISO 3166-1 alpha-2 country code", title="Country Code")
- description: Optional[str] = Field(None, description="Additional information about the ASN", title="Description")
- cidrs: List[CIDR] = Field(default_factory=list, description="List of announced CIDR blocks", title="CIDR Blocks")
+
+ number: int = Field(
+ ..., description="Autonomous System Number (e.g., 15169)", title="ASN Number"
+ )
+ name: Optional[str] = Field(
+ None,
+ description="Name of the organization owning the ASN",
+ title="Organization Name",
+ )
+ country: Optional[str] = Field(
+ None, description="ISO 3166-1 alpha-2 country code", title="Country Code"
+ )
+ description: Optional[str] = Field(
+ None, description="Additional information about the ASN", title="Description"
+ )
+ cidrs: List[CIDR] = Field(
+ default_factory=list,
+ description="List of announced CIDR blocks",
+ title="CIDR Blocks",
+ )
+
ASN.model_rebuild()
diff --git a/flowsint-types/src/flowsint_types/bank_account.py b/flowsint-types/src/flowsint_types/bank_account.py
index fbee130e..bbafc745 100644
--- a/flowsint-types/src/flowsint_types/bank_account.py
+++ b/flowsint-types/src/flowsint_types/bank_account.py
@@ -1,23 +1,63 @@
from pydantic import BaseModel, Field
from typing import Optional, List
+
class BankAccount(BaseModel):
"""Represents a bank account with financial and security information."""
- account_number: str = Field(..., description="Bank account number", title="Account Number")
+
+ account_number: str = Field(
+ ..., description="Bank account number", title="Account Number"
+ )
bank_name: Optional[str] = Field(None, description="Bank name", title="Bank Name")
- account_type: Optional[str] = Field(None, description="Type of account (checking, savings, etc.)", title="Account Type")
- routing_number: Optional[str] = Field(None, description="Bank routing number", title="Routing Number")
- iban: Optional[str] = Field(None, description="International Bank Account Number", title="IBAN")
- swift_code: Optional[str] = Field(None, description="SWIFT/BIC code", title="SWIFT Code")
- country: Optional[str] = Field(None, description="Country where account is held", title="Country")
- currency: Optional[str] = Field(None, description="Account currency", title="Currency")
- balance: Optional[float] = Field(None, description="Account balance", title="Balance")
- account_holder: Optional[str] = Field(None, description="Account holder name", title="Account Holder")
- status: Optional[str] = Field(None, description="Account status (active, closed, etc.)", title="Status")
- opened_date: Optional[str] = Field(None, description="Account opening date", title="Opened Date")
- closed_date: Optional[str] = Field(None, description="Account closing date", title="Closed Date")
- is_joint: Optional[bool] = Field(None, description="Whether account is joint", title="Is Joint")
- associated_individuals: Optional[List[str]] = Field(None, description="Individuals associated with account", title="Associated Individuals")
- source: Optional[str] = Field(None, description="Source of account information", title="Source")
- is_compromised: Optional[bool] = Field(None, description="Whether account has been compromised", title="Is Compromised")
- breach_source: Optional[str] = Field(None, description="Source of breach if compromised", title="Breach Source")
\ No newline at end of file
+ account_type: Optional[str] = Field(
+ None,
+ description="Type of account (checking, savings, etc.)",
+ title="Account Type",
+ )
+ routing_number: Optional[str] = Field(
+ None, description="Bank routing number", title="Routing Number"
+ )
+ iban: Optional[str] = Field(
+ None, description="International Bank Account Number", title="IBAN"
+ )
+ swift_code: Optional[str] = Field(
+ None, description="SWIFT/BIC code", title="SWIFT Code"
+ )
+ country: Optional[str] = Field(
+ None, description="Country where account is held", title="Country"
+ )
+ currency: Optional[str] = Field(
+ None, description="Account currency", title="Currency"
+ )
+ balance: Optional[float] = Field(
+ None, description="Account balance", title="Balance"
+ )
+ account_holder: Optional[str] = Field(
+ None, description="Account holder name", title="Account Holder"
+ )
+ status: Optional[str] = Field(
+ None, description="Account status (active, closed, etc.)", title="Status"
+ )
+ opened_date: Optional[str] = Field(
+ None, description="Account opening date", title="Opened Date"
+ )
+ closed_date: Optional[str] = Field(
+ None, description="Account closing date", title="Closed Date"
+ )
+ is_joint: Optional[bool] = Field(
+ None, description="Whether account is joint", title="Is Joint"
+ )
+ associated_individuals: Optional[List[str]] = Field(
+ None,
+ description="Individuals associated with account",
+ title="Associated Individuals",
+ )
+ source: Optional[str] = Field(
+ None, description="Source of account information", title="Source"
+ )
+ is_compromised: Optional[bool] = Field(
+ None, description="Whether account has been compromised", title="Is Compromised"
+ )
+ breach_source: Optional[str] = Field(
+ None, description="Source of breach if compromised", title="Breach Source"
+ )
diff --git a/flowsint-types/src/flowsint_types/breach.py b/flowsint-types/src/flowsint_types/breach.py
index 53ae5730..e3b5b050 100644
--- a/flowsint-types/src/flowsint_types/breach.py
+++ b/flowsint-types/src/flowsint_types/breach.py
@@ -1,24 +1,68 @@
from typing import List, Optional, Dict
from pydantic import BaseModel, Field
+
class Breach(BaseModel):
"""Represents a data breach incident with affected accounts and details."""
- name: str = Field(..., description="The name of the breach or service", title="Breach Name")
- title: Optional[str] = Field(None, description="Title of the breach", title="Breach Title")
- domain: Optional[str] = Field(None, description="Domain of the breached service", title="Domain")
- breachdate: Optional[str] = Field(None, description="Date of the breach", title="Breach Date")
- addeddate: Optional[str] = Field(None, description="Date breach was added to HIBP", title="Added Date")
- modifieddate: Optional[str] = Field(None, description="Date breach was last modified", title="Modified Date")
- pwncount: Optional[int] = Field(None, description="Number of accounts affected", title="Affected Accounts")
- description: Optional[str] = Field(None, description="Description of the breach", title="Description")
- logopath: Optional[str] = Field(None, description="Logo path for the breach", title="Logo Path")
- dataclasses: Optional[List[str]] = Field(None, description="Types of data compromised", title="Data Classes")
- isverified: Optional[bool] = Field(None, description="Whether the breach is verified", title="Is Verified")
- isfabricated: Optional[bool] = Field(None, description="Whether the breach is fabricated", title="Is Fabricated")
- issensitive: Optional[bool] = Field(None, description="Whether the breach is sensitive", title="Is Sensitive")
- isretired: Optional[bool] = Field(None, description="Whether the breach is retired", title="Is Retired")
- isspamlist: Optional[bool] = Field(None, description="Whether the breach is a spam list", title="Is Spam List")
- ismalware: Optional[bool] = Field(None, description="Whether the breach is related to malware", title="Is Malware")
- isstealerlog: Optional[bool] = Field(None, description="Whether the breach is a stealer log", title="Is Stealer Log")
- issubscriptionfree: Optional[bool] = Field(None, description="Whether the breach is subscription free", title="Is Subscription Free")
- breach: Optional[Dict] = Field(None, description="Full breach data as returned by the API", title="Full Breach Data")
\ No newline at end of file
+
+ name: str = Field(
+ ..., description="The name of the breach or service", title="Breach Name"
+ )
+ title: Optional[str] = Field(
+ None, description="Title of the breach", title="Breach Title"
+ )
+ domain: Optional[str] = Field(
+ None, description="Domain of the breached service", title="Domain"
+ )
+ breachdate: Optional[str] = Field(
+ None, description="Date of the breach", title="Breach Date"
+ )
+ addeddate: Optional[str] = Field(
+ None, description="Date breach was added to HIBP", title="Added Date"
+ )
+ modifieddate: Optional[str] = Field(
+ None, description="Date breach was last modified", title="Modified Date"
+ )
+ pwncount: Optional[int] = Field(
+ None, description="Number of accounts affected", title="Affected Accounts"
+ )
+ description: Optional[str] = Field(
+ None, description="Description of the breach", title="Description"
+ )
+ logopath: Optional[str] = Field(
+ None, description="Logo path for the breach", title="Logo Path"
+ )
+ dataclasses: Optional[List[str]] = Field(
+ None, description="Types of data compromised", title="Data Classes"
+ )
+ isverified: Optional[bool] = Field(
+ None, description="Whether the breach is verified", title="Is Verified"
+ )
+ isfabricated: Optional[bool] = Field(
+ None, description="Whether the breach is fabricated", title="Is Fabricated"
+ )
+ issensitive: Optional[bool] = Field(
+ None, description="Whether the breach is sensitive", title="Is Sensitive"
+ )
+ isretired: Optional[bool] = Field(
+ None, description="Whether the breach is retired", title="Is Retired"
+ )
+ isspamlist: Optional[bool] = Field(
+ None, description="Whether the breach is a spam list", title="Is Spam List"
+ )
+ ismalware: Optional[bool] = Field(
+ None, description="Whether the breach is related to malware", title="Is Malware"
+ )
+ isstealerlog: Optional[bool] = Field(
+ None, description="Whether the breach is a stealer log", title="Is Stealer Log"
+ )
+ issubscriptionfree: Optional[bool] = Field(
+ None,
+ description="Whether the breach is subscription free",
+ title="Is Subscription Free",
+ )
+ breach: Optional[Dict] = Field(
+ None,
+ description="Full breach data as returned by the API",
+ title="Full Breach Data",
+ )
diff --git a/flowsint-types/src/flowsint_types/cidr.py b/flowsint-types/src/flowsint_types/cidr.py
index ec58c351..5cbc24bb 100644
--- a/flowsint-types/src/flowsint_types/cidr.py
+++ b/flowsint-types/src/flowsint_types/cidr.py
@@ -1,5 +1,9 @@
from pydantic import BaseModel, IPvAnyNetwork, Field
+
class CIDR(BaseModel):
"""Represents a CIDR (Classless Inter-Domain Routing) network block."""
- network: IPvAnyNetwork = Field(..., description="CIDR block (e.g., 8.8.8.0/24)", title="Network Block")
+
+ network: IPvAnyNetwork = Field(
+ ..., description="CIDR block (e.g., 8.8.8.0/24)", title="Network Block"
+ )
diff --git a/flowsint-types/src/flowsint_types/credential.py b/flowsint-types/src/flowsint_types/credential.py
index 38094ad5..a7c75e9c 100644
--- a/flowsint-types/src/flowsint_types/credential.py
+++ b/flowsint-types/src/flowsint_types/credential.py
@@ -1,22 +1,64 @@
from pydantic import BaseModel, Field
from typing import Optional, List
+
class Credential(BaseModel):
"""Represents user credentials with compromise and usage information."""
+
username: str = Field(..., description="Username or identifier", title="Username")
- service: Optional[str] = Field(None, description="Service or platform where credential is used", title="Service")
- type: Optional[str] = Field(None, description="Type of credential (password, token, key, etc.)", title="Credential Type")
- first_seen: Optional[str] = Field(None, description="First time credential was observed", title="First Seen")
- last_seen: Optional[str] = Field(None, description="Last time credential was observed", title="Last Seen")
- is_active: Optional[bool] = Field(None, description="Whether credential is currently active", title="Is Active")
- is_compromised: Optional[bool] = Field(None, description="Whether credential has been compromised", title="Is Compromised")
- breach_source: Optional[str] = Field(None, description="Source of breach if compromised", title="Breach Source")
- password_hash: Optional[str] = Field(None, description="Hashed password (if available)", title="Password Hash")
- hash_type: Optional[str] = Field(None, description="Type of hash algorithm used", title="Hash Type")
- associated_emails: Optional[List[str]] = Field(None, description="Associated email addresses", title="Associated Emails")
- associated_phones: Optional[List[str]] = Field(None, description="Associated phone numbers", title="Associated Phones")
- ip_addresses: Optional[List[str]] = Field(None, description="IP addresses used with credential", title="IP Addresses")
- user_agents: Optional[List[str]] = Field(None, description="User agents used with credential", title="User Agents")
- locations: Optional[List[str]] = Field(None, description="Geographic locations where credential was used", title="Locations")
- description: Optional[str] = Field(None, description="Additional notes about credential", title="Description")
- source: Optional[str] = Field(None, description="Source of credential information", title="Source")
\ No newline at end of file
+ service: Optional[str] = Field(
+ None,
+ description="Service or platform where credential is used",
+ title="Service",
+ )
+ type: Optional[str] = Field(
+ None,
+ description="Type of credential (password, token, key, etc.)",
+ title="Credential Type",
+ )
+ first_seen: Optional[str] = Field(
+ None, description="First time credential was observed", title="First Seen"
+ )
+ last_seen: Optional[str] = Field(
+ None, description="Last time credential was observed", title="Last Seen"
+ )
+ is_active: Optional[bool] = Field(
+ None, description="Whether credential is currently active", title="Is Active"
+ )
+ is_compromised: Optional[bool] = Field(
+ None,
+ description="Whether credential has been compromised",
+ title="Is Compromised",
+ )
+ breach_source: Optional[str] = Field(
+ None, description="Source of breach if compromised", title="Breach Source"
+ )
+ password_hash: Optional[str] = Field(
+ None, description="Hashed password (if available)", title="Password Hash"
+ )
+ hash_type: Optional[str] = Field(
+ None, description="Type of hash algorithm used", title="Hash Type"
+ )
+ associated_emails: Optional[List[str]] = Field(
+ None, description="Associated email addresses", title="Associated Emails"
+ )
+ associated_phones: Optional[List[str]] = Field(
+ None, description="Associated phone numbers", title="Associated Phones"
+ )
+ ip_addresses: Optional[List[str]] = Field(
+ None, description="IP addresses used with credential", title="IP Addresses"
+ )
+ user_agents: Optional[List[str]] = Field(
+ None, description="User agents used with credential", title="User Agents"
+ )
+ locations: Optional[List[str]] = Field(
+ None,
+ description="Geographic locations where credential was used",
+ title="Locations",
+ )
+ description: Optional[str] = Field(
+ None, description="Additional notes about credential", title="Description"
+ )
+ source: Optional[str] = Field(
+ None, description="Source of credential information", title="Source"
+ )
diff --git a/flowsint-types/src/flowsint_types/credit_card.py b/flowsint-types/src/flowsint_types/credit_card.py
index a2ab78d1..f6ef1fc8 100644
--- a/flowsint-types/src/flowsint_types/credit_card.py
+++ b/flowsint-types/src/flowsint_types/credit_card.py
@@ -1,24 +1,58 @@
from pydantic import BaseModel, Field
from typing import Optional, List
+
class CreditCard(BaseModel):
"""Represents a credit card with financial details and security status."""
+
card_number: str = Field(..., description="Credit card number", title="Card Number")
- card_type: Optional[str] = Field(None, description="Type of card (Visa, Mastercard, etc.)", title="Card Type")
+ card_type: Optional[str] = Field(
+ None, description="Type of card (Visa, Mastercard, etc.)", title="Card Type"
+ )
issuer: Optional[str] = Field(None, description="Card issuer bank", title="Issuer")
- expiry_date: Optional[str] = Field(None, description="Card expiry date", title="Expiry Date")
+ expiry_date: Optional[str] = Field(
+ None, description="Card expiry date", title="Expiry Date"
+ )
cvv: Optional[str] = Field(None, description="Card verification value", title="CVV")
- cardholder_name: Optional[str] = Field(None, description="Cardholder name", title="Cardholder Name")
- billing_address: Optional[str] = Field(None, description="Billing address", title="Billing Address")
- credit_limit: Optional[float] = Field(None, description="Credit limit", title="Credit Limit")
- available_credit: Optional[float] = Field(None, description="Available credit", title="Available Credit")
- status: Optional[str] = Field(None, description="Card status (active, suspended, etc.)", title="Status")
- issued_date: Optional[str] = Field(None, description="Card issue date", title="Issued Date")
- is_virtual: Optional[bool] = Field(None, description="Whether card is virtual", title="Is Virtual")
- is_prepaid: Optional[bool] = Field(None, description="Whether card is prepaid", title="Is Prepaid")
- is_business: Optional[bool] = Field(None, description="Whether card is business card", title="Is Business")
- associated_accounts: Optional[List[str]] = Field(None, description="Associated bank accounts", title="Associated Accounts")
- source: Optional[str] = Field(None, description="Source of card information", title="Source")
- is_compromised: Optional[bool] = Field(None, description="Whether card has been compromised", title="Is Compromised")
- breach_source: Optional[str] = Field(None, description="Source of breach if compromised", title="Breach Source")
- last_used: Optional[str] = Field(None, description="Last time card was used", title="Last Used")
\ No newline at end of file
+ cardholder_name: Optional[str] = Field(
+ None, description="Cardholder name", title="Cardholder Name"
+ )
+ billing_address: Optional[str] = Field(
+ None, description="Billing address", title="Billing Address"
+ )
+ credit_limit: Optional[float] = Field(
+ None, description="Credit limit", title="Credit Limit"
+ )
+ available_credit: Optional[float] = Field(
+ None, description="Available credit", title="Available Credit"
+ )
+ status: Optional[str] = Field(
+ None, description="Card status (active, suspended, etc.)", title="Status"
+ )
+ issued_date: Optional[str] = Field(
+ None, description="Card issue date", title="Issued Date"
+ )
+ is_virtual: Optional[bool] = Field(
+ None, description="Whether card is virtual", title="Is Virtual"
+ )
+ is_prepaid: Optional[bool] = Field(
+ None, description="Whether card is prepaid", title="Is Prepaid"
+ )
+ is_business: Optional[bool] = Field(
+ None, description="Whether card is business card", title="Is Business"
+ )
+ associated_accounts: Optional[List[str]] = Field(
+ None, description="Associated bank accounts", title="Associated Accounts"
+ )
+ source: Optional[str] = Field(
+ None, description="Source of card information", title="Source"
+ )
+ is_compromised: Optional[bool] = Field(
+ None, description="Whether card has been compromised", title="Is Compromised"
+ )
+ breach_source: Optional[str] = Field(
+ None, description="Source of breach if compromised", title="Breach Source"
+ )
+ last_used: Optional[str] = Field(
+ None, description="Last time card was used", title="Last Used"
+ )
diff --git a/flowsint-types/src/flowsint_types/device.py b/flowsint-types/src/flowsint_types/device.py
index d2d04b99..829c7c15 100644
--- a/flowsint-types/src/flowsint_types/device.py
+++ b/flowsint-types/src/flowsint_types/device.py
@@ -1,25 +1,65 @@
from pydantic import BaseModel, Field
from typing import Optional, List
+
class Device(BaseModel):
"""Represents a device with hardware, software, and network information."""
- device_id: str = Field(..., description="Unique device identifier", title="Device ID")
- type: Optional[str] = Field(None, description="Type of device (mobile, desktop, server, etc.)", title="Device Type")
- manufacturer: Optional[str] = Field(None, description="Device manufacturer", title="Manufacturer")
+
+ device_id: str = Field(
+ ..., description="Unique device identifier", title="Device ID"
+ )
+ type: Optional[str] = Field(
+ None,
+ description="Type of device (mobile, desktop, server, etc.)",
+ title="Device Type",
+ )
+ manufacturer: Optional[str] = Field(
+ None, description="Device manufacturer", title="Manufacturer"
+ )
model: Optional[str] = Field(None, description="Device model", title="Model")
- os: Optional[str] = Field(None, description="Operating system", title="Operating System")
- os_version: Optional[str] = Field(None, description="Operating system version", title="OS Version")
+ os: Optional[str] = Field(
+ None, description="Operating system", title="Operating System"
+ )
+ os_version: Optional[str] = Field(
+ None, description="Operating system version", title="OS Version"
+ )
browser: Optional[str] = Field(None, description="Browser used", title="Browser")
- browser_version: Optional[str] = Field(None, description="Browser version", title="Browser Version")
- screen_resolution: Optional[str] = Field(None, description="Screen resolution", title="Screen Resolution")
- user_agent: Optional[str] = Field(None, description="User agent string", title="User Agent")
- mac_address: Optional[str] = Field(None, description="MAC address", title="MAC Address")
- ip_addresses: Optional[List[str]] = Field(None, description="IP addresses associated with device", title="IP Addresses")
- first_seen: Optional[str] = Field(None, description="First time device was observed", title="First Seen")
- last_seen: Optional[str] = Field(None, description="Last time device was observed", title="Last Seen")
- location: Optional[str] = Field(None, description="Geographic location", title="Location")
- is_mobile: Optional[bool] = Field(None, description="Whether device is mobile", title="Is Mobile")
- is_tablet: Optional[bool] = Field(None, description="Whether device is a tablet", title="Is Tablet")
- is_desktop: Optional[bool] = Field(None, description="Whether device is a desktop", title="Is Desktop")
- associated_users: Optional[List[str]] = Field(None, description="Users associated with device", title="Associated Users")
- source: Optional[str] = Field(None, description="Source of device information", title="Source")
\ No newline at end of file
+ browser_version: Optional[str] = Field(
+ None, description="Browser version", title="Browser Version"
+ )
+ screen_resolution: Optional[str] = Field(
+ None, description="Screen resolution", title="Screen Resolution"
+ )
+ user_agent: Optional[str] = Field(
+ None, description="User agent string", title="User Agent"
+ )
+ mac_address: Optional[str] = Field(
+ None, description="MAC address", title="MAC Address"
+ )
+ ip_addresses: Optional[List[str]] = Field(
+ None, description="IP addresses associated with device", title="IP Addresses"
+ )
+ first_seen: Optional[str] = Field(
+ None, description="First time device was observed", title="First Seen"
+ )
+ last_seen: Optional[str] = Field(
+ None, description="Last time device was observed", title="Last Seen"
+ )
+ location: Optional[str] = Field(
+ None, description="Geographic location", title="Location"
+ )
+ is_mobile: Optional[bool] = Field(
+ None, description="Whether device is mobile", title="Is Mobile"
+ )
+ is_tablet: Optional[bool] = Field(
+ None, description="Whether device is a tablet", title="Is Tablet"
+ )
+ is_desktop: Optional[bool] = Field(
+ None, description="Whether device is a desktop", title="Is Desktop"
+ )
+ associated_users: Optional[List[str]] = Field(
+ None, description="Users associated with device", title="Associated Users"
+ )
+ source: Optional[str] = Field(
+ None, description="Source of device information", title="Source"
+ )
diff --git a/flowsint-types/src/flowsint_types/dns_record.py b/flowsint-types/src/flowsint_types/dns_record.py
index bb5fb1f6..aa20a02c 100644
--- a/flowsint-types/src/flowsint_types/dns_record.py
+++ b/flowsint-types/src/flowsint_types/dns_record.py
@@ -1,20 +1,50 @@
from pydantic import BaseModel, Field
from typing import Optional, List
+
class DNSRecord(BaseModel):
"""Represents a DNS record with type, value, and security information."""
- record_type: str = Field(..., description="Type of DNS record (A, AAAA, CNAME, MX, etc.)", title="Record Type")
+
+ record_type: str = Field(
+ ...,
+ description="Type of DNS record (A, AAAA, CNAME, MX, etc.)",
+ title="Record Type",
+ )
name: str = Field(..., description="Domain name", title="Domain Name")
value: str = Field(..., description="Record value", title="Record Value")
ttl: Optional[int] = Field(None, description="Time to live in seconds", title="TTL")
- priority: Optional[int] = Field(None, description="Priority for MX records", title="Priority")
- first_seen: Optional[str] = Field(None, description="First time record was observed", title="First Seen")
- last_seen: Optional[str] = Field(None, description="Last time record was observed", title="Last Seen")
- is_active: Optional[bool] = Field(None, description="Whether record is currently active", title="Is Active")
- nameserver: Optional[str] = Field(None, description="Nameserver that provided the record", title="Nameserver")
- source: Optional[str] = Field(None, description="Source of DNS information", title="Source")
- associated_domains: Optional[List[str]] = Field(None, description="Related domain names", title="Associated Domains")
- description: Optional[str] = Field(None, description="Additional notes about the record", title="Description")
- is_suspicious: Optional[bool] = Field(None, description="Whether record is suspicious", title="Is Suspicious")
- malware_family: Optional[str] = Field(None, description="Malware family if record is malicious", title="Malware Family")
- threat_level: Optional[str] = Field(None, description="Threat level assessment", title="Threat Level")
\ No newline at end of file
+ priority: Optional[int] = Field(
+ None, description="Priority for MX records", title="Priority"
+ )
+ first_seen: Optional[str] = Field(
+ None, description="First time record was observed", title="First Seen"
+ )
+ last_seen: Optional[str] = Field(
+ None, description="Last time record was observed", title="Last Seen"
+ )
+ is_active: Optional[bool] = Field(
+ None, description="Whether record is currently active", title="Is Active"
+ )
+ nameserver: Optional[str] = Field(
+ None, description="Nameserver that provided the record", title="Nameserver"
+ )
+ source: Optional[str] = Field(
+ None, description="Source of DNS information", title="Source"
+ )
+ associated_domains: Optional[List[str]] = Field(
+ None, description="Related domain names", title="Associated Domains"
+ )
+ description: Optional[str] = Field(
+ None, description="Additional notes about the record", title="Description"
+ )
+ is_suspicious: Optional[bool] = Field(
+ None, description="Whether record is suspicious", title="Is Suspicious"
+ )
+ malware_family: Optional[str] = Field(
+ None,
+ description="Malware family if record is malicious",
+ title="Malware Family",
+ )
+ threat_level: Optional[str] = Field(
+ None, description="Threat level assessment", title="Threat Level"
+ )
diff --git a/flowsint-types/src/flowsint_types/document.py b/flowsint-types/src/flowsint_types/document.py
index 7c5c8bd6..f8e5f0d6 100644
--- a/flowsint-types/src/flowsint_types/document.py
+++ b/flowsint-types/src/flowsint_types/document.py
@@ -1,24 +1,58 @@
from pydantic import BaseModel, Field
from typing import Optional, List
+
class Document(BaseModel):
"""Represents a document with metadata, security, and content information."""
+
title: str = Field(..., description="Document title", title="Title")
- type: Optional[str] = Field(None, description="Type of document (PDF, DOC, etc.)", title="Document Type")
- file_size: Optional[int] = Field(None, description="File size in bytes", title="File Size")
- created_date: Optional[str] = Field(None, description="Document creation date", title="Created Date")
- modified_date: Optional[str] = Field(None, description="Document last modified date", title="Modified Date")
+ type: Optional[str] = Field(
+ None, description="Type of document (PDF, DOC, etc.)", title="Document Type"
+ )
+ file_size: Optional[int] = Field(
+ None, description="File size in bytes", title="File Size"
+ )
+ created_date: Optional[str] = Field(
+ None, description="Document creation date", title="Created Date"
+ )
+ modified_date: Optional[str] = Field(
+ None, description="Document last modified date", title="Modified Date"
+ )
author: Optional[str] = Field(None, description="Document author", title="Author")
- subject: Optional[str] = Field(None, description="Document subject", title="Subject")
- keywords: Optional[List[str]] = Field(None, description="Document keywords", title="Keywords")
- language: Optional[str] = Field(None, description="Document language", title="Language")
- page_count: Optional[int] = Field(None, description="Number of pages", title="Page Count")
+ subject: Optional[str] = Field(
+ None, description="Document subject", title="Subject"
+ )
+ keywords: Optional[List[str]] = Field(
+ None, description="Document keywords", title="Keywords"
+ )
+ language: Optional[str] = Field(
+ None, description="Document language", title="Language"
+ )
+ page_count: Optional[int] = Field(
+ None, description="Number of pages", title="Page Count"
+ )
hash_md5: Optional[str] = Field(None, description="MD5 hash", title="MD5 Hash")
hash_sha1: Optional[str] = Field(None, description="SHA1 hash", title="SHA1 Hash")
- hash_sha256: Optional[str] = Field(None, description="SHA256 hash", title="SHA256 Hash")
- source_url: Optional[str] = Field(None, description="Source URL if downloaded", title="Source URL")
- location: Optional[str] = Field(None, description="File location or path", title="Location")
- is_encrypted: Optional[bool] = Field(None, description="Whether document is encrypted", title="Is Encrypted")
- is_password_protected: Optional[bool] = Field(None, description="Whether document is password protected", title="Is Password Protected")
- description: Optional[str] = Field(None, description="Document description", title="Description")
- source: Optional[str] = Field(None, description="Source of document information", title="Source")
\ No newline at end of file
+ hash_sha256: Optional[str] = Field(
+ None, description="SHA256 hash", title="SHA256 Hash"
+ )
+ source_url: Optional[str] = Field(
+ None, description="Source URL if downloaded", title="Source URL"
+ )
+ location: Optional[str] = Field(
+ None, description="File location or path", title="Location"
+ )
+ is_encrypted: Optional[bool] = Field(
+ None, description="Whether document is encrypted", title="Is Encrypted"
+ )
+ is_password_protected: Optional[bool] = Field(
+ None,
+ description="Whether document is password protected",
+ title="Is Password Protected",
+ )
+ description: Optional[str] = Field(
+ None, description="Document description", title="Description"
+ )
+ source: Optional[str] = Field(
+ None, description="Source of document information", title="Source"
+ )
diff --git a/flowsint-types/src/flowsint_types/domain.py b/flowsint-types/src/flowsint_types/domain.py
index eaba0822..62231616 100644
--- a/flowsint-types/src/flowsint_types/domain.py
+++ b/flowsint-types/src/flowsint_types/domain.py
@@ -1,10 +1,14 @@
from typing import Optional
from pydantic import BaseModel, Field
+
class Domain(BaseModel):
"""Represents a domain name and its properties."""
+
domain: str = Field(..., description="Domain name", title="Domain Name")
- root: Optional[bool]= Field(True, description="Is root or not", title="Is Root Domain")
-
-Domain.model_rebuild()
+ root: Optional[bool] = Field(
+ True, description="Is root or not", title="Is Root Domain"
+ )
+
+Domain.model_rebuild()
diff --git a/flowsint-types/src/flowsint_types/email.py b/flowsint-types/src/flowsint_types/email.py
index 9a4818e4..529d9897 100644
--- a/flowsint-types/src/flowsint_types/email.py
+++ b/flowsint-types/src/flowsint_types/email.py
@@ -1,5 +1,7 @@
from pydantic import BaseModel, Field
+
class Email(BaseModel):
"""Represents an email address."""
- email: str = Field(..., description="Email address", title="Email Address")
\ No newline at end of file
+
+ email: str = Field(..., description="Email address", title="Email Address")
diff --git a/flowsint-types/src/flowsint_types/file.py b/flowsint-types/src/flowsint_types/file.py
index 046264d2..608ec3a3 100644
--- a/flowsint-types/src/flowsint_types/file.py
+++ b/flowsint-types/src/flowsint_types/file.py
@@ -1,27 +1,63 @@
from pydantic import BaseModel, Field
from typing import Optional, List
+
class File(BaseModel):
"""Represents a file with metadata, type information, and security assessment."""
+
filename: str = Field(..., description="File name", title="Filename")
- file_type: Optional[str] = Field(None, description="File type or extension", title="File Type")
- file_size: Optional[int] = Field(None, description="File size in bytes", title="File Size")
- created_date: Optional[str] = Field(None, description="File creation date", title="Created Date")
- modified_date: Optional[str] = Field(None, description="File last modified date", title="Modified Date")
- accessed_date: Optional[str] = Field(None, description="File last accessed date", title="Accessed Date")
+ file_type: Optional[str] = Field(
+ None, description="File type or extension", title="File Type"
+ )
+ file_size: Optional[int] = Field(
+ None, description="File size in bytes", title="File Size"
+ )
+ created_date: Optional[str] = Field(
+ None, description="File creation date", title="Created Date"
+ )
+ modified_date: Optional[str] = Field(
+ None, description="File last modified date", title="Modified Date"
+ )
+ accessed_date: Optional[str] = Field(
+ None, description="File last accessed date", title="Accessed Date"
+ )
path: Optional[str] = Field(None, description="File path", title="Path")
hash_md5: Optional[str] = Field(None, description="MD5 hash", title="MD5 Hash")
hash_sha1: Optional[str] = Field(None, description="SHA1 hash", title="SHA1 Hash")
- hash_sha256: Optional[str] = Field(None, description="SHA256 hash", title="SHA256 Hash")
- is_executable: Optional[bool] = Field(None, description="Whether file is executable", title="Is Executable")
- is_archive: Optional[bool] = Field(None, description="Whether file is an archive", title="Is Archive")
- is_image: Optional[bool] = Field(None, description="Whether file is an image", title="Is Image")
- is_video: Optional[bool] = Field(None, description="Whether file is a video", title="Is Video")
- is_audio: Optional[bool] = Field(None, description="Whether file is an audio file", title="Is Audio")
- is_text: Optional[bool] = Field(None, description="Whether file is a text file", title="Is Text")
+ hash_sha256: Optional[str] = Field(
+ None, description="SHA256 hash", title="SHA256 Hash"
+ )
+ is_executable: Optional[bool] = Field(
+ None, description="Whether file is executable", title="Is Executable"
+ )
+ is_archive: Optional[bool] = Field(
+ None, description="Whether file is an archive", title="Is Archive"
+ )
+ is_image: Optional[bool] = Field(
+ None, description="Whether file is an image", title="Is Image"
+ )
+ is_video: Optional[bool] = Field(
+ None, description="Whether file is a video", title="Is Video"
+ )
+ is_audio: Optional[bool] = Field(
+ None, description="Whether file is an audio file", title="Is Audio"
+ )
+ is_text: Optional[bool] = Field(
+ None, description="Whether file is a text file", title="Is Text"
+ )
mime_type: Optional[str] = Field(None, description="MIME type", title="MIME Type")
- description: Optional[str] = Field(None, description="File description", title="Description")
- source: Optional[str] = Field(None, description="Source of file information", title="Source")
- is_malicious: Optional[bool] = Field(None, description="Whether file is malicious", title="Is Malicious")
- malware_family: Optional[str] = Field(None, description="Malware family if malicious", title="Malware Family")
- threat_level: Optional[str] = Field(None, description="Threat level assessment", title="Threat Level")
\ No newline at end of file
+ description: Optional[str] = Field(
+ None, description="File description", title="Description"
+ )
+ source: Optional[str] = Field(
+ None, description="Source of file information", title="Source"
+ )
+ is_malicious: Optional[bool] = Field(
+ None, description="Whether file is malicious", title="Is Malicious"
+ )
+ malware_family: Optional[str] = Field(
+ None, description="Malware family if malicious", title="Malware Family"
+ )
+ threat_level: Optional[str] = Field(
+ None, description="Threat level assessment", title="Threat Level"
+ )
diff --git a/flowsint-types/src/flowsint_types/gravatar.py b/flowsint-types/src/flowsint_types/gravatar.py
index eddc149a..cc1cb059 100644
--- a/flowsint-types/src/flowsint_types/gravatar.py
+++ b/flowsint-types/src/flowsint_types/gravatar.py
@@ -1,18 +1,44 @@
from pydantic import BaseModel, Field, HttpUrl
from typing import Optional, List
+
class Gravatar(BaseModel):
"""Represents a Gravatar profile with image and user information."""
+
src: HttpUrl = Field(..., description="Gravatar image URL", title="Image URL")
hash: str = Field(..., description="Gravatar hash", title="Hash")
- size: Optional[int] = Field(None, description="Image size in pixels", title="Image Size")
- rating: Optional[str] = Field(None, description="Content rating (g, pg, r, x)", title="Content Rating")
- default_image: Optional[str] = Field(None, description="Default image type when no gravatar exists", title="Default Image")
- force_default: Optional[bool] = Field(None, description="Whether to force default image", title="Force Default")
- exists: Optional[bool] = Field(None, description="Whether the gravatar exists", title="Exists")
- profile_url: Optional[HttpUrl] = Field(None, description="URL to the Gravatar profile page", title="Profile URL")
- display_name: Optional[str] = Field(None, description="Display name from Gravatar profile", title="Display Name")
- location: Optional[str] = Field(None, description="Location from Gravatar profile", title="Location")
- about_me: Optional[str] = Field(None, description="Bio/about me text from Gravatar profile", title="About Me")
- thumbnail_url: Optional[HttpUrl] = Field(None, description="Smaller version of the image", title="Thumbnail URL")
- large_url: Optional[HttpUrl] = Field(None, description="Larger version of the image", title="Large Image URL")
+ size: Optional[int] = Field(
+ None, description="Image size in pixels", title="Image Size"
+ )
+ rating: Optional[str] = Field(
+ None, description="Content rating (g, pg, r, x)", title="Content Rating"
+ )
+ default_image: Optional[str] = Field(
+ None,
+ description="Default image type when no gravatar exists",
+ title="Default Image",
+ )
+ force_default: Optional[bool] = Field(
+ None, description="Whether to force default image", title="Force Default"
+ )
+ exists: Optional[bool] = Field(
+ None, description="Whether the gravatar exists", title="Exists"
+ )
+ profile_url: Optional[HttpUrl] = Field(
+ None, description="URL to the Gravatar profile page", title="Profile URL"
+ )
+ display_name: Optional[str] = Field(
+ None, description="Display name from Gravatar profile", title="Display Name"
+ )
+ location: Optional[str] = Field(
+ None, description="Location from Gravatar profile", title="Location"
+ )
+ about_me: Optional[str] = Field(
+ None, description="Bio/about me text from Gravatar profile", title="About Me"
+ )
+ thumbnail_url: Optional[HttpUrl] = Field(
+ None, description="Smaller version of the image", title="Thumbnail URL"
+ )
+ large_url: Optional[HttpUrl] = Field(
+ None, description="Larger version of the image", title="Large Image URL"
+ )
diff --git a/flowsint-types/src/flowsint_types/individual.py b/flowsint-types/src/flowsint_types/individual.py
index a7693d32..469cfe48 100644
--- a/flowsint-types/src/flowsint_types/individual.py
+++ b/flowsint-types/src/flowsint_types/individual.py
@@ -2,118 +2,307 @@ from pydantic import BaseModel, Field
from typing import Optional, Literal, List
from .address import PhysicalAddress
+
class Individual(BaseModel):
"""Represents an individual person with comprehensive personal information."""
+
# Basic Information
- first_name: str = Field(..., description="First name of the individual", title="First Name")
- last_name: str = Field(..., description="Last name of the individual", title="Last Name")
- full_name: str = Field(..., description="Full name of the individual", title="Full Name")
- middle_name: Optional[str] = Field(None, description="Middle name or initial", title="Middle Name")
- maiden_name: Optional[str] = Field(None, description="Maiden name (if applicable)", title="Maiden Name")
- aliases: Optional[List[str]] = Field(None, description="Known aliases or nicknames", title="Aliases")
-
+ first_name: str = Field(
+ ..., description="First name of the individual", title="First Name"
+ )
+ last_name: str = Field(
+ ..., description="Last name of the individual", title="Last Name"
+ )
+ full_name: str = Field(
+ ..., description="Full name of the individual", title="Full Name"
+ )
+ middle_name: Optional[str] = Field(
+ None, description="Middle name or initial", title="Middle Name"
+ )
+ maiden_name: Optional[str] = Field(
+ None, description="Maiden name (if applicable)", title="Maiden Name"
+ )
+ aliases: Optional[List[str]] = Field(
+ None, description="Known aliases or nicknames", title="Aliases"
+ )
+
# Birth Information
- birth_date: Optional[str] = Field(None, description="Date of birth", title="Date of Birth")
- birth_place: Optional[PhysicalAddress] = Field(None, description="Place of birth", title="Birth Place")
+ birth_date: Optional[str] = Field(
+ None, description="Date of birth", title="Date of Birth"
+ )
+ birth_place: Optional[PhysicalAddress] = Field(
+ None, description="Place of birth", title="Birth Place"
+ )
age: Optional[int] = Field(None, description="Current age", title="Age")
-
+
# Physical Characteristics
- gender: Optional[Literal["male", "female", "other"]] = Field(None, description="Gender of the individual", title="Gender")
- height: Optional[str] = Field(None, description="Height (e.g., '5ft 10in' or '178cm')", title="Height")
- weight: Optional[str] = Field(None, description="Weight (e.g., '170lbs' or '77kg')", title="Weight")
- eye_color: Optional[Literal["brown", "blue", "green", "hazel", "gray", "amber", "other"]] = Field(None, description="Eye color", title="Eye Color")
- hair_color: Optional[Literal["black", "brown", "blonde", "red", "gray", "white", "bald", "other"]] = Field(None, description="Hair color", title="Hair Color")
- skin_tone: Optional[str] = Field(None, description="Skin tone description", title="Skin Tone")
- ethnicity: Optional[str] = Field(None, description="Ethnicity or ethnic background", title="Ethnicity")
- blood_type: Optional[Literal["A+", "A-", "B+", "B-", "AB+", "AB-", "O+", "O-"]] = Field(None, description="Blood type", title="Blood Type")
- distinguishing_marks: Optional[List[str]] = Field(None, description="Distinguishing marks, scars, tattoos, etc.", title="Distinguishing Marks")
- disabilities: Optional[List[str]] = Field(None, description="Known disabilities or medical conditions", title="Disabilities")
-
+ gender: Optional[Literal["male", "female", "other"]] = Field(
+ None, description="Gender of the individual", title="Gender"
+ )
+ height: Optional[str] = Field(
+ None, description="Height (e.g., '5ft 10in' or '178cm')", title="Height"
+ )
+ weight: Optional[str] = Field(
+ None, description="Weight (e.g., '170lbs' or '77kg')", title="Weight"
+ )
+ eye_color: Optional[
+ Literal["brown", "blue", "green", "hazel", "gray", "amber", "other"]
+ ] = Field(None, description="Eye color", title="Eye Color")
+ hair_color: Optional[
+ Literal["black", "brown", "blonde", "red", "gray", "white", "bald", "other"]
+ ] = Field(None, description="Hair color", title="Hair Color")
+ skin_tone: Optional[str] = Field(
+ None, description="Skin tone description", title="Skin Tone"
+ )
+ ethnicity: Optional[str] = Field(
+ None, description="Ethnicity or ethnic background", title="Ethnicity"
+ )
+ blood_type: Optional[Literal["A+", "A-", "B+", "B-", "AB+", "AB-", "O+", "O-"]] = (
+ Field(None, description="Blood type", title="Blood Type")
+ )
+ distinguishing_marks: Optional[List[str]] = Field(
+ None,
+ description="Distinguishing marks, scars, tattoos, etc.",
+ title="Distinguishing Marks",
+ )
+ disabilities: Optional[List[str]] = Field(
+ None,
+ description="Known disabilities or medical conditions",
+ title="Disabilities",
+ )
+
# Health and Lifestyle
- is_smoker: Optional[bool] = Field(None, description="Whether the individual smokes", title="Is Smoker")
- is_deceased: Optional[bool] = Field(None, description="Whether the individual is deceased", title="Is Deceased")
- death_date: Optional[str] = Field(None, description="Date of death (if applicable)", title="Death Date")
- death_place: Optional[PhysicalAddress] = Field(None, description="Place of death (if applicable)", title="Death Place")
- cause_of_death: Optional[str] = Field(None, description="Cause of death (if applicable)", title="Cause of Death")
-
+ is_smoker: Optional[bool] = Field(
+ None, description="Whether the individual smokes", title="Is Smoker"
+ )
+ is_deceased: Optional[bool] = Field(
+ None, description="Whether the individual is deceased", title="Is Deceased"
+ )
+ death_date: Optional[str] = Field(
+ None, description="Date of death (if applicable)", title="Death Date"
+ )
+ death_place: Optional[PhysicalAddress] = Field(
+ None, description="Place of death (if applicable)", title="Death Place"
+ )
+ cause_of_death: Optional[str] = Field(
+ None, description="Cause of death (if applicable)", title="Cause of Death"
+ )
+
# Contact Information
- phone_numbers: Optional[List[str]] = Field(None, description="Known phone numbers", title="Phone Numbers")
- email_addresses: Optional[List[str]] = Field(None, description="Known email addresses", title="Email Addresses")
- social_media_profiles: Optional[List[str]] = Field(None, description="Social media profile URLs or usernames", title="Social Media Profiles")
-
+ phone_numbers: Optional[List[str]] = Field(
+ None, description="Known phone numbers", title="Phone Numbers"
+ )
+ email_addresses: Optional[List[str]] = Field(
+ None, description="Known email addresses", title="Email Addresses"
+ )
+ social_media_profiles: Optional[List[str]] = Field(
+ None,
+ description="Social media profile URLs or usernames",
+ title="Social Media Profiles",
+ )
+
# Location Information
- current_address: Optional[PhysicalAddress] = Field(None, description="Current residential address", title="Current Address")
- previous_addresses: Optional[List[PhysicalAddress]] = Field(None, description="Previous known addresses", title="Previous Addresses")
- nationality: Optional[str] = Field(None, description="Nationality or country of citizenship", title="Nationality")
- citizenship: Optional[List[str]] = Field(None, description="Countries of citizenship (dual citizenship possible)", title="Citizenship")
- place_of_residence: Optional[str] = Field(None, description="Current place of residence (city/state/country)", title="Place of Residence")
-
+ current_address: Optional[PhysicalAddress] = Field(
+ None, description="Current residential address", title="Current Address"
+ )
+ previous_addresses: Optional[List[PhysicalAddress]] = Field(
+ None, description="Previous known addresses", title="Previous Addresses"
+ )
+ nationality: Optional[str] = Field(
+ None, description="Nationality or country of citizenship", title="Nationality"
+ )
+ citizenship: Optional[List[str]] = Field(
+ None,
+ description="Countries of citizenship (dual citizenship possible)",
+ title="Citizenship",
+ )
+ place_of_residence: Optional[str] = Field(
+ None,
+ description="Current place of residence (city/state/country)",
+ title="Place of Residence",
+ )
+
# Identification Documents
- passport_number: Optional[str] = Field(None, description="Passport number", title="Passport Number")
- drivers_license: Optional[str] = Field(None, description="Driver's license number", title="Driver's License")
- social_security_number: Optional[str] = Field(None, description="Social Security Number (SSN)", title="Social Security Number")
- national_id: Optional[str] = Field(None, description="National ID number", title="National ID")
- voter_id: Optional[str] = Field(None, description="Voter ID number", title="Voter ID")
- tax_id: Optional[str] = Field(None, description="Tax identification number", title="Tax ID")
-
+ passport_number: Optional[str] = Field(
+ None, description="Passport number", title="Passport Number"
+ )
+ drivers_license: Optional[str] = Field(
+ None, description="Driver's license number", title="Driver's License"
+ )
+ social_security_number: Optional[str] = Field(
+ None, description="Social Security Number (SSN)", title="Social Security Number"
+ )
+ national_id: Optional[str] = Field(
+ None, description="National ID number", title="National ID"
+ )
+ voter_id: Optional[str] = Field(
+ None, description="Voter ID number", title="Voter ID"
+ )
+ tax_id: Optional[str] = Field(
+ None, description="Tax identification number", title="Tax ID"
+ )
+
# Personal Relationships
- marital_status: Optional[Literal["single", "married", "divorced", "widowed", "separated", "domestic_partnership"]] = Field(None, description="Marital status", title="Marital Status")
- spouse_name: Optional[str] = Field(None, description="Name of spouse or partner", title="Spouse Name")
- children_count: Optional[int] = Field(None, description="Number of children", title="Children Count")
- children_names: Optional[List[str]] = Field(None, description="Names of children", title="Children Names")
- parents_names: Optional[List[str]] = Field(None, description="Names of parents", title="Parents Names")
- emergency_contact: Optional[str] = Field(None, description="Emergency contact information", title="Emergency Contact")
-
+ marital_status: Optional[
+ Literal[
+ "single",
+ "married",
+ "divorced",
+ "widowed",
+ "separated",
+ "domestic_partnership",
+ ]
+ ] = Field(None, description="Marital status", title="Marital Status")
+ spouse_name: Optional[str] = Field(
+ None, description="Name of spouse or partner", title="Spouse Name"
+ )
+ children_count: Optional[int] = Field(
+ None, description="Number of children", title="Children Count"
+ )
+ children_names: Optional[List[str]] = Field(
+ None, description="Names of children", title="Children Names"
+ )
+ parents_names: Optional[List[str]] = Field(
+ None, description="Names of parents", title="Parents Names"
+ )
+ emergency_contact: Optional[str] = Field(
+ None, description="Emergency contact information", title="Emergency Contact"
+ )
+
# Professional Information
- occupation: Optional[str] = Field(None, description="Current occupation or job title", title="Occupation")
- employer: Optional[str] = Field(None, description="Current employer", title="Employer")
- employer_address: Optional[PhysicalAddress] = Field(None, description="Employer's address", title="Employer Address")
- job_title: Optional[str] = Field(None, description="Current job title or position", title="Job Title")
- annual_income: Optional[float] = Field(None, description="Annual income", title="Annual Income")
- employment_history: Optional[List[str]] = Field(None, description="Previous employment history", title="Employment History")
-
+ occupation: Optional[str] = Field(
+ None, description="Current occupation or job title", title="Occupation"
+ )
+ employer: Optional[str] = Field(
+ None, description="Current employer", title="Employer"
+ )
+ employer_address: Optional[PhysicalAddress] = Field(
+ None, description="Employer's address", title="Employer Address"
+ )
+ job_title: Optional[str] = Field(
+ None, description="Current job title or position", title="Job Title"
+ )
+ annual_income: Optional[float] = Field(
+ None, description="Annual income", title="Annual Income"
+ )
+ employment_history: Optional[List[str]] = Field(
+ None, description="Previous employment history", title="Employment History"
+ )
+
# Education
- education_level: Optional[Literal["elementary", "high_school", "bachelor", "master", "doctorate", "other"]] = Field(None, description="Highest education level achieved", title="Education Level")
- schools_attended: Optional[List[str]] = Field(None, description="Schools or universities attended", title="Schools Attended")
- degrees: Optional[List[str]] = Field(None, description="Degrees or certifications obtained", title="Degrees")
-
+ education_level: Optional[
+ Literal["elementary", "high_school", "bachelor", "master", "doctorate", "other"]
+ ] = Field(
+ None, description="Highest education level achieved", title="Education Level"
+ )
+ schools_attended: Optional[List[str]] = Field(
+ None, description="Schools or universities attended", title="Schools Attended"
+ )
+ degrees: Optional[List[str]] = Field(
+ None, description="Degrees or certifications obtained", title="Degrees"
+ )
+
# Personal Characteristics
- languages_spoken: Optional[List[str]] = Field(None, description="Languages spoken", title="Languages Spoken")
- religion: Optional[str] = Field(None, description="Religious affiliation", title="Religion")
- political_affiliation: Optional[str] = Field(None, description="Political party affiliation", title="Political Affiliation")
- hobbies: Optional[List[str]] = Field(None, description="Known hobbies or interests", title="Hobbies")
- personality_traits: Optional[List[str]] = Field(None, description="Known personality traits", title="Personality Traits")
-
+ languages_spoken: Optional[List[str]] = Field(
+ None, description="Languages spoken", title="Languages Spoken"
+ )
+ religion: Optional[str] = Field(
+ None, description="Religious affiliation", title="Religion"
+ )
+ political_affiliation: Optional[str] = Field(
+ None, description="Political party affiliation", title="Political Affiliation"
+ )
+ hobbies: Optional[List[str]] = Field(
+ None, description="Known hobbies or interests", title="Hobbies"
+ )
+ personality_traits: Optional[List[str]] = Field(
+ None, description="Known personality traits", title="Personality Traits"
+ )
+
# Biometric Information
- fingerprints: Optional[str] = Field(None, description="Fingerprint data or hash", title="Fingerprints")
- dna_profile: Optional[str] = Field(None, description="DNA profile or hash", title="DNA Profile")
- facial_recognition_id: Optional[str] = Field(None, description="Facial recognition system ID", title="Facial Recognition ID")
- iris_scan: Optional[str] = Field(None, description="Iris scan data or hash", title="Iris Scan")
- voice_print: Optional[str] = Field(None, description="Voice print data or hash", title="Voice Print")
-
+ fingerprints: Optional[str] = Field(
+ None, description="Fingerprint data or hash", title="Fingerprints"
+ )
+ dna_profile: Optional[str] = Field(
+ None, description="DNA profile or hash", title="DNA Profile"
+ )
+ facial_recognition_id: Optional[str] = Field(
+ None, description="Facial recognition system ID", title="Facial Recognition ID"
+ )
+ iris_scan: Optional[str] = Field(
+ None, description="Iris scan data or hash", title="Iris Scan"
+ )
+ voice_print: Optional[str] = Field(
+ None, description="Voice print data or hash", title="Voice Print"
+ )
+
# Financial Information
- bank_accounts: Optional[List[str]] = Field(None, description="Known bank account numbers", title="Bank Accounts")
- credit_score: Optional[int] = Field(None, description="Credit score", title="Credit Score")
- net_worth: Optional[float] = Field(None, description="Estimated net worth", title="Net Worth")
- assets: Optional[List[str]] = Field(None, description="Known assets (property, vehicles, etc.)", title="Assets")
-
+ bank_accounts: Optional[List[str]] = Field(
+ None, description="Known bank account numbers", title="Bank Accounts"
+ )
+ credit_score: Optional[int] = Field(
+ None, description="Credit score", title="Credit Score"
+ )
+ net_worth: Optional[float] = Field(
+ None, description="Estimated net worth", title="Net Worth"
+ )
+ assets: Optional[List[str]] = Field(
+ None, description="Known assets (property, vehicles, etc.)", title="Assets"
+ )
+
# Legal Information
- criminal_record: Optional[bool] = Field(None, description="Whether individual has a criminal record", title="Criminal Record")
- arrest_history: Optional[List[str]] = Field(None, description="Known arrests or charges", title="Arrest History")
- convictions: Optional[List[str]] = Field(None, description="Criminal convictions", title="Convictions")
- civil_cases: Optional[List[str]] = Field(None, description="Civil court cases", title="Civil Cases")
- security_clearance: Optional[str] = Field(None, description="Security clearance level", title="Security Clearance")
-
+ criminal_record: Optional[bool] = Field(
+ None,
+ description="Whether individual has a criminal record",
+ title="Criminal Record",
+ )
+ arrest_history: Optional[List[str]] = Field(
+ None, description="Known arrests or charges", title="Arrest History"
+ )
+ convictions: Optional[List[str]] = Field(
+ None, description="Criminal convictions", title="Convictions"
+ )
+ civil_cases: Optional[List[str]] = Field(
+ None, description="Civil court cases", title="Civil Cases"
+ )
+ security_clearance: Optional[str] = Field(
+ None, description="Security clearance level", title="Security Clearance"
+ )
+
# Digital Footprint
- ip_addresses: Optional[List[str]] = Field(None, description="Known IP addresses", title="IP Addresses")
- usernames: Optional[List[str]] = Field(None, description="Known online usernames", title="Usernames")
- device_ids: Optional[List[str]] = Field(None, description="Known device identifiers", title="Device IDs")
-
+ ip_addresses: Optional[List[str]] = Field(
+ None, description="Known IP addresses", title="IP Addresses"
+ )
+ usernames: Optional[List[str]] = Field(
+ None, description="Known online usernames", title="Usernames"
+ )
+ device_ids: Optional[List[str]] = Field(
+ None, description="Known device identifiers", title="Device IDs"
+ )
+
# Metadata
- first_seen: Optional[str] = Field(None, description="First time individual was observed in system", title="First Seen")
- last_seen: Optional[str] = Field(None, description="Last time individual was observed in system", title="Last Seen")
- source: Optional[str] = Field(None, description="Source of individual information", title="Source")
- confidence: Optional[float] = Field(None, description="Confidence score for individual data", title="Confidence")
- notes: Optional[str] = Field(None, description="Additional notes or observations", title="Notes")
- src: Optional[str] = Field(None, description="URL to profile picture", title="Profile Picture URL")
- last_updated: Optional[str] = Field(None, description="Last update timestamp", title="Last Updated")
\ No newline at end of file
+ first_seen: Optional[str] = Field(
+ None,
+ description="First time individual was observed in system",
+ title="First Seen",
+ )
+ last_seen: Optional[str] = Field(
+ None,
+ description="Last time individual was observed in system",
+ title="Last Seen",
+ )
+ source: Optional[str] = Field(
+ None, description="Source of individual information", title="Source"
+ )
+ confidence: Optional[float] = Field(
+ None, description="Confidence score for individual data", title="Confidence"
+ )
+ notes: Optional[str] = Field(
+ None, description="Additional notes or observations", title="Notes"
+ )
+ src: Optional[str] = Field(
+ None, description="URL to profile picture", title="Profile Picture URL"
+ )
+ last_updated: Optional[str] = Field(
+ None, description="Last update timestamp", title="Last Updated"
+ )
diff --git a/flowsint-types/src/flowsint_types/ip.py b/flowsint-types/src/flowsint_types/ip.py
index f3fe3d6f..d40c6cd2 100644
--- a/flowsint-types/src/flowsint_types/ip.py
+++ b/flowsint-types/src/flowsint_types/ip.py
@@ -1,11 +1,23 @@
from pydantic import BaseModel, Field
from typing import Optional
+
class Ip(BaseModel):
"""Represents an IP address with geolocation and ISP information."""
+
address: str = Field(..., description="IP address", title="IP Address")
- latitude: Optional[float] = Field(None, description="Latitude coordinate of the IP location", title="Latitude")
- longitude: Optional[float] = Field(None, description="Longitude coordinate of the IP location", title="Longitude")
- country: Optional[str] = Field(None, description="Country where the IP is located", title="Country")
- city: Optional[str] = Field(None, description="City where the IP is located", title="City")
- isp: Optional[str] = Field(None, description="Internet Service Provider", title="ISP")
+ latitude: Optional[float] = Field(
+ None, description="Latitude coordinate of the IP location", title="Latitude"
+ )
+ longitude: Optional[float] = Field(
+ None, description="Longitude coordinate of the IP location", title="Longitude"
+ )
+ country: Optional[str] = Field(
+ None, description="Country where the IP is located", title="Country"
+ )
+ city: Optional[str] = Field(
+ None, description="City where the IP is located", title="City"
+ )
+ isp: Optional[str] = Field(
+ None, description="Internet Service Provider", title="ISP"
+ )
diff --git a/flowsint-types/src/flowsint_types/leak.py b/flowsint-types/src/flowsint_types/leak.py
index 0aaf97e1..ec392304 100644
--- a/flowsint-types/src/flowsint_types/leak.py
+++ b/flowsint-types/src/flowsint_types/leak.py
@@ -1,7 +1,13 @@
from typing import List, Dict, Optional
from pydantic import BaseModel, Field
+
class Leak(BaseModel):
"""Represents a data leak or breach with associated data."""
- name: str = Field(..., description= "The name of the leak or service brea", title="Leak Name")
- leak: Optional[List[Dict]] = Field(None, description="List of data leaks found", title="Leak Data")
+
+ name: str = Field(
+ ..., description="The name of the leak or service brea", title="Leak Name"
+ )
+ leak: Optional[List[Dict]] = Field(
+ None, description="List of data leaks found", title="Leak Data"
+ )
diff --git a/flowsint-types/src/flowsint_types/malware.py b/flowsint-types/src/flowsint_types/malware.py
index 5e80d25f..b6c3c019 100644
--- a/flowsint-types/src/flowsint_types/malware.py
+++ b/flowsint-types/src/flowsint_types/malware.py
@@ -1,24 +1,60 @@
from pydantic import BaseModel, Field
from typing import Optional, List
+
class Malware(BaseModel):
"""Represents malware with family, capabilities, and threat intelligence."""
+
name: str = Field(..., description="Malware name or identifier", title="Name")
family: Optional[str] = Field(None, description="Malware family", title="Family")
- type: Optional[str] = Field(None, description="Type of malware (trojan, ransomware, etc.)", title="Type")
- description: Optional[str] = Field(None, description="Malware description", title="Description")
- first_seen: Optional[str] = Field(None, description="First time malware was observed", title="First Seen")
- last_seen: Optional[str] = Field(None, description="Last time malware was observed", title="Last Seen")
- is_active: Optional[bool] = Field(None, description="Whether malware is currently active", title="Is Active")
- threat_level: Optional[str] = Field(None, description="Threat level assessment", title="Threat Level")
- targets: Optional[List[str]] = Field(None, description="Target platforms or systems", title="Targets")
- capabilities: Optional[List[str]] = Field(None, description="Malware capabilities", title="Capabilities")
- indicators: Optional[List[str]] = Field(None, description="IOCs (Indicators of Compromise)", title="Indicators")
- c2_servers: Optional[List[str]] = Field(None, description="Command and control servers", title="C2 Servers")
- distribution_methods: Optional[List[str]] = Field(None, description="Distribution methods", title="Distribution Methods")
- evasion_techniques: Optional[List[str]] = Field(None, description="Evasion techniques used", title="Evasion Techniques")
- persistence_mechanisms: Optional[List[str]] = Field(None, description="Persistence mechanisms", title="Persistence Mechanisms")
- encryption_methods: Optional[List[str]] = Field(None, description="Encryption methods used", title="Encryption Methods")
- source: Optional[str] = Field(None, description="Source of malware information", title="Source")
- analysis_report: Optional[str] = Field(None, description="Link to analysis report", title="Analysis Report")
- sample_hashes: Optional[List[str]] = Field(None, description="Sample file hashes", title="Sample Hashes")
\ No newline at end of file
+ type: Optional[str] = Field(
+ None, description="Type of malware (trojan, ransomware, etc.)", title="Type"
+ )
+ description: Optional[str] = Field(
+ None, description="Malware description", title="Description"
+ )
+ first_seen: Optional[str] = Field(
+ None, description="First time malware was observed", title="First Seen"
+ )
+ last_seen: Optional[str] = Field(
+ None, description="Last time malware was observed", title="Last Seen"
+ )
+ is_active: Optional[bool] = Field(
+ None, description="Whether malware is currently active", title="Is Active"
+ )
+ threat_level: Optional[str] = Field(
+ None, description="Threat level assessment", title="Threat Level"
+ )
+ targets: Optional[List[str]] = Field(
+ None, description="Target platforms or systems", title="Targets"
+ )
+ capabilities: Optional[List[str]] = Field(
+ None, description="Malware capabilities", title="Capabilities"
+ )
+ indicators: Optional[List[str]] = Field(
+ None, description="IOCs (Indicators of Compromise)", title="Indicators"
+ )
+ c2_servers: Optional[List[str]] = Field(
+ None, description="Command and control servers", title="C2 Servers"
+ )
+ distribution_methods: Optional[List[str]] = Field(
+ None, description="Distribution methods", title="Distribution Methods"
+ )
+ evasion_techniques: Optional[List[str]] = Field(
+ None, description="Evasion techniques used", title="Evasion Techniques"
+ )
+ persistence_mechanisms: Optional[List[str]] = Field(
+ None, description="Persistence mechanisms", title="Persistence Mechanisms"
+ )
+ encryption_methods: Optional[List[str]] = Field(
+ None, description="Encryption methods used", title="Encryption Methods"
+ )
+ source: Optional[str] = Field(
+ None, description="Source of malware information", title="Source"
+ )
+ analysis_report: Optional[str] = Field(
+ None, description="Link to analysis report", title="Analysis Report"
+ )
+ sample_hashes: Optional[List[str]] = Field(
+ None, description="Sample file hashes", title="Sample Hashes"
+ )
diff --git a/flowsint-types/src/flowsint_types/message.py b/flowsint-types/src/flowsint_types/message.py
index b642b030..a16875e3 100644
--- a/flowsint-types/src/flowsint_types/message.py
+++ b/flowsint-types/src/flowsint_types/message.py
@@ -1,24 +1,62 @@
from pydantic import BaseModel, Field
from typing import Optional, List
+
class Message(BaseModel):
"""Represents a message with content, metadata, and security analysis."""
- message_id: str = Field(..., description="Unique message identifier", title="Message ID")
+
+ message_id: str = Field(
+ ..., description="Unique message identifier", title="Message ID"
+ )
content: str = Field(..., description="Message content", title="Content")
sender: Optional[str] = Field(None, description="Message sender", title="Sender")
- recipient: Optional[str] = Field(None, description="Message recipient", title="Recipient")
- timestamp: Optional[str] = Field(None, description="Message timestamp", title="Timestamp")
- platform: Optional[str] = Field(None, description="Platform where message was sent", title="Platform")
- message_type: Optional[str] = Field(None, description="Type of message (text, email, chat, etc.)", title="Message Type")
- subject: Optional[str] = Field(None, description="Message subject (for emails)", title="Subject")
- is_read: Optional[bool] = Field(None, description="Whether message has been read", title="Is Read")
- is_reply: Optional[bool] = Field(None, description="Whether message is a reply", title="Is Reply")
- parent_message_id: Optional[str] = Field(None, description="Parent message ID if reply", title="Parent Message ID")
- attachments: Optional[List[str]] = Field(None, description="List of attachment file names", title="Attachments")
- language: Optional[str] = Field(None, description="Message language", title="Language")
- sentiment: Optional[str] = Field(None, description="Message sentiment analysis", title="Sentiment")
- keywords: Optional[List[str]] = Field(None, description="Extracted keywords", title="Keywords")
- entities: Optional[List[str]] = Field(None, description="Named entities found in message", title="Entities")
- source: Optional[str] = Field(None, description="Source of message information", title="Source")
- is_suspicious: Optional[bool] = Field(None, description="Whether message is suspicious", title="Is Suspicious")
- threat_level: Optional[str] = Field(None, description="Threat level assessment", title="Threat Level")
\ No newline at end of file
+ recipient: Optional[str] = Field(
+ None, description="Message recipient", title="Recipient"
+ )
+ timestamp: Optional[str] = Field(
+ None, description="Message timestamp", title="Timestamp"
+ )
+ platform: Optional[str] = Field(
+ None, description="Platform where message was sent", title="Platform"
+ )
+ message_type: Optional[str] = Field(
+ None,
+ description="Type of message (text, email, chat, etc.)",
+ title="Message Type",
+ )
+ subject: Optional[str] = Field(
+ None, description="Message subject (for emails)", title="Subject"
+ )
+ is_read: Optional[bool] = Field(
+ None, description="Whether message has been read", title="Is Read"
+ )
+ is_reply: Optional[bool] = Field(
+ None, description="Whether message is a reply", title="Is Reply"
+ )
+ parent_message_id: Optional[str] = Field(
+ None, description="Parent message ID if reply", title="Parent Message ID"
+ )
+ attachments: Optional[List[str]] = Field(
+ None, description="List of attachment file names", title="Attachments"
+ )
+ language: Optional[str] = Field(
+ None, description="Message language", title="Language"
+ )
+ sentiment: Optional[str] = Field(
+ None, description="Message sentiment analysis", title="Sentiment"
+ )
+ keywords: Optional[List[str]] = Field(
+ None, description="Extracted keywords", title="Keywords"
+ )
+ entities: Optional[List[str]] = Field(
+ None, description="Named entities found in message", title="Entities"
+ )
+ source: Optional[str] = Field(
+ None, description="Source of message information", title="Source"
+ )
+ is_suspicious: Optional[bool] = Field(
+ None, description="Whether message is suspicious", title="Is Suspicious"
+ )
+ threat_level: Optional[str] = Field(
+ None, description="Threat level assessment", title="Threat Level"
+ )
diff --git a/flowsint-types/src/flowsint_types/organization.py b/flowsint-types/src/flowsint_types/organization.py
index 7b610e6f..33302bbb 100644
--- a/flowsint-types/src/flowsint_types/organization.py
+++ b/flowsint-types/src/flowsint_types/organization.py
@@ -3,113 +3,361 @@ from typing import Optional, List
from .individual import Individual
from .address import PhysicalAddress
+
class Organization(BaseModel):
"""Represents an organization with detailed business and administrative information."""
+
# Basic information
name: str = Field(..., description="Organization name", title="Organization Name")
siren: Optional[str] = Field(None, description="SIREN number", title="SIREN Number")
- nom_complet: Optional[str] = Field(None, description="Complete name", title="Complete Name")
- nom_raison_sociale: Optional[str] = Field(None, description="Company name", title="Company Name")
+ nom_complet: Optional[str] = Field(
+ None, description="Complete name", title="Complete Name"
+ )
+ nom_raison_sociale: Optional[str] = Field(
+ None, description="Company name", title="Company Name"
+ )
sigle: Optional[str] = Field(None, description="Acronym", title="Acronym")
# Establishment information
- nombre_etablissements: Optional[int] = Field(None, description="Number of establishments", title="Number of Establishments")
- nombre_etablissements_ouverts: Optional[int] = Field(None, description="Number of open establishments", title="Number of Open Establishments")
+ nombre_etablissements: Optional[int] = Field(
+ None, description="Number of establishments", title="Number of Establishments"
+ )
+ nombre_etablissements_ouverts: Optional[int] = Field(
+ None,
+ description="Number of open establishments",
+ title="Number of Open Establishments",
+ )
# Activity information
- activite_principale: Optional[str] = Field(None, description="Main activity code", title="Main Activity Code")
- section_activite_principale: Optional[str] = Field(None, description="Main activity section", title="Main Activity Section")
+ activite_principale: Optional[str] = Field(
+ None, description="Main activity code", title="Main Activity Code"
+ )
+ section_activite_principale: Optional[str] = Field(
+ None, description="Main activity section", title="Main Activity Section"
+ )
# Company category
- categorie_entreprise: Optional[str] = Field(None, description="Company category", title="Company Category")
- annee_categorie_entreprise: Optional[str] = Field(None, description="Year of company category", title="Company Category Year")
+ categorie_entreprise: Optional[str] = Field(
+ None, description="Company category", title="Company Category"
+ )
+ annee_categorie_entreprise: Optional[str] = Field(
+ None, description="Year of company category", title="Company Category Year"
+ )
# Employment information
- caractere_employeur: Optional[str] = Field(None, description="Employer character", title="Employer Character")
- tranche_effectif_salarie: Optional[str] = Field(None, description="Employee size range", title="Employee Size Range")
- annee_tranche_effectif_salarie: Optional[str] = Field(None, description="Year of employee size range", title="Employee Size Range Year")
+ caractere_employeur: Optional[str] = Field(
+ None, description="Employer character", title="Employer Character"
+ )
+ tranche_effectif_salarie: Optional[str] = Field(
+ None, description="Employee size range", title="Employee Size Range"
+ )
+ annee_tranche_effectif_salarie: Optional[str] = Field(
+ None,
+ description="Year of employee size range",
+ title="Employee Size Range Year",
+ )
# Dates
- date_creation: Optional[str] = Field(None, description="Creation date", title="Creation Date")
- date_fermeture: Optional[str] = Field(None, description="Closing date", title="Closing Date")
- date_mise_a_jour: Optional[str] = Field(None, description="Update date", title="Update Date")
- date_mise_a_jour_insee: Optional[str] = Field(None, description="INSEE update date", title="INSEE Update Date")
- date_mise_a_jour_rne: Optional[str] = Field(None, description="RNE update date", title="RNE Update Date")
+ date_creation: Optional[str] = Field(
+ None, description="Creation date", title="Creation Date"
+ )
+ date_fermeture: Optional[str] = Field(
+ None, description="Closing date", title="Closing Date"
+ )
+ date_mise_a_jour: Optional[str] = Field(
+ None, description="Update date", title="Update Date"
+ )
+ date_mise_a_jour_insee: Optional[str] = Field(
+ None, description="INSEE update date", title="INSEE Update Date"
+ )
+ date_mise_a_jour_rne: Optional[str] = Field(
+ None, description="RNE update date", title="RNE Update Date"
+ )
# Legal information
- nature_juridique: Optional[str] = Field(None, description="Legal nature", title="Legal Nature")
- etat_administratif: Optional[str] = Field(None, description="Administrative status", title="Administrative Status")
- statut_diffusion: Optional[str] = Field(None, description="Diffusion status", title="Diffusion Status")
+ nature_juridique: Optional[str] = Field(
+ None, description="Legal nature", title="Legal Nature"
+ )
+ etat_administratif: Optional[str] = Field(
+ None, description="Administrative status", title="Administrative Status"
+ )
+ statut_diffusion: Optional[str] = Field(
+ None, description="Diffusion status", title="Diffusion Status"
+ )
# Siege (Headquarters) information
- siege_activite_principale: Optional[str] = Field(None, description="Siege main activity code", title="Headquarters Main Activity")
- siege_activite_principale_registre_metier: Optional[str] = Field(None, description="Siege main activity in trade register", title="Headquarters Trade Register Activity")
- siege_annee_tranche_effectif_salarie: Optional[str] = Field(None, description="Siege year of employee size range", title="Headquarters Employee Size Year")
- siege_adresse: Optional[str] = Field(None, description="Siege address", title="Headquarters Address")
- siege_caractere_employeur: Optional[str] = Field(None, description="Siege employer character", title="Headquarters Employer Character")
- siege_cedex: Optional[str] = Field(None, description="Siege CEDEX code", title="Headquarters CEDEX")
- siege_code_pays_etranger: Optional[str] = Field(None, description="Siege foreign country code", title="Headquarters Foreign Country Code")
- siege_code_postal: Optional[str] = Field(None, description="Siege postal code", title="Headquarters Postal Code")
- siege_commune: Optional[str] = Field(None, description="Siege municipality code", title="Headquarters Municipality Code")
- siege_complement_adresse: Optional[str] = Field(None, description="Siege address complement", title="Headquarters Address Complement")
- siege_coordonnees: Optional[str] = Field(None, description="Siege coordinates", title="Headquarters Coordinates")
- siege_date_creation: Optional[str] = Field(None, description="Siege creation date", title="Headquarters Creation Date")
- siege_date_debut_activite: Optional[str] = Field(None, description="Siege activity start date", title="Headquarters Activity Start Date")
- siege_date_fermeture: Optional[str] = Field(None, description="Siege closing date", title="Headquarters Closing Date")
- siege_date_mise_a_jour: Optional[str] = Field(None, description="Siege update date", title="Headquarters Update Date")
- siege_date_mise_a_jour_insee: Optional[str] = Field(None, description="Siege INSEE update date", title="Headquarters INSEE Update Date")
- siege_departement: Optional[str] = Field(None, description="Siege department code", title="Headquarters Department")
- siege_distribution_speciale: Optional[str] = Field(None, description="Siege special distribution", title="Headquarters Special Distribution")
- siege_epci: Optional[str] = Field(None, description="Siege EPCI code", title="Headquarters EPCI Code")
- siege_est_siege: Optional[bool] = Field(None, description="Siege is headquarters", title="Is Headquarters")
- siege_etat_administratif: Optional[str] = Field(None, description="Siege administrative status", title="Headquarters Administrative Status")
- siege_geo_adresse: Optional[PhysicalAddress] = Field(None, description="Siege geocoded address", title="Headquarters Geocoded Address")
- siege_geo_id: Optional[str] = Field(None, description="Siege geographic ID", title="Headquarters Geographic ID")
- siege_indice_repetition: Optional[str] = Field(None, description="Siege repetition index", title="Headquarters Repetition Index")
- siege_latitude: Optional[str] = Field(None, description="Siege latitude", title="Headquarters Latitude")
- siege_libelle_cedex: Optional[str] = Field(None, description="Siege CEDEX label", title="Headquarters CEDEX Label")
- siege_libelle_commune: Optional[str] = Field(None, description="Siege municipality label", title="Headquarters Municipality Label")
- siege_libelle_commune_etranger: Optional[str] = Field(None, description="Siege foreign municipality label", title="Headquarters Foreign Municipality Label")
- siege_libelle_pays_etranger: Optional[str] = Field(None, description="Siege foreign country label", title="Headquarters Foreign Country Label")
- siege_libelle_voie: Optional[str] = Field(None, description="Siege street label", title="Headquarters Street Label")
- siege_liste_enseignes: Optional[List[str]] = Field(None, description="Siege list of brands", title="Headquarters Brands List")
- siege_liste_finess: Optional[List[str]] = Field(None, description="Siege list of FINESS", title="Headquarters FINESS List")
- siege_liste_id_bio: Optional[List[str]] = Field(None, description="Siege list of bio IDs", title="Headquarters Bio IDs List")
- siege_liste_idcc: Optional[List[str]] = Field(None, description="Siege list of IDCC", title="Headquarters IDCC List")
- siege_liste_id_organisme_formation: Optional[List[str]] = Field(None, description="Siege list of training organization IDs", title="Headquarters Training Organization IDs")
- siege_liste_rge: Optional[List[str]] = Field(None, description="Siege list of RGE", title="Headquarters RGE List")
- siege_liste_uai: Optional[List[str]] = Field(None, description="Siege list of UAI", title="Headquarters UAI List")
- siege_longitude: Optional[str] = Field(None, description="Siege longitude", title="Headquarters Longitude")
- siege_nom_commercial: Optional[str] = Field(None, description="Siege commercial name", title="Headquarters Commercial Name")
- siege_numero_voie: Optional[str] = Field(None, description="Siege street number", title="Headquarters Street Number")
- siege_region: Optional[str] = Field(None, description="Siege region code", title="Headquarters Region")
- siege_siret: Optional[str] = Field(None, description="Siege SIRET number", title="Headquarters SIRET")
- siege_statut_diffusion_etablissement: Optional[str] = Field(None, description="Siege establishment diffusion status", title="Headquarters Establishment Diffusion Status")
- siege_tranche_effectif_salarie: Optional[str] = Field(None, description="Siege employee size range", title="Headquarters Employee Size Range")
- siege_type_voie: Optional[str] = Field(None, description="Siege street type", title="Headquarters Street Type")
+ siege_activite_principale: Optional[str] = Field(
+ None, description="Siege main activity code", title="Headquarters Main Activity"
+ )
+ siege_activite_principale_registre_metier: Optional[str] = Field(
+ None,
+ description="Siege main activity in trade register",
+ title="Headquarters Trade Register Activity",
+ )
+ siege_annee_tranche_effectif_salarie: Optional[str] = Field(
+ None,
+ description="Siege year of employee size range",
+ title="Headquarters Employee Size Year",
+ )
+ siege_adresse: Optional[str] = Field(
+ None, description="Siege address", title="Headquarters Address"
+ )
+ siege_caractere_employeur: Optional[str] = Field(
+ None,
+ description="Siege employer character",
+ title="Headquarters Employer Character",
+ )
+ siege_cedex: Optional[str] = Field(
+ None, description="Siege CEDEX code", title="Headquarters CEDEX"
+ )
+ siege_code_pays_etranger: Optional[str] = Field(
+ None,
+ description="Siege foreign country code",
+ title="Headquarters Foreign Country Code",
+ )
+ siege_code_postal: Optional[str] = Field(
+ None, description="Siege postal code", title="Headquarters Postal Code"
+ )
+ siege_commune: Optional[str] = Field(
+ None,
+ description="Siege municipality code",
+ title="Headquarters Municipality Code",
+ )
+ siege_complement_adresse: Optional[str] = Field(
+ None,
+ description="Siege address complement",
+ title="Headquarters Address Complement",
+ )
+ siege_coordonnees: Optional[str] = Field(
+ None, description="Siege coordinates", title="Headquarters Coordinates"
+ )
+ siege_date_creation: Optional[str] = Field(
+ None, description="Siege creation date", title="Headquarters Creation Date"
+ )
+ siege_date_debut_activite: Optional[str] = Field(
+ None,
+ description="Siege activity start date",
+ title="Headquarters Activity Start Date",
+ )
+ siege_date_fermeture: Optional[str] = Field(
+ None, description="Siege closing date", title="Headquarters Closing Date"
+ )
+ siege_date_mise_a_jour: Optional[str] = Field(
+ None, description="Siege update date", title="Headquarters Update Date"
+ )
+ siege_date_mise_a_jour_insee: Optional[str] = Field(
+ None,
+ description="Siege INSEE update date",
+ title="Headquarters INSEE Update Date",
+ )
+ siege_departement: Optional[str] = Field(
+ None, description="Siege department code", title="Headquarters Department"
+ )
+ siege_distribution_speciale: Optional[str] = Field(
+ None,
+ description="Siege special distribution",
+ title="Headquarters Special Distribution",
+ )
+ siege_epci: Optional[str] = Field(
+ None, description="Siege EPCI code", title="Headquarters EPCI Code"
+ )
+ siege_est_siege: Optional[bool] = Field(
+ None, description="Siege is headquarters", title="Is Headquarters"
+ )
+ siege_etat_administratif: Optional[str] = Field(
+ None,
+ description="Siege administrative status",
+ title="Headquarters Administrative Status",
+ )
+ siege_geo_adresse: Optional[PhysicalAddress] = Field(
+ None,
+ description="Siege geocoded address",
+ title="Headquarters Geocoded Address",
+ )
+ siege_geo_id: Optional[str] = Field(
+ None, description="Siege geographic ID", title="Headquarters Geographic ID"
+ )
+ siege_indice_repetition: Optional[str] = Field(
+ None,
+ description="Siege repetition index",
+ title="Headquarters Repetition Index",
+ )
+ siege_latitude: Optional[str] = Field(
+ None, description="Siege latitude", title="Headquarters Latitude"
+ )
+ siege_libelle_cedex: Optional[str] = Field(
+ None, description="Siege CEDEX label", title="Headquarters CEDEX Label"
+ )
+ siege_libelle_commune: Optional[str] = Field(
+ None,
+ description="Siege municipality label",
+ title="Headquarters Municipality Label",
+ )
+ siege_libelle_commune_etranger: Optional[str] = Field(
+ None,
+ description="Siege foreign municipality label",
+ title="Headquarters Foreign Municipality Label",
+ )
+ siege_libelle_pays_etranger: Optional[str] = Field(
+ None,
+ description="Siege foreign country label",
+ title="Headquarters Foreign Country Label",
+ )
+ siege_libelle_voie: Optional[str] = Field(
+ None, description="Siege street label", title="Headquarters Street Label"
+ )
+ siege_liste_enseignes: Optional[List[str]] = Field(
+ None, description="Siege list of brands", title="Headquarters Brands List"
+ )
+ siege_liste_finess: Optional[List[str]] = Field(
+ None, description="Siege list of FINESS", title="Headquarters FINESS List"
+ )
+ siege_liste_id_bio: Optional[List[str]] = Field(
+ None, description="Siege list of bio IDs", title="Headquarters Bio IDs List"
+ )
+ siege_liste_idcc: Optional[List[str]] = Field(
+ None, description="Siege list of IDCC", title="Headquarters IDCC List"
+ )
+ siege_liste_id_organisme_formation: Optional[List[str]] = Field(
+ None,
+ description="Siege list of training organization IDs",
+ title="Headquarters Training Organization IDs",
+ )
+ siege_liste_rge: Optional[List[str]] = Field(
+ None, description="Siege list of RGE", title="Headquarters RGE List"
+ )
+ siege_liste_uai: Optional[List[str]] = Field(
+ None, description="Siege list of UAI", title="Headquarters UAI List"
+ )
+ siege_longitude: Optional[str] = Field(
+ None, description="Siege longitude", title="Headquarters Longitude"
+ )
+ siege_nom_commercial: Optional[str] = Field(
+ None, description="Siege commercial name", title="Headquarters Commercial Name"
+ )
+ siege_numero_voie: Optional[str] = Field(
+ None, description="Siege street number", title="Headquarters Street Number"
+ )
+ siege_region: Optional[str] = Field(
+ None, description="Siege region code", title="Headquarters Region"
+ )
+ siege_siret: Optional[str] = Field(
+ None, description="Siege SIRET number", title="Headquarters SIRET"
+ )
+ siege_statut_diffusion_etablissement: Optional[str] = Field(
+ None,
+ description="Siege establishment diffusion status",
+ title="Headquarters Establishment Diffusion Status",
+ )
+ siege_tranche_effectif_salarie: Optional[str] = Field(
+ None,
+ description="Siege employee size range",
+ title="Headquarters Employee Size Range",
+ )
+ siege_type_voie: Optional[str] = Field(
+ None, description="Siege street type", title="Headquarters Street Type"
+ )
# Dirigeants (Leaders) - using Individual type
- dirigeants: Optional[List[Individual]] = Field(None, description="List of leaders", title="Leaders")
+ dirigeants: Optional[List[Individual]] = Field(
+ None, description="List of leaders", title="Leaders"
+ )
# Matching establishments
- matching_etablissements: Optional[List[dict]] = Field(None, description="Matching establishments", title="Matching Establishments")
+ matching_etablissements: Optional[List[dict]] = Field(
+ None, description="Matching establishments", title="Matching Establishments"
+ )
# Financial information
- finances: Optional[dict] = Field(None, description="Financial information", title="Financial Information")
+ finances: Optional[dict] = Field(
+ None, description="Financial information", title="Financial Information"
+ )
# Complements (Additional information)
- complements_collectivite_territoriale: Optional[str] = Field(None, description="Complements territorial community", title="Territorial Community")
- complements_convention_collective_renseignee: Optional[bool] = Field(None, description="Complements collective agreement informed", title="Collective Agreement Informed")
- complements_liste_idcc: Optional[List[str]] = Field(None, description="Complements list of IDCC", title="IDCC List")
- complements_egapro_renseignee: Optional[bool] = Field(None, description="Complements Egapro informed", title="Egapro Informed")
- complements_est_achats_responsables: Optional[bool] = Field(None, description="Complements is responsible purchases", title="Is Responsible Purchases")
- complements_est_alim_confiance: Optional[bool] = Field(None, description="Complements is food trust", title="Is Food Trust")
- complements_est_association: Optional[bool] = Field(None, description="Complements is association", title="Is Association")
- complements_est_bio: Optional[bool] = Field(None, description="Complements is bio", title="Is Bio")
- complements_est_entrepreneur_individuel: Optional[bool] = Field(None, description="Complements is individual entrepreneur", title="Is Individual Entrepreneur")
- complements_est_entrepreneur_spectacle: Optional[bool] = Field(None, description="Complements is show entrepreneur", title="Is Show Entrepreneur")
- complements_est_ess: Optional[bool] = Field(None, description="Complements is ESS", title="Is ESS")
- complements_est_finess: Optional[bool] = Field(None, description="Complements is FINESS", title="Is FINESS")
- complements_est_organisme_formation: Optional[bool] = Field(None, description="Complements is training organization", title="Is Training Organization")
- complements_est_qualiopi: Optional[bool] = Field(None, description="Complements is Qualiopi", title="Is Qualiopi")
- complements_liste_id_organisme_formation: Optional[List[str]] = Field(None, description="Complements list of training organization IDs", title="Training Organization IDs")
- complements_est_rge: Optional[bool] = Field(None, description="Complements is RGE", title="Is RGE")
- complements_est_service_public: Optional[bool] = Field(None, description="Complements is public service", title="Is Public Service")
- complements_est_l100_3: Optional[bool] = Field(None, description="Complements is L100-3", title="Is L100-3")
- complements_est_siae: Optional[bool] = Field(None, description="Complements is SIAE", title="Is SIAE")
- complements_est_societe_mission: Optional[bool] = Field(None, description="Complements is mission company", title="Is Mission Company")
- complements_est_uai: Optional[bool] = Field(None, description="Complements is UAI", title="Is UAI")
- complements_est_patrimoine_vivant: Optional[bool] = Field(None, description="Complements is living heritage", title="Is Living Heritage")
- complements_bilan_ges_renseigne: Optional[bool] = Field(None, description="Complements GES balance informed", title="GES Balance Informed")
- complements_identifiant_association: Optional[str] = Field(None, description="Complements association identifier", title="Association Identifier")
- complements_statut_entrepreneur_spectacle: Optional[str] = Field(None, description="Complements show entrepreneur status", title="Show Entrepreneur Status")
- complements_type_siae: Optional[str] = Field(None, description="Complements SIAE type", title="SIAE Type")
\ No newline at end of file
+ complements_collectivite_territoriale: Optional[str] = Field(
+ None,
+ description="Complements territorial community",
+ title="Territorial Community",
+ )
+ complements_convention_collective_renseignee: Optional[bool] = Field(
+ None,
+ description="Complements collective agreement informed",
+ title="Collective Agreement Informed",
+ )
+ complements_liste_idcc: Optional[List[str]] = Field(
+ None, description="Complements list of IDCC", title="IDCC List"
+ )
+ complements_egapro_renseignee: Optional[bool] = Field(
+ None, description="Complements Egapro informed", title="Egapro Informed"
+ )
+ complements_est_achats_responsables: Optional[bool] = Field(
+ None,
+ description="Complements is responsible purchases",
+ title="Is Responsible Purchases",
+ )
+ complements_est_alim_confiance: Optional[bool] = Field(
+ None, description="Complements is food trust", title="Is Food Trust"
+ )
+ complements_est_association: Optional[bool] = Field(
+ None, description="Complements is association", title="Is Association"
+ )
+ complements_est_bio: Optional[bool] = Field(
+ None, description="Complements is bio", title="Is Bio"
+ )
+ complements_est_entrepreneur_individuel: Optional[bool] = Field(
+ None,
+ description="Complements is individual entrepreneur",
+ title="Is Individual Entrepreneur",
+ )
+ complements_est_entrepreneur_spectacle: Optional[bool] = Field(
+ None,
+ description="Complements is show entrepreneur",
+ title="Is Show Entrepreneur",
+ )
+ complements_est_ess: Optional[bool] = Field(
+ None, description="Complements is ESS", title="Is ESS"
+ )
+ complements_est_finess: Optional[bool] = Field(
+ None, description="Complements is FINESS", title="Is FINESS"
+ )
+ complements_est_organisme_formation: Optional[bool] = Field(
+ None,
+ description="Complements is training organization",
+ title="Is Training Organization",
+ )
+ complements_est_qualiopi: Optional[bool] = Field(
+ None, description="Complements is Qualiopi", title="Is Qualiopi"
+ )
+ complements_liste_id_organisme_formation: Optional[List[str]] = Field(
+ None,
+ description="Complements list of training organization IDs",
+ title="Training Organization IDs",
+ )
+ complements_est_rge: Optional[bool] = Field(
+ None, description="Complements is RGE", title="Is RGE"
+ )
+ complements_est_service_public: Optional[bool] = Field(
+ None, description="Complements is public service", title="Is Public Service"
+ )
+ complements_est_l100_3: Optional[bool] = Field(
+ None, description="Complements is L100-3", title="Is L100-3"
+ )
+ complements_est_siae: Optional[bool] = Field(
+ None, description="Complements is SIAE", title="Is SIAE"
+ )
+ complements_est_societe_mission: Optional[bool] = Field(
+ None, description="Complements is mission company", title="Is Mission Company"
+ )
+ complements_est_uai: Optional[bool] = Field(
+ None, description="Complements is UAI", title="Is UAI"
+ )
+ complements_est_patrimoine_vivant: Optional[bool] = Field(
+ None, description="Complements is living heritage", title="Is Living Heritage"
+ )
+ complements_bilan_ges_renseigne: Optional[bool] = Field(
+ None,
+ description="Complements GES balance informed",
+ title="GES Balance Informed",
+ )
+ complements_identifiant_association: Optional[str] = Field(
+ None,
+ description="Complements association identifier",
+ title="Association Identifier",
+ )
+ complements_statut_entrepreneur_spectacle: Optional[str] = Field(
+ None,
+ description="Complements show entrepreneur status",
+ title="Show Entrepreneur Status",
+ )
+ complements_type_siae: Optional[str] = Field(
+ None, description="Complements SIAE type", title="SIAE Type"
+ )
diff --git a/flowsint-types/src/flowsint_types/phone.py b/flowsint-types/src/flowsint_types/phone.py
index 0c5eb04a..368e1c96 100644
--- a/flowsint-types/src/flowsint_types/phone.py
+++ b/flowsint-types/src/flowsint_types/phone.py
@@ -1,8 +1,14 @@
from pydantic import BaseModel, Field
from typing import Optional
+
class Phone(BaseModel):
"""Represents a phone number with country and carrier information."""
+
number: str = Field(..., description="Phone number", title="Phone Number")
- country: Optional[str] = Field(None, description="Country code (ISO 3166-1 alpha-2)", title="Country Code")
- carrier: Optional[str] = Field(None, description="Mobile carrier or service provider", title="Carrier")
\ No newline at end of file
+ country: Optional[str] = Field(
+ None, description="Country code (ISO 3166-1 alpha-2)", title="Country Code"
+ )
+ carrier: Optional[str] = Field(
+ None, description="Mobile carrier or service provider", title="Carrier"
+ )
diff --git a/flowsint-types/src/flowsint_types/phrase.py b/flowsint-types/src/flowsint_types/phrase.py
index a0b47649..678832c8 100644
--- a/flowsint-types/src/flowsint_types/phrase.py
+++ b/flowsint-types/src/flowsint_types/phrase.py
@@ -1,7 +1,10 @@
from pydantic import BaseModel, Field
from typing import Any
+
class Phrase(BaseModel):
"""Represents a phrase or text content."""
- text: Any = Field(..., description="The content of the phrase.", title="Phrase text value.")
-
+
+ text: Any = Field(
+ ..., description="The content of the phrase.", title="Phrase text value."
+ )
diff --git a/flowsint-types/src/flowsint_types/reputation_score.py b/flowsint-types/src/flowsint_types/reputation_score.py
index 81cc0928..c08693e9 100644
--- a/flowsint-types/src/flowsint_types/reputation_score.py
+++ b/flowsint-types/src/flowsint_types/reputation_score.py
@@ -1,20 +1,56 @@
from pydantic import BaseModel, Field
from typing import Optional, List
+
class ReputationScore(BaseModel):
"""Represents a reputation score for an entity with historical data and trends."""
+
entity_id: str = Field(..., description="Entity identifier", title="Entity ID")
- entity_type: Optional[str] = Field(None, description="Type of entity (domain, IP, email, etc.)", title="Entity Type")
- score: Optional[float] = Field(None, description="Reputation score (0-100)", title="Score")
- score_type: Optional[str] = Field(None, description="Type of score (trust, spam, malware, etc.)", title="Score Type")
- provider: Optional[str] = Field(None, description="Reputation provider", title="Provider")
- last_updated: Optional[str] = Field(None, description="Last update timestamp", title="Last Updated")
- confidence: Optional[float] = Field(None, description="Confidence in the score", title="Confidence")
- factors: Optional[List[str]] = Field(None, description="Factors contributing to score", title="Factors")
- category: Optional[str] = Field(None, description="Category (good, suspicious, malicious, etc.)", title="Category")
- description: Optional[str] = Field(None, description="Description of reputation", title="Description")
- source: Optional[str] = Field(None, description="Source of reputation data", title="Source")
- historical_scores: Optional[List[dict]] = Field(None, description="Historical score data", title="Historical Scores")
- trend: Optional[str] = Field(None, description="Score trend (improving, declining, stable)", title="Trend")
- risk_level: Optional[str] = Field(None, description="Risk level assessment", title="Risk Level")
- recommendations: Optional[List[str]] = Field(None, description="Recommendations based on score", title="Recommendations")
\ No newline at end of file
+ entity_type: Optional[str] = Field(
+ None,
+ description="Type of entity (domain, IP, email, etc.)",
+ title="Entity Type",
+ )
+ score: Optional[float] = Field(
+ None, description="Reputation score (0-100)", title="Score"
+ )
+ score_type: Optional[str] = Field(
+ None,
+ description="Type of score (trust, spam, malware, etc.)",
+ title="Score Type",
+ )
+ provider: Optional[str] = Field(
+ None, description="Reputation provider", title="Provider"
+ )
+ last_updated: Optional[str] = Field(
+ None, description="Last update timestamp", title="Last Updated"
+ )
+ confidence: Optional[float] = Field(
+ None, description="Confidence in the score", title="Confidence"
+ )
+ factors: Optional[List[str]] = Field(
+ None, description="Factors contributing to score", title="Factors"
+ )
+ category: Optional[str] = Field(
+ None,
+ description="Category (good, suspicious, malicious, etc.)",
+ title="Category",
+ )
+ description: Optional[str] = Field(
+ None, description="Description of reputation", title="Description"
+ )
+ source: Optional[str] = Field(
+ None, description="Source of reputation data", title="Source"
+ )
+ historical_scores: Optional[List[dict]] = Field(
+ None, description="Historical score data", title="Historical Scores"
+ )
+ trend: Optional[str] = Field(
+ None, description="Score trend (improving, declining, stable)", title="Trend"
+ )
+ risk_level: Optional[str] = Field(
+ None, description="Risk level assessment", title="Risk Level"
+ )
+ recommendations: Optional[List[str]] = Field(
+ None, description="Recommendations based on score", title="Recommendations"
+ )
diff --git a/flowsint-types/src/flowsint_types/risk_profile.py b/flowsint-types/src/flowsint_types/risk_profile.py
index 677a672e..38a52326 100644
--- a/flowsint-types/src/flowsint_types/risk_profile.py
+++ b/flowsint-types/src/flowsint_types/risk_profile.py
@@ -1,27 +1,73 @@
from pydantic import BaseModel, Field
from typing import Optional, List
+
class RiskProfile(BaseModel):
"""Represents a comprehensive risk assessment profile for an entity."""
+
entity_id: str = Field(..., description="Entity identifier", title="Entity ID")
- entity_type: Optional[str] = Field(None, description="Type of entity", title="Entity Type")
- overall_risk_score: Optional[float] = Field(None, description="Overall risk score (0-100)", title="Overall Risk Score")
- risk_level: Optional[str] = Field(None, description="Risk level (low, medium, high, critical)", title="Risk Level")
- assessment_date: Optional[str] = Field(None, description="Date of risk assessment", title="Assessment Date")
- last_updated: Optional[str] = Field(None, description="Last update timestamp", title="Last Updated")
- risk_factors: Optional[List[str]] = Field(None, description="Identified risk factors", title="Risk Factors")
- threat_actors: Optional[List[str]] = Field(None, description="Associated threat actors", title="Threat Actors")
- attack_vectors: Optional[List[str]] = Field(None, description="Potential attack vectors", title="Attack Vectors")
- vulnerabilities: Optional[List[str]] = Field(None, description="Identified vulnerabilities", title="Vulnerabilities")
- exposure_surface: Optional[str] = Field(None, description="Exposure surface description", title="Exposure Surface")
- data_classification: Optional[str] = Field(None, description="Data classification level", title="Data Classification")
- compliance_risks: Optional[List[str]] = Field(None, description="Compliance-related risks", title="Compliance Risks")
- financial_impact: Optional[str] = Field(None, description="Potential financial impact", title="Financial Impact")
- reputation_impact: Optional[str] = Field(None, description="Potential reputation impact", title="Reputation Impact")
- operational_impact: Optional[str] = Field(None, description="Potential operational impact", title="Operational Impact")
- mitigation_strategies: Optional[List[str]] = Field(None, description="Recommended mitigation strategies", title="Mitigation Strategies")
- monitoring_recommendations: Optional[List[str]] = Field(None, description="Monitoring recommendations", title="Monitoring Recommendations")
- source: Optional[str] = Field(None, description="Source of risk assessment", title="Source")
+ entity_type: Optional[str] = Field(
+ None, description="Type of entity", title="Entity Type"
+ )
+ overall_risk_score: Optional[float] = Field(
+ None, description="Overall risk score (0-100)", title="Overall Risk Score"
+ )
+ risk_level: Optional[str] = Field(
+ None, description="Risk level (low, medium, high, critical)", title="Risk Level"
+ )
+ assessment_date: Optional[str] = Field(
+ None, description="Date of risk assessment", title="Assessment Date"
+ )
+ last_updated: Optional[str] = Field(
+ None, description="Last update timestamp", title="Last Updated"
+ )
+ risk_factors: Optional[List[str]] = Field(
+ None, description="Identified risk factors", title="Risk Factors"
+ )
+ threat_actors: Optional[List[str]] = Field(
+ None, description="Associated threat actors", title="Threat Actors"
+ )
+ attack_vectors: Optional[List[str]] = Field(
+ None, description="Potential attack vectors", title="Attack Vectors"
+ )
+ vulnerabilities: Optional[List[str]] = Field(
+ None, description="Identified vulnerabilities", title="Vulnerabilities"
+ )
+ exposure_surface: Optional[str] = Field(
+ None, description="Exposure surface description", title="Exposure Surface"
+ )
+ data_classification: Optional[str] = Field(
+ None, description="Data classification level", title="Data Classification"
+ )
+ compliance_risks: Optional[List[str]] = Field(
+ None, description="Compliance-related risks", title="Compliance Risks"
+ )
+ financial_impact: Optional[str] = Field(
+ None, description="Potential financial impact", title="Financial Impact"
+ )
+ reputation_impact: Optional[str] = Field(
+ None, description="Potential reputation impact", title="Reputation Impact"
+ )
+ operational_impact: Optional[str] = Field(
+ None, description="Potential operational impact", title="Operational Impact"
+ )
+ mitigation_strategies: Optional[List[str]] = Field(
+ None,
+ description="Recommended mitigation strategies",
+ title="Mitigation Strategies",
+ )
+ monitoring_recommendations: Optional[List[str]] = Field(
+ None,
+ description="Monitoring recommendations",
+ title="Monitoring Recommendations",
+ )
+ source: Optional[str] = Field(
+ None, description="Source of risk assessment", title="Source"
+ )
assessor: Optional[str] = Field(None, description="Risk assessor", title="Assessor")
- confidence: Optional[float] = Field(None, description="Confidence in assessment", title="Confidence")
- next_review_date: Optional[str] = Field(None, description="Next review date", title="Next Review Date")
\ No newline at end of file
+ confidence: Optional[float] = Field(
+ None, description="Confidence in assessment", title="Confidence"
+ )
+ next_review_date: Optional[str] = Field(
+ None, description="Next review date", title="Next Review Date"
+ )
diff --git a/flowsint-types/src/flowsint_types/script.py b/flowsint-types/src/flowsint_types/script.py
index 833cfbdf..7d564fe9 100644
--- a/flowsint-types/src/flowsint_types/script.py
+++ b/flowsint-types/src/flowsint_types/script.py
@@ -1,29 +1,65 @@
from pydantic import BaseModel, Field
from typing import Optional, List
+
class Script(BaseModel):
"""Represents a script or code file with analysis and security information."""
- script_id: str = Field(..., description="Unique script identifier", title="Script ID")
+
+ script_id: str = Field(
+ ..., description="Unique script identifier", title="Script ID"
+ )
name: Optional[str] = Field(None, description="Script name", title="Name")
- language: Optional[str] = Field(None, description="Programming language", title="Language")
- type: Optional[str] = Field(None, description="Type of script (JavaScript, Python, etc.)", title="Type")
+ language: Optional[str] = Field(
+ None, description="Programming language", title="Language"
+ )
+ type: Optional[str] = Field(
+ None, description="Type of script (JavaScript, Python, etc.)", title="Type"
+ )
content: Optional[str] = Field(None, description="Script content", title="Content")
- file_path: Optional[str] = Field(None, description="File path if stored", title="File Path")
+ file_path: Optional[str] = Field(
+ None, description="File path if stored", title="File Path"
+ )
url: Optional[str] = Field(None, description="URL if loaded from web", title="URL")
hash_md5: Optional[str] = Field(None, description="MD5 hash", title="MD5 Hash")
hash_sha1: Optional[str] = Field(None, description="SHA1 hash", title="SHA1 Hash")
- hash_sha256: Optional[str] = Field(None, description="SHA256 hash", title="SHA256 Hash")
- file_size: Optional[int] = Field(None, description="File size in bytes", title="File Size")
- created_date: Optional[str] = Field(None, description="Creation date", title="Created Date")
- modified_date: Optional[str] = Field(None, description="Last modified date", title="Modified Date")
+ hash_sha256: Optional[str] = Field(
+ None, description="SHA256 hash", title="SHA256 Hash"
+ )
+ file_size: Optional[int] = Field(
+ None, description="File size in bytes", title="File Size"
+ )
+ created_date: Optional[str] = Field(
+ None, description="Creation date", title="Created Date"
+ )
+ modified_date: Optional[str] = Field(
+ None, description="Last modified date", title="Modified Date"
+ )
author: Optional[str] = Field(None, description="Script author", title="Author")
version: Optional[str] = Field(None, description="Script version", title="Version")
- description: Optional[str] = Field(None, description="Script description", title="Description")
- dependencies: Optional[List[str]] = Field(None, description="Script dependencies", title="Dependencies")
- functions: Optional[List[str]] = Field(None, description="Functions defined in script", title="Functions")
- is_malicious: Optional[bool] = Field(None, description="Whether script is malicious", title="Is Malicious")
- malware_family: Optional[str] = Field(None, description="Malware family if malicious", title="Malware Family")
- threat_level: Optional[str] = Field(None, description="Threat level assessment", title="Threat Level")
- source: Optional[str] = Field(None, description="Source of script information", title="Source")
- obfuscated: Optional[bool] = Field(None, description="Whether script is obfuscated", title="Obfuscated")
- minified: Optional[bool] = Field(None, description="Whether script is minified", title="Minified")
\ No newline at end of file
+ description: Optional[str] = Field(
+ None, description="Script description", title="Description"
+ )
+ dependencies: Optional[List[str]] = Field(
+ None, description="Script dependencies", title="Dependencies"
+ )
+ functions: Optional[List[str]] = Field(
+ None, description="Functions defined in script", title="Functions"
+ )
+ is_malicious: Optional[bool] = Field(
+ None, description="Whether script is malicious", title="Is Malicious"
+ )
+ malware_family: Optional[str] = Field(
+ None, description="Malware family if malicious", title="Malware Family"
+ )
+ threat_level: Optional[str] = Field(
+ None, description="Threat level assessment", title="Threat Level"
+ )
+ source: Optional[str] = Field(
+ None, description="Source of script information", title="Source"
+ )
+ obfuscated: Optional[bool] = Field(
+ None, description="Whether script is obfuscated", title="Obfuscated"
+ )
+ minified: Optional[bool] = Field(
+ None, description="Whether script is minified", title="Minified"
+ )
diff --git a/flowsint-types/src/flowsint_types/session.py b/flowsint-types/src/flowsint_types/session.py
index 7d4294bd..9d025e4e 100644
--- a/flowsint-types/src/flowsint_types/session.py
+++ b/flowsint-types/src/flowsint_types/session.py
@@ -1,21 +1,51 @@
from pydantic import BaseModel, Field
from typing import Optional, List
+
class Session(BaseModel):
"""Represents a user session with device and activity information."""
- session_id: str = Field(..., description="Unique session identifier", title="Session ID")
+
+ session_id: str = Field(
+ ..., description="Unique session identifier", title="Session ID"
+ )
user_id: Optional[str] = Field(None, description="User identifier", title="User ID")
- service: Optional[str] = Field(None, description="Service or platform", title="Service")
- start_time: Optional[str] = Field(None, description="Session start timestamp", title="Start Time")
- end_time: Optional[str] = Field(None, description="Session end timestamp", title="End Time")
- duration: Optional[int] = Field(None, description="Session duration in seconds", title="Duration")
- ip_address: Optional[str] = Field(None, description="IP address used for session", title="IP Address")
- user_agent: Optional[str] = Field(None, description="User agent string", title="User Agent")
- location: Optional[str] = Field(None, description="Geographic location", title="Location")
- device_type: Optional[str] = Field(None, description="Type of device used", title="Device Type")
+ service: Optional[str] = Field(
+ None, description="Service or platform", title="Service"
+ )
+ start_time: Optional[str] = Field(
+ None, description="Session start timestamp", title="Start Time"
+ )
+ end_time: Optional[str] = Field(
+ None, description="Session end timestamp", title="End Time"
+ )
+ duration: Optional[int] = Field(
+ None, description="Session duration in seconds", title="Duration"
+ )
+ ip_address: Optional[str] = Field(
+ None, description="IP address used for session", title="IP Address"
+ )
+ user_agent: Optional[str] = Field(
+ None, description="User agent string", title="User Agent"
+ )
+ location: Optional[str] = Field(
+ None, description="Geographic location", title="Location"
+ )
+ device_type: Optional[str] = Field(
+ None, description="Type of device used", title="Device Type"
+ )
browser: Optional[str] = Field(None, description="Browser used", title="Browser")
- os: Optional[str] = Field(None, description="Operating system", title="Operating System")
- is_active: Optional[bool] = Field(None, description="Whether session is currently active", title="Is Active")
- is_suspicious: Optional[bool] = Field(None, description="Whether session is suspicious", title="Is Suspicious")
- activities: Optional[List[str]] = Field(None, description="Activities performed during session", title="Activities")
- source: Optional[str] = Field(None, description="Source of session information", title="Source")
\ No newline at end of file
+ os: Optional[str] = Field(
+ None, description="Operating system", title="Operating System"
+ )
+ is_active: Optional[bool] = Field(
+ None, description="Whether session is currently active", title="Is Active"
+ )
+ is_suspicious: Optional[bool] = Field(
+ None, description="Whether session is suspicious", title="Is Suspicious"
+ )
+ activities: Optional[List[str]] = Field(
+ None, description="Activities performed during session", title="Activities"
+ )
+ source: Optional[str] = Field(
+ None, description="Source of session information", title="Source"
+ )
diff --git a/flowsint-types/src/flowsint_types/social.py b/flowsint-types/src/flowsint_types/social.py
index 37726c1e..fe9a23b5 100644
--- a/flowsint-types/src/flowsint_types/social.py
+++ b/flowsint-types/src/flowsint_types/social.py
@@ -1,14 +1,31 @@
from pydantic import BaseModel, Field
from typing import Optional
+
class SocialProfile(BaseModel):
"""Represents a social media profile with engagement metrics."""
- username: str = Field(..., description="Username on the social platform", title="Username")
- profile_url: Optional[str] = Field(None, description="URL to the user's profile page", title="Profile URL")
- platform: Optional[str] = Field(None, description="Name of the social media platform", title="Platform")
- profile_picture_url: Optional[str] = Field(None, description="URL to the user's profile picture", title="Profile Picture")
- bio: Optional[str] = Field(None, description="User's biography or description", title="Bio")
- followers_count: Optional[int] = Field(None, description="Number of followers", title="Followers Count")
- following_count: Optional[int] = Field(None, description="Number of users being followed", title="Following Count")
- posts_count: Optional[int] = Field(None, description="Number of posts made by the user", title="Posts Count")
-
\ No newline at end of file
+
+ username: str = Field(
+ ..., description="Username on the social platform", title="Username"
+ )
+ profile_url: Optional[str] = Field(
+ None, description="URL to the user's profile page", title="Profile URL"
+ )
+ platform: Optional[str] = Field(
+ None, description="Name of the social media platform", title="Platform"
+ )
+ profile_picture_url: Optional[str] = Field(
+ None, description="URL to the user's profile picture", title="Profile Picture"
+ )
+ bio: Optional[str] = Field(
+ None, description="User's biography or description", title="Bio"
+ )
+ followers_count: Optional[int] = Field(
+ None, description="Number of followers", title="Followers Count"
+ )
+ following_count: Optional[int] = Field(
+ None, description="Number of users being followed", title="Following Count"
+ )
+ posts_count: Optional[int] = Field(
+ None, description="Number of posts made by the user", title="Posts Count"
+ )
diff --git a/flowsint-types/src/flowsint_types/ssl_certificate.py b/flowsint-types/src/flowsint_types/ssl_certificate.py
index 5f3ccabb..3e33c652 100644
--- a/flowsint-types/src/flowsint_types/ssl_certificate.py
+++ b/flowsint-types/src/flowsint_types/ssl_certificate.py
@@ -1,25 +1,71 @@
from pydantic import BaseModel, Field
from typing import Optional, List
+
class SSLCertificate(BaseModel):
"""Represents an SSL/TLS certificate with validation and security details."""
- subject: str = Field(..., description="Certificate subject (domain name)", title="Subject")
- issuer: Optional[str] = Field(None, description="Certificate authority that issued the certificate", title="Issuer")
- serial_number: Optional[str] = Field(None, description="Certificate serial number", title="Serial Number")
- valid_from: Optional[str] = Field(None, description="Certificate validity start date", title="Valid From")
- valid_until: Optional[str] = Field(None, description="Certificate validity end date", title="Valid Until")
- signature_algorithm: Optional[str] = Field(None, description="Signature algorithm used", title="Signature Algorithm")
- key_size: Optional[int] = Field(None, description="Key size in bits", title="Key Size")
- key_type: Optional[str] = Field(None, description="Type of key (RSA, ECDSA, etc.)", title="Key Type")
- san_domains: Optional[List[str]] = Field(None, description="Subject Alternative Names", title="SAN Domains")
- is_valid: Optional[bool] = Field(None, description="Whether certificate is currently valid", title="Is Valid")
- is_expired: Optional[bool] = Field(None, description="Whether certificate has expired", title="Is Expired")
- is_self_signed: Optional[bool] = Field(None, description="Whether certificate is self-signed", title="Is Self Signed")
- is_wildcard: Optional[bool] = Field(None, description="Whether certificate is a wildcard certificate", title="Is Wildcard")
- ocsp_status: Optional[str] = Field(None, description="OCSP revocation status", title="OCSP Status")
- crl_status: Optional[str] = Field(None, description="Certificate Revocation List status", title="CRL Status")
- first_seen: Optional[str] = Field(None, description="First time certificate was observed", title="First Seen")
- last_seen: Optional[str] = Field(None, description="Last time certificate was observed", title="Last Seen")
- source: Optional[str] = Field(None, description="Source of certificate information", title="Source")
- fingerprint_sha1: Optional[str] = Field(None, description="SHA1 fingerprint", title="SHA1 Fingerprint")
- fingerprint_sha256: Optional[str] = Field(None, description="SHA256 fingerprint", title="SHA256 Fingerprint")
\ No newline at end of file
+
+ subject: str = Field(
+ ..., description="Certificate subject (domain name)", title="Subject"
+ )
+ issuer: Optional[str] = Field(
+ None,
+ description="Certificate authority that issued the certificate",
+ title="Issuer",
+ )
+ serial_number: Optional[str] = Field(
+ None, description="Certificate serial number", title="Serial Number"
+ )
+ valid_from: Optional[str] = Field(
+ None, description="Certificate validity start date", title="Valid From"
+ )
+ valid_until: Optional[str] = Field(
+ None, description="Certificate validity end date", title="Valid Until"
+ )
+ signature_algorithm: Optional[str] = Field(
+ None, description="Signature algorithm used", title="Signature Algorithm"
+ )
+ key_size: Optional[int] = Field(
+ None, description="Key size in bits", title="Key Size"
+ )
+ key_type: Optional[str] = Field(
+ None, description="Type of key (RSA, ECDSA, etc.)", title="Key Type"
+ )
+ san_domains: Optional[List[str]] = Field(
+ None, description="Subject Alternative Names", title="SAN Domains"
+ )
+ is_valid: Optional[bool] = Field(
+ None, description="Whether certificate is currently valid", title="Is Valid"
+ )
+ is_expired: Optional[bool] = Field(
+ None, description="Whether certificate has expired", title="Is Expired"
+ )
+ is_self_signed: Optional[bool] = Field(
+ None, description="Whether certificate is self-signed", title="Is Self Signed"
+ )
+ is_wildcard: Optional[bool] = Field(
+ None,
+ description="Whether certificate is a wildcard certificate",
+ title="Is Wildcard",
+ )
+ ocsp_status: Optional[str] = Field(
+ None, description="OCSP revocation status", title="OCSP Status"
+ )
+ crl_status: Optional[str] = Field(
+ None, description="Certificate Revocation List status", title="CRL Status"
+ )
+ first_seen: Optional[str] = Field(
+ None, description="First time certificate was observed", title="First Seen"
+ )
+ last_seen: Optional[str] = Field(
+ None, description="Last time certificate was observed", title="Last Seen"
+ )
+ source: Optional[str] = Field(
+ None, description="Source of certificate information", title="Source"
+ )
+ fingerprint_sha1: Optional[str] = Field(
+ None, description="SHA1 fingerprint", title="SHA1 Fingerprint"
+ )
+ fingerprint_sha256: Optional[str] = Field(
+ None, description="SHA256 fingerprint", title="SHA256 Fingerprint"
+ )
diff --git a/flowsint-types/src/flowsint_types/username.py b/flowsint-types/src/flowsint_types/username.py
index 7c73bf39..76389513 100644
--- a/flowsint-types/src/flowsint_types/username.py
+++ b/flowsint-types/src/flowsint_types/username.py
@@ -1,22 +1,54 @@
from pydantic import BaseModel, Field
from typing import Optional, List
+
class Username(BaseModel):
"""Represents a username or handle on a platform with associated profile information."""
+
username: str = Field(..., description="Username or handle", title="Username")
- platform: Optional[str] = Field(None, description="Platform or service where username is used", title="Platform")
- display_name: Optional[str] = Field(None, description="Display name associated with username", title="Display Name")
- created_at: Optional[str] = Field(None, description="Account creation date", title="Creation Date")
- last_seen: Optional[str] = Field(None, description="Last activity timestamp", title="Last Seen")
- followers_count: Optional[int] = Field(None, description="Number of followers", title="Followers Count")
- following_count: Optional[int] = Field(None, description="Number of accounts being followed", title="Following Count")
- posts_count: Optional[int] = Field(None, description="Number of posts or content items", title="Posts Count")
- verified: Optional[bool] = Field(None, description="Whether account is verified", title="Verified")
- bio: Optional[str] = Field(None, description="User biography or description", title="Bio")
+ platform: Optional[str] = Field(
+ None, description="Platform or service where username is used", title="Platform"
+ )
+ display_name: Optional[str] = Field(
+ None, description="Display name associated with username", title="Display Name"
+ )
+ created_at: Optional[str] = Field(
+ None, description="Account creation date", title="Creation Date"
+ )
+ last_seen: Optional[str] = Field(
+ None, description="Last activity timestamp", title="Last Seen"
+ )
+ followers_count: Optional[int] = Field(
+ None, description="Number of followers", title="Followers Count"
+ )
+ following_count: Optional[int] = Field(
+ None, description="Number of accounts being followed", title="Following Count"
+ )
+ posts_count: Optional[int] = Field(
+ None, description="Number of posts or content items", title="Posts Count"
+ )
+ verified: Optional[bool] = Field(
+ None, description="Whether account is verified", title="Verified"
+ )
+ bio: Optional[str] = Field(
+ None, description="User biography or description", title="Bio"
+ )
location: Optional[str] = Field(None, description="User location", title="Location")
- website: Optional[str] = Field(None, description="User website URL", title="Website")
- avatar_url: Optional[str] = Field(None, description="Profile picture URL", title="Avatar URL")
- is_private: Optional[bool] = Field(None, description="Whether account is private", title="Is Private")
- is_suspended: Optional[bool] = Field(None, description="Whether account is suspended", title="Is Suspended")
- associated_emails: Optional[List[str]] = Field(None, description="Associated email addresses", title="Associated Emails")
- associated_phones: Optional[List[str]] = Field(None, description="Associated phone numbers", title="Associated Phones")
\ No newline at end of file
+ website: Optional[str] = Field(
+ None, description="User website URL", title="Website"
+ )
+ avatar_url: Optional[str] = Field(
+ None, description="Profile picture URL", title="Avatar URL"
+ )
+ is_private: Optional[bool] = Field(
+ None, description="Whether account is private", title="Is Private"
+ )
+ is_suspended: Optional[bool] = Field(
+ None, description="Whether account is suspended", title="Is Suspended"
+ )
+ associated_emails: Optional[List[str]] = Field(
+ None, description="Associated email addresses", title="Associated Emails"
+ )
+ associated_phones: Optional[List[str]] = Field(
+ None, description="Associated phone numbers", title="Associated Phones"
+ )
diff --git a/flowsint-types/src/flowsint_types/wallet.py b/flowsint-types/src/flowsint_types/wallet.py
index 09f1f3fc..6fd47b66 100644
--- a/flowsint-types/src/flowsint_types/wallet.py
+++ b/flowsint-types/src/flowsint_types/wallet.py
@@ -1,54 +1,135 @@
from typing import Optional
from pydantic import BaseModel, Field, HttpUrl
+
class CryptoWallet(BaseModel):
"""Represents a cryptocurrency wallet."""
+
address: str = Field(..., description="Wallet address", title="Wallet Address")
- node_id: Optional[str] = Field(None, description="Wallet Explorer node ID", title="Node ID")
+ node_id: Optional[str] = Field(
+ None, description="Wallet Explorer node ID", title="Node ID"
+ )
+
class CryptoWalletTransaction(BaseModel):
"""Represents a cryptocurrency transaction."""
- source: CryptoWallet = Field(..., description="Source wallet", title="Source Wallet")
- target: Optional[CryptoWallet] = Field(None, description="Target wallet", title="Target Wallet")
- hash: Optional[str] = Field(None, description="Transaction hash", title="Transaction Hash")
- value: Optional[float] = Field(None, description="Transaction value in cryptocurrency", title="Value")
- amount: Optional[float] = Field(None, description="Transaction amount in cryptocurrency", title="Amount")
- amount_usd: Optional[float] = Field(None, description="Transaction amount in USD", title="Amount USD")
+
+ source: CryptoWallet = Field(
+ ..., description="Source wallet", title="Source Wallet"
+ )
+ target: Optional[CryptoWallet] = Field(
+ None, description="Target wallet", title="Target Wallet"
+ )
+ hash: Optional[str] = Field(
+ None, description="Transaction hash", title="Transaction Hash"
+ )
+ value: Optional[float] = Field(
+ None, description="Transaction value in cryptocurrency", title="Value"
+ )
+ amount: Optional[float] = Field(
+ None, description="Transaction amount in cryptocurrency", title="Amount"
+ )
+ amount_usd: Optional[float] = Field(
+ None, description="Transaction amount in USD", title="Amount USD"
+ )
date: Optional[str] = Field(None, description="Transaction date", title="Date")
- hop: Optional[int] = Field(None, description="Hop distance from original wallet", title="Hop Distance")
- timestamp: Optional[str] = Field(None, description="Transaction timestamp (unix epoch)", title="Timestamp")
- block_number: Optional[int] = Field(None, description="Block number", title="Block Number")
- block_hash: Optional[str] = Field(None, description="Block hash", title="Block Hash")
+ hop: Optional[int] = Field(
+ None, description="Hop distance from original wallet", title="Hop Distance"
+ )
+ timestamp: Optional[str] = Field(
+ None, description="Transaction timestamp (unix epoch)", title="Timestamp"
+ )
+ block_number: Optional[int] = Field(
+ None, description="Block number", title="Block Number"
+ )
+ block_hash: Optional[str] = Field(
+ None, description="Block hash", title="Block Hash"
+ )
nonce: Optional[int] = Field(None, description="Transaction nonce", title="Nonce")
- transaction_index: Optional[int] = Field(None, description="Transaction index in block", title="Transaction Index")
+ transaction_index: Optional[int] = Field(
+ None, description="Transaction index in block", title="Transaction Index"
+ )
gas: Optional[int] = Field(None, description="Gas provided", title="Gas")
- gas_price: Optional[int] = Field(None, description="Gas price in wei", title="Gas Price")
+ gas_price: Optional[int] = Field(
+ None, description="Gas price in wei", title="Gas Price"
+ )
gas_used: Optional[int] = Field(None, description="Gas used", title="Gas Used")
- cumulative_gas_used: Optional[int] = Field(None, description="Cumulative gas used", title="Cumulative Gas Used")
+ cumulative_gas_used: Optional[int] = Field(
+ None, description="Cumulative gas used", title="Cumulative Gas Used"
+ )
input: Optional[str] = Field(None, description="Input data", title="Input Data")
- contract_address: Optional[str] = Field(None, description="Contract address", title="Contract Address")
+ contract_address: Optional[str] = Field(
+ None, description="Contract address", title="Contract Address"
+ )
method_id: Optional[str] = Field(None, description="Method ID", title="Method ID")
- function_name: Optional[str] = Field(None, description="Function name", title="Function Name")
- confirmations: Optional[int] = Field(None, description="Number of confirmations", title="Confirmations")
- is_error: Optional[bool] = Field(None, description="Whether the transaction resulted in an error", title="Is Error")
- txreceipt_status: Optional[str] = Field(None, description="Transaction receipt status", title="Transaction Receipt Status")
- error_message: Optional[str] = Field(None, description="Error message if transaction failed", title="Error Message")
+ function_name: Optional[str] = Field(
+ None, description="Function name", title="Function Name"
+ )
+ confirmations: Optional[int] = Field(
+ None, description="Number of confirmations", title="Confirmations"
+ )
+ is_error: Optional[bool] = Field(
+ None,
+ description="Whether the transaction resulted in an error",
+ title="Is Error",
+ )
+ txreceipt_status: Optional[str] = Field(
+ None,
+ description="Transaction receipt status",
+ title="Transaction Receipt Status",
+ )
+ error_message: Optional[str] = Field(
+ None, description="Error message if transaction failed", title="Error Message"
+ )
class CryptoNFT(BaseModel):
"""Represents a Non-Fungible Token (NFT) held or minted by a wallet."""
+
wallet: CryptoWallet = Field(..., description="Source wallet", title="Wallet")
- contract_address: str = Field(..., description="Address of the NFT smart contract (ERC-721/1155)", title="Contract Address")
- token_id: str = Field(..., description="Unique token ID of the NFT within the contract", title="Token ID")
- collection_name: Optional[str] = Field(None, description="Name of the NFT collection", title="Collection Name")
- metadata_url: Optional[HttpUrl] = Field(None, description="URL to the metadata JSON or IPFS resource", title="Metadata URL")
- image_url: Optional[HttpUrl] = Field(None, description="URL to the image or media representing the NFT", title="Image URL")
- name: Optional[str] = Field(None, description="Name or title of the NFT", title="NFT Name")
- description: Optional[str] = Field(None, description="Text description of the NFT", title="Description")
- owner_address: Optional[str] = Field(None, description="Current owner of the NFT", title="Owner Address")
- creator_address: Optional[str] = Field(None, description="Original minter or creator address", title="Creator Address")
- last_transfer_date: Optional[str] = Field(None, description="Date of last transfer or update (ISO format)", title="Last Transfer Date")
- node_id: Optional[str] = Field(None, description="NFT node ID in the Explorer graph", title="Node ID")
+ contract_address: str = Field(
+ ...,
+ description="Address of the NFT smart contract (ERC-721/1155)",
+ title="Contract Address",
+ )
+ token_id: str = Field(
+ ...,
+ description="Unique token ID of the NFT within the contract",
+ title="Token ID",
+ )
+ collection_name: Optional[str] = Field(
+ None, description="Name of the NFT collection", title="Collection Name"
+ )
+ metadata_url: Optional[HttpUrl] = Field(
+ None,
+ description="URL to the metadata JSON or IPFS resource",
+ title="Metadata URL",
+ )
+ image_url: Optional[HttpUrl] = Field(
+ None,
+ description="URL to the image or media representing the NFT",
+ title="Image URL",
+ )
+ name: Optional[str] = Field(
+ None, description="Name or title of the NFT", title="NFT Name"
+ )
+ description: Optional[str] = Field(
+ None, description="Text description of the NFT", title="Description"
+ )
+ owner_address: Optional[str] = Field(
+ None, description="Current owner of the NFT", title="Owner Address"
+ )
+ creator_address: Optional[str] = Field(
+ None, description="Original minter or creator address", title="Creator Address"
+ )
+ last_transfer_date: Optional[str] = Field(
+ None,
+ description="Date of last transfer or update (ISO format)",
+ title="Last Transfer Date",
+ )
+ node_id: Optional[str] = Field(
+ None, description="NFT node ID in the Explorer graph", title="Node ID"
+ )
@property
def uid(self):
@@ -58,4 +139,4 @@ class CryptoNFT(BaseModel):
# Update forward references
CryptoWallet.model_rebuild()
CryptoWalletTransaction.model_rebuild()
-CryptoNFT.model_rebuild()
\ No newline at end of file
+CryptoNFT.model_rebuild()
diff --git a/flowsint-types/src/flowsint_types/weapon.py b/flowsint-types/src/flowsint_types/weapon.py
index 417ce6b3..ee0f4249 100644
--- a/flowsint-types/src/flowsint_types/weapon.py
+++ b/flowsint-types/src/flowsint_types/weapon.py
@@ -1,35 +1,114 @@
from pydantic import BaseModel, Field
from typing import Literal, Optional, List
+
class Weapon(BaseModel):
"""Represents a weapon with detailed specifications and forensic information."""
+
name: str = Field(..., description="Weapon name or identifier", title="Name")
- type: Optional[Literal["firearm", "melee", "explosive", "bladed", "blunt", "chemical", "biological", "radiological", "nuclear", "other"]] = Field(None, description="Weapon category (handgun, rifle, shotgun, knife, etc.)", title="Category")
- description: Optional[str] = Field(None, description="Weapon description", title="Description")
- manufacturer: Optional[str] = Field(None, description="Weapon manufacturer", title="Manufacturer")
+ type: Optional[
+ Literal[
+ "firearm",
+ "melee",
+ "explosive",
+ "bladed",
+ "blunt",
+ "chemical",
+ "biological",
+ "radiological",
+ "nuclear",
+ "other",
+ ]
+ ] = Field(
+ None,
+ description="Weapon category (handgun, rifle, shotgun, knife, etc.)",
+ title="Category",
+ )
+ description: Optional[str] = Field(
+ None, description="Weapon description", title="Description"
+ )
+ manufacturer: Optional[str] = Field(
+ None, description="Weapon manufacturer", title="Manufacturer"
+ )
model: Optional[str] = Field(None, description="Weapon model", title="Model")
- caliber: Optional[str] = Field(None, description="Caliber or ammunition type", title="Caliber")
- serial_number: Optional[str] = Field(None, description="Weapon serial number", title="Serial Number")
- year_manufactured: Optional[int] = Field(None, description="Year weapon was manufactured", title="Year Manufactured")
- country_of_origin: Optional[str] = Field(None, description="Country where weapon was manufactured", title="Country of Origin")
- legal_status: Optional[str] = Field(None, description="Legal status (legal, illegal, restricted, etc.)", title="Legal Status")
- registration_status: Optional[str] = Field(None, description="Registration status", title="Registration Status")
- condition: Optional[str] = Field(None, description="Physical condition of the weapon", title="Condition")
- modifications: Optional[List[str]] = Field(None, description="Any modifications made to the weapon", title="Modifications")
- accessories: Optional[List[str]] = Field(None, description="Attached accessories", title="Accessories")
- capacity: Optional[str] = Field(None, description="Magazine capacity or ammunition capacity", title="Capacity")
- barrel_length: Optional[str] = Field(None, description="Barrel length", title="Barrel Length")
- overall_length: Optional[str] = Field(None, description="Overall weapon length", title="Overall Length")
+ caliber: Optional[str] = Field(
+ None, description="Caliber or ammunition type", title="Caliber"
+ )
+ serial_number: Optional[str] = Field(
+ None, description="Weapon serial number", title="Serial Number"
+ )
+ year_manufactured: Optional[int] = Field(
+ None, description="Year weapon was manufactured", title="Year Manufactured"
+ )
+ country_of_origin: Optional[str] = Field(
+ None,
+ description="Country where weapon was manufactured",
+ title="Country of Origin",
+ )
+ legal_status: Optional[str] = Field(
+ None,
+ description="Legal status (legal, illegal, restricted, etc.)",
+ title="Legal Status",
+ )
+ registration_status: Optional[str] = Field(
+ None, description="Registration status", title="Registration Status"
+ )
+ condition: Optional[str] = Field(
+ None, description="Physical condition of the weapon", title="Condition"
+ )
+ modifications: Optional[List[str]] = Field(
+ None, description="Any modifications made to the weapon", title="Modifications"
+ )
+ accessories: Optional[List[str]] = Field(
+ None, description="Attached accessories", title="Accessories"
+ )
+ capacity: Optional[str] = Field(
+ None, description="Magazine capacity or ammunition capacity", title="Capacity"
+ )
+ barrel_length: Optional[str] = Field(
+ None, description="Barrel length", title="Barrel Length"
+ )
+ overall_length: Optional[str] = Field(
+ None, description="Overall weapon length", title="Overall Length"
+ )
weight: Optional[str] = Field(None, description="Weapon weight", title="Weight")
- action_type: Optional[str] = Field(None, description="Action type (semi-automatic, bolt-action, etc.)", title="Action Type")
- firing_mode: Optional[str] = Field(None, description="Firing mode (single shot, burst, full auto, etc.)", title="Firing Mode")
- safety_features: Optional[List[str]] = Field(None, description="Safety features present", title="Safety Features")
- source: Optional[str] = Field(None, description="Source of weapon information", title="Source")
- location_found: Optional[str] = Field(None, description="Location where weapon was found", title="Location Found")
- date_found: Optional[str] = Field(None, description="Date weapon was found", title="Date Found")
- evidence_number: Optional[str] = Field(None, description="Evidence number or case reference", title="Evidence Number")
- owner: Optional[str] = Field(None, description="Registered owner or possessor", title="Owner")
- purchase_date: Optional[str] = Field(None, description="Date weapon was purchased", title="Purchase Date")
- purchase_location: Optional[str] = Field(None, description="Location where weapon was purchased", title="Purchase Location")
+ action_type: Optional[str] = Field(
+ None,
+ description="Action type (semi-automatic, bolt-action, etc.)",
+ title="Action Type",
+ )
+ firing_mode: Optional[str] = Field(
+ None,
+ description="Firing mode (single shot, burst, full auto, etc.)",
+ title="Firing Mode",
+ )
+ safety_features: Optional[List[str]] = Field(
+ None, description="Safety features present", title="Safety Features"
+ )
+ source: Optional[str] = Field(
+ None, description="Source of weapon information", title="Source"
+ )
+ location_found: Optional[str] = Field(
+ None, description="Location where weapon was found", title="Location Found"
+ )
+ date_found: Optional[str] = Field(
+ None, description="Date weapon was found", title="Date Found"
+ )
+ evidence_number: Optional[str] = Field(
+ None, description="Evidence number or case reference", title="Evidence Number"
+ )
+ owner: Optional[str] = Field(
+ None, description="Registered owner or possessor", title="Owner"
+ )
+ purchase_date: Optional[str] = Field(
+ None, description="Date weapon was purchased", title="Purchase Date"
+ )
+ purchase_location: Optional[str] = Field(
+ None,
+ description="Location where weapon was purchased",
+ title="Purchase Location",
+ )
price: Optional[float] = Field(None, description="Purchase price", title="Price")
- notes: Optional[str] = Field(None, description="Additional notes or observations", title="Notes")
\ No newline at end of file
+ notes: Optional[str] = Field(
+ None, description="Additional notes or observations", title="Notes"
+ )
diff --git a/flowsint-types/src/flowsint_types/web_tracker.py b/flowsint-types/src/flowsint_types/web_tracker.py
index 09f9386b..36ba0450 100644
--- a/flowsint-types/src/flowsint_types/web_tracker.py
+++ b/flowsint-types/src/flowsint_types/web_tracker.py
@@ -1,23 +1,59 @@
from pydantic import BaseModel, Field
from typing import Optional, List
+
class WebTracker(BaseModel):
"""Represents a web tracking technology with privacy and compliance information."""
- tracker_id: str = Field(..., description="Unique tracker identifier", title="Tracker ID")
+
+ tracker_id: str = Field(
+ ..., description="Unique tracker identifier", title="Tracker ID"
+ )
name: Optional[str] = Field(None, description="Tracker name", title="Name")
- type: Optional[str] = Field(None, description="Type of tracker (analytics, advertising, etc.)", title="Type")
- domain: Optional[str] = Field(None, description="Domain where tracker is deployed", title="Domain")
- script_url: Optional[str] = Field(None, description="URL of tracking script", title="Script URL")
- company: Optional[str] = Field(None, description="Company providing the tracker", title="Company")
- purpose: Optional[str] = Field(None, description="Purpose of tracking", title="Purpose")
- data_collected: Optional[List[str]] = Field(None, description="Types of data collected", title="Data Collected")
- privacy_policy: Optional[str] = Field(None, description="Privacy policy URL", title="Privacy Policy")
- opt_out_url: Optional[str] = Field(None, description="Opt-out URL", title="Opt-out URL")
- first_seen: Optional[str] = Field(None, description="First time tracker was observed", title="First Seen")
- last_seen: Optional[str] = Field(None, description="Last time tracker was observed", title="Last Seen")
- is_active: Optional[bool] = Field(None, description="Whether tracker is currently active", title="Is Active")
- is_third_party: Optional[bool] = Field(None, description="Whether tracker is third-party", title="Is Third Party")
- cookie_duration: Optional[int] = Field(None, description="Cookie duration in days", title="Cookie Duration")
- gdpr_compliant: Optional[bool] = Field(None, description="Whether tracker is GDPR compliant", title="GDPR Compliant")
- source: Optional[str] = Field(None, description="Source of tracker information", title="Source")
- risk_level: Optional[str] = Field(None, description="Privacy risk level", title="Risk Level")
\ No newline at end of file
+ type: Optional[str] = Field(
+ None, description="Type of tracker (analytics, advertising, etc.)", title="Type"
+ )
+ domain: Optional[str] = Field(
+ None, description="Domain where tracker is deployed", title="Domain"
+ )
+ script_url: Optional[str] = Field(
+ None, description="URL of tracking script", title="Script URL"
+ )
+ company: Optional[str] = Field(
+ None, description="Company providing the tracker", title="Company"
+ )
+ purpose: Optional[str] = Field(
+ None, description="Purpose of tracking", title="Purpose"
+ )
+ data_collected: Optional[List[str]] = Field(
+ None, description="Types of data collected", title="Data Collected"
+ )
+ privacy_policy: Optional[str] = Field(
+ None, description="Privacy policy URL", title="Privacy Policy"
+ )
+ opt_out_url: Optional[str] = Field(
+ None, description="Opt-out URL", title="Opt-out URL"
+ )
+ first_seen: Optional[str] = Field(
+ None, description="First time tracker was observed", title="First Seen"
+ )
+ last_seen: Optional[str] = Field(
+ None, description="Last time tracker was observed", title="Last Seen"
+ )
+ is_active: Optional[bool] = Field(
+ None, description="Whether tracker is currently active", title="Is Active"
+ )
+ is_third_party: Optional[bool] = Field(
+ None, description="Whether tracker is third-party", title="Is Third Party"
+ )
+ cookie_duration: Optional[int] = Field(
+ None, description="Cookie duration in days", title="Cookie Duration"
+ )
+ gdpr_compliant: Optional[bool] = Field(
+ None, description="Whether tracker is GDPR compliant", title="GDPR Compliant"
+ )
+ source: Optional[str] = Field(
+ None, description="Source of tracker information", title="Source"
+ )
+ risk_level: Optional[str] = Field(
+ None, description="Privacy risk level", title="Risk Level"
+ )
diff --git a/flowsint-types/src/flowsint_types/website.py b/flowsint-types/src/flowsint_types/website.py
index 15868c02..3e986360 100644
--- a/flowsint-types/src/flowsint_types/website.py
+++ b/flowsint-types/src/flowsint_types/website.py
@@ -2,9 +2,19 @@ from typing import List, Optional
from pydantic import BaseModel, Field, HttpUrl
from .domain import Domain
+
class Website(BaseModel):
"""Represents a website with its URL, domain, and redirect information."""
- url: HttpUrl = Field(..., description="Full URL of the website", title="Website URL")
- redirects: Optional[List[HttpUrl]] = Field([], description="List of redirects from the website", title="Redirects")
- domain: Optional[Domain] = Field(None, description="Domain information for the website", title="Domain")
- active: Optional[bool] = Field(False, description="Whether the website is active", title="Is Active")
\ No newline at end of file
+
+ url: HttpUrl = Field(
+ ..., description="Full URL of the website", title="Website URL"
+ )
+ redirects: Optional[List[HttpUrl]] = Field(
+ [], description="List of redirects from the website", title="Redirects"
+ )
+ domain: Optional[Domain] = Field(
+ None, description="Domain information for the website", title="Domain"
+ )
+ active: Optional[bool] = Field(
+ False, description="Whether the website is active", title="Is Active"
+ )
diff --git a/flowsint-types/src/flowsint_types/whois.py b/flowsint-types/src/flowsint_types/whois.py
index cab163d3..f19e6b56 100644
--- a/flowsint-types/src/flowsint_types/whois.py
+++ b/flowsint-types/src/flowsint_types/whois.py
@@ -2,13 +2,31 @@ from typing import Optional
from pydantic import BaseModel, Field
from .email import Email
+
class Whois(BaseModel):
"""Represents WHOIS domain registration information."""
+
domain: str = Field(..., description="Domain name", title="Domain")
- registrar: Optional[str] = Field(None, description="Domain registrar name", title="Registrar")
- org: Optional[str] = Field(None, description="Organization name associated with the domain", title="Organization")
- city: Optional[str] = Field(None, description="City where the domain is registered", title="City")
- country: Optional[str] = Field(None, description="Country where the domain is registered", title="Country")
- email: Optional[Email] = Field(None, description="Contact email for the domain", title="Contact Email")
- creation_date: Optional[str] = Field(None, description="Date when the domain was created", title="Creation Date")
- expiration_date: Optional[str] = Field(None, description="Date when the domain expires", title="Expiration Date")
+ registrar: Optional[str] = Field(
+ None, description="Domain registrar name", title="Registrar"
+ )
+ org: Optional[str] = Field(
+ None,
+ description="Organization name associated with the domain",
+ title="Organization",
+ )
+ city: Optional[str] = Field(
+ None, description="City where the domain is registered", title="City"
+ )
+ country: Optional[str] = Field(
+ None, description="Country where the domain is registered", title="Country"
+ )
+ email: Optional[Email] = Field(
+ None, description="Contact email for the domain", title="Contact Email"
+ )
+ creation_date: Optional[str] = Field(
+ None, description="Date when the domain was created", title="Creation Date"
+ )
+ expiration_date: Optional[str] = Field(
+ None, description="Date when the domain expires", title="Expiration Date"
+ )