A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat: python formatter finally working
dextmorgn7620a6d145ac
diff --git a/flowsint-api/alembic/env.py b/flowsint-api/alembic/env.pyindex 2280ed23..a9734b5b 100644--- a/flowsint-api/alembic/env.py+++ b/flowsint-api/alembic/env.py@@ -10,10 +10,9 @@ from dotenv import load_dotenvload_dotenv()# Permet les imports comme from app.models.base import Base-sys.path.append(os.path.join(os.path.dirname(__file__), '..'))+sys.path.append(os.path.join(os.path.dirname(__file__), ".."))# Importer Base et toutes les tables pour qu'Alembic les détecte-from app.models.base import Basefrom app.models.models import * # noqa# Configuration Alembic@@ -30,6 +29,7 @@ config.set_main_option("sqlalchemy.url", database_url)# Définir la métadonnée cible pour autogeneratetarget_metadata = Base.metadata+def run_migrations_offline() -> None:"""Run migrations in 'offline' mode."""url = config.get_main_option("sqlalchemy.url")@@ -42,6 +42,7 @@ def run_migrations_offline() -> None:with context.begin_transaction():context.run_migrations()+def run_migrations_online() -> None:"""Run migrations in 'online' mode."""connectable = engine_from_config(@@ -55,6 +56,7 @@ def run_migrations_online() -> None:with context.begin_transaction():context.run_migrations()+if context.is_offline_mode():run_migrations_offline()else:diff --git a/flowsint-api/app/api/deps.py b/flowsint-api/app/api/deps.pyindex d33ac39c..bf4d8fa6 100644--- a/flowsint-api/app/api/deps.py+++ b/flowsint-api/app/api/deps.py@@ -8,6 +8,7 @@ from app.models.models import Profileimport osfrom dotenv import load_dotenv+# Remplace avec ton URL de BDDAUTH_SECRET = os.getenv("AUTH_SECRET")@@ -15,7 +16,10 @@ load_dotenv()oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")-def get_current_user(token: str = Depends(oauth2_scheme), db: Session = Depends(get_db)) -> Profile:++def get_current_user(+ token: str = Depends(oauth2_scheme), db: Session = Depends(get_db)+) -> Profile:credentials_exception = HTTPException(status_code=status.HTTP_401_UNAUTHORIZED,detail="Could not validate credentials",diff --git a/flowsint-api/app/api/routes/analysis.py b/flowsint-api/app/api/routes/analysis.pyindex bebc31da..643632fa 100644--- a/flowsint-api/app/api/routes/analysis.py+++ b/flowsint-api/app/api/routes/analysis.py@@ -10,15 +10,25 @@ from app.api.schemas.analysis import AnalysisRead, AnalysisCreate, AnalysisUpdatrouter = APIRouter()+# Get the list of all analyses for the current user@router.get("", response_model=List[AnalysisRead])-def get_analyses(db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):+def get_analyses(+ db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)+):analyses = db.query(Analysis).filter(Analysis.owner_id == current_user.id).all()return analyses+# Create a new analysis[email protected]("/create", response_model=AnalysisRead, status_code=status.HTTP_201_CREATED)-def create_analysis(payload: AnalysisCreate, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):+ "/create", response_model=AnalysisRead, status_code=status.HTTP_201_CREATED+)+def create_analysis(+ payload: AnalysisCreate,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):new_analysis = Analysis(id=uuid4(),title=payload.title,@@ -34,31 +44,55 @@ def create_analysis(payload: AnalysisCreate, db: Session = Depends(get_db), currdb.refresh(new_analysis)return new_analysis+# Get an analysis by ID@router.get("/{analysis_id}", response_model=AnalysisRead)-def get_analysis_by_id(analysis_id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):- analysis = db.query(Analysis).filter(Analysis.id == analysis_id, Analysis.owner_id == current_user.id).first()+def get_analysis_by_id(+ analysis_id: UUID,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):+ analysis = (+ db.query(Analysis)+ .filter(Analysis.id == analysis_id, Analysis.owner_id == current_user.id)+ .first()+ )if not analysis:raise HTTPException(status_code=404, detail="Analysis not found")return analysis+# Get analyses by investigation ID@router.get("/investigation/{investigation_id}", response_model=List[AnalysisRead])def get_analyses_by_investigation(investigation_id: UUID,db: Session = Depends(get_db),- current_user: Profile = Depends(get_current_user)+ current_user: Profile = Depends(get_current_user),):- analyses = db.query(Analysis).filter(- Analysis.investigation_id == investigation_id,- Analysis.owner_id == current_user.id- ).all()+ analyses = (+ db.query(Analysis)+ .filter(+ Analysis.investigation_id == investigation_id,+ Analysis.owner_id == current_user.id,+ )+ .all()+ )return analyses+# Update an analysis by ID@router.put("/{analysis_id}", response_model=AnalysisRead)-def update_analysis(analysis_id: UUID, payload: AnalysisUpdate, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):- analysis = db.query(Analysis).filter(Analysis.id == analysis_id, Analysis.owner_id == current_user.id).first()+def update_analysis(+ analysis_id: UUID,+ payload: AnalysisUpdate,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):+ analysis = (+ db.query(Analysis)+ .filter(Analysis.id == analysis_id, Analysis.owner_id == current_user.id)+ .first()+ )if not analysis:raise HTTPException(status_code=404, detail="Analysis not found")if payload.title is not None:@@ -74,12 +108,21 @@ def update_analysis(analysis_id: UUID, payload: AnalysisUpdate, db: Session = Dedb.refresh(analysis)return analysis+# Delete an analysis by ID@router.delete("/{analysis_id}", status_code=status.HTTP_204_NO_CONTENT)-def delete_analysis(analysis_id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):- analysis = db.query(Analysis).filter(Analysis.id == analysis_id, Analysis.owner_id == current_user.id).first()+def delete_analysis(+ analysis_id: UUID,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):+ analysis = (+ db.query(Analysis)+ .filter(Analysis.id == analysis_id, Analysis.owner_id == current_user.id)+ .first()+ )if not analysis:raise HTTPException(status_code=404, detail="Analysis not found")db.delete(analysis)db.commit()- return None+ return Nonediff --git a/flowsint-api/app/api/routes/auth.py b/flowsint-api/app/api/routes/auth.pyindex 6a87ff76..3e8c7164 100644--- a/flowsint-api/app/api/routes/auth.py+++ b/flowsint-api/app/api/routes/auth.py@@ -10,14 +10,18 @@ from flowsint_core.core.postgre_db import get_dbrouter = APIRouter()+@router.post("/token")-def login_for_access_token(form_data: OAuth2PasswordRequestForm = Depends(), db: Session = Depends(get_db)):+def login_for_access_token(+ form_data: OAuth2PasswordRequestForm = Depends(), db: Session = Depends(get_db)+):user = db.query(Profile).filter(Profile.email == form_data.username).first()if not user or not verify_password(form_data.password, user.hashed_password):raise HTTPException(status_code=400, detail="Incorrect email or password")access_token = create_access_token(data={"sub": user.email})return {"access_token": access_token, "user_id": user.id, "token_type": "bearer"}+@router.post("/register", status_code=201)def register(user: ProfileCreate, db: Session = Depends(get_db)):print(user)@@ -30,6 +34,5 @@ def register(user: ProfileCreate, db: Session = Depends(get_db)):db.add(new_user)db.commit()db.refresh(new_user)-return {"message": "User registered successfully", "email": new_user.email}diff --git a/flowsint-api/app/api/routes/chat.py b/flowsint-api/app/api/routes/chat.pyindex 0b93d9f2..b7fd1cb8 100644--- a/flowsint-api/app/api/routes/chat.py+++ b/flowsint-api/app/api/routes/chat.py@@ -14,6 +14,7 @@ from flowsint_core.core.postgre_db import get_dbfrom app.models.models import Chat, ChatMessage, Profilefrom app.api.deps import get_current_userfrom app.api.schemas.chat import ChatCreate, ChatRead+router = APIRouter()@@ -25,22 +26,23 @@ def clean_context(context: List[Dict]) -> List[Dict]:if isinstance(item, dict):# Create a copy and remove unwanted keyscleaned_item = item["data"].copy()-+# Remove top-level keys- cleaned_item.pop('id', None)- cleaned_item.pop('sketch_id', None)-+ cleaned_item.pop("id", None)+ cleaned_item.pop("sketch_id", None)+# Remove from data if it exists- if 'data' in cleaned_item and isinstance(cleaned_item['data'], dict):- cleaned_item['data'].pop('sketch_id', None)-+ if "data" in cleaned_item and isinstance(cleaned_item["data"], dict):+ cleaned_item["data"].pop("sketch_id", None)+# Remove measured/dimensions- cleaned_item.pop('measured', None)-+ cleaned_item.pop("measured", None)+cleaned.append(cleaned_item)print(cleaned)return cleaned+class ChatRequest(BaseModel):prompt: strcontext: Optional[List[Dict]] = None@@ -49,17 +51,14 @@ class ChatRequest(BaseModel):# Get all chats@router.get("/", response_model=List[ChatRead])def get_chats(- db: Session = Depends(get_db),- current_user: Profile = Depends(get_current_user)+ db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):- chats = db.query(Chat).filter(- Chat.owner_id == current_user.id- ).all()-+ chats = db.query(Chat).filter(Chat.owner_id == current_user.id).all()+# Sort messages for each chat by created_at in ascending orderfor chat in chats:chat.messages.sort(key=lambda x: x.created_at)-+return chats@@ -69,7 +68,7 @@ def get_chats(# for chat in chats:# db.delete(chat)# db.commit()-# return {"result": "done"}+# return {"result": "done"}# Get analyses by investigation ID@@ -77,26 +76,39 @@ def get_chats(def get_chats_by_investigation(investigation_id: UUID,db: Session = Depends(get_db),- current_user: Profile = Depends(get_current_user)+ current_user: Profile = Depends(get_current_user),):- chats = db.query(Chat).filter(- Chat.investigation_id == investigation_id,- Chat.owner_id == current_user.id- ).all()-+ chats = (+ db.query(Chat)+ .filter(+ Chat.investigation_id == investigation_id, Chat.owner_id == current_user.id+ )+ .all()+ )+# Sort messages for each chat by created_at in ascending orderfor chat in chats:chat.messages.sort(key=lambda x: x.created_at)-+return chats+@router.post("/stream/{chat_id}")-async def stream_chat(chat_id: UUID, payload: ChatRequest, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):+async def stream_chat(+ chat_id: UUID,+ payload: ChatRequest,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):# Check if Chat exists- chat = db.query(Chat).filter(Chat.id == chat_id, Chat.owner_id == current_user.id).first()+ chat = (+ db.query(Chat)+ .filter(Chat.id == chat_id, Chat.owner_id == current_user.id)+ .first()+ )if not chat:raise HTTPException(status_code=404, detail="Chat not found")-+# Update chat's last_updated_atchat.last_updated_at = datetime.utcnow()db.commit()@@ -112,19 +124,25 @@ async def stream_chat(chat_id: UUID, payload: ChatRequest, db: Session = Dependsdb.add(user_message)db.commit()db.refresh(user_message)-+try:api_key = os.environ.get("MISTRAL_API_KEY")if not api_key:- raise HTTPException(status_code=500, detail="Mistral API key not configured")-+ raise HTTPException(+ status_code=500, detail="Mistral API key not configured"+ )+client = Mistral(api_key=api_key)model = "mistral-small-latest"accumulated_content = []context_message = None# Convert database messages to Mistral format- messages = [SystemMessage(content="You are a CTI/OSINT investigator and you are trying to investigate on a variety of real life cases. Use your knowledge and analytics capabilities to analyse the context and answer the question the best you can. If you need to reference some items (an IP, a domain or something particular) please use the code brackets, like : `12.23.34.54` to reference it.")]-+ messages = [+ SystemMessage(+ content="You are a CTI/OSINT investigator and you are trying to investigate on a variety of real life cases. Use your knowledge and analytics capabilities to analyse the context and answer the question the best you can. If you need to reference some items (an IP, a domain or something particular) please use the code brackets, like : `12.23.34.54` to reference it."+ )+ ]+# Add context as a single system message if providedif payload.context:try:@@ -139,16 +157,22 @@ async def stream_chat(chat_id: UUID, payload: ChatRequest, db: Session = Dependsexcept Exception as e:# If context processing fails, skip itprint(f"Context processing error: {e}")-+# Sort messages by created_at in ascending order and get recent messagessorted_messages = sorted(chat.messages, key=lambda x: x.created_at)- recent_messages = sorted_messages[-5:] if len(sorted_messages) > 5 else sorted_messages+ recent_messages = (+ sorted_messages[-5:] if len(sorted_messages) > 5 else sorted_messages+ )for message in recent_messages:if message.is_bot:- messages.append(AssistantMessage(content=json.dumps(message.content, default=str)))+ messages.append(+ AssistantMessage(content=json.dumps(message.content, default=str))+ )else:- messages.append(UserMessage(content=json.dumps(message.content, default=str)))-+ messages.append(+ UserMessage(content=json.dumps(message.content, default=str))+ )+# Add the current contextif context_message:messages.append(SystemMessage(content=context_message))@@ -156,17 +180,14 @@ async def stream_chat(chat_id: UUID, payload: ChatRequest, db: Session = Dependsmessages.append(UserMessage(content=payload.prompt))async def generate():- response = await client.chat.stream_async(- model=model,- messages=messages- )-+ response = await client.chat.stream_async(model=model, messages=messages)+async for chunk in response:if chunk.data.choices[0].delta.content is not None:content_chunk = chunk.data.choices[0].delta.contentaccumulated_content.append(content_chunk)yield f"data: {json.dumps({'content': content_chunk})}\n\n"-+# Create the bot message after all chunks have been processedchat_message = ChatMessage(id=uuid4(),@@ -175,24 +196,25 @@ async def stream_chat(chat_id: UUID, payload: ChatRequest, db: Session = Dependsis_bot=True,created_at=datetime.utcnow(),)-+db.add(chat_message)db.commit()db.refresh(chat_message)-+yield "data: [DONE]\n\n"- return StreamingResponse(- generate(),- media_type="text/event-stream"- )+ return StreamingResponse(generate(), media_type="text/event-stream")except Exception as e:raise HTTPException(status_code=500, detail=str(e))--++# Create a new chat@router.post("/create", response_model=ChatRead, status_code=status.HTTP_201_CREATED)-def create_chat(payload: ChatCreate, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):+def create_chat(+ payload: ChatCreate,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):new_chat = Chat(id=uuid4(),title=payload.title,@@ -207,29 +229,42 @@ def create_chat(payload: ChatCreate, db: Session = Depends(get_db), current_userdb.refresh(new_chat)return new_chat+# Get a chat by ID@router.get("/{chat_id}", response_model=ChatRead)-def get_chat_by_id(chat_id: UUID, db: Session = Depends(get_db),- current_user: Profile = Depends(get_current_user)- ):- chat = db.query(Chat).filter(Chat.id == chat_id,- Chat.owner_id == current_user.id- ).first()+def get_chat_by_id(+ chat_id: UUID,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):+ chat = (+ db.query(Chat)+ .filter(Chat.id == chat_id, Chat.owner_id == current_user.id)+ .first()+ )if not chat:raise HTTPException(status_code=404, detail="Chat not found")-+# Sort messages by created_at in ascending orderchat.messages.sort(key=lambda x: x.created_at)-+return chat# Delete an chat by ID@router.delete("/{chat_id}", status_code=status.HTTP_204_NO_CONTENT)-def delete_chat(chat_id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):- chat = db.query(Chat).filter(Chat.id == chat_id, Chat.owner_id == current_user.id).first()+def delete_chat(+ chat_id: UUID,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):+ chat = (+ db.query(Chat)+ .filter(Chat.id == chat_id, Chat.owner_id == current_user.id)+ .first()+ )if not chat:raise HTTPException(status_code=404, detail="Chat not found")db.delete(chat)db.commit()- return None+ return Nonediff --git a/flowsint-api/app/api/routes/events.py b/flowsint-api/app/api/routes/events.pyindex d56187e9..92b3f260 100644--- a/flowsint-api/app/api/routes/events.py+++ b/flowsint-api/app/api/routes/events.py@@ -13,6 +13,7 @@ from datetime import datetime, timedeltarouter = APIRouter()+@router.get("/sketch/{sketch_id}/logs")def get_logs_by_sketch(sketch_id: str,@@ -20,27 +21,33 @@ def get_logs_by_sketch(since: datetime | None = None,db: Session = Depends(get_db),# current_user: Profile = Depends(get_current_user)-):+):"""Get historical logs for a specific sketch with optional filtering"""# Check if sketch existssketch = db.query(Sketch).filter(Sketch.id == sketch_id).first()if not sketch:- raise HTTPException(status_code=404, detail=f"Sketch with id {sketch_id} not found")+ raise HTTPException(+ status_code=404, detail=f"Sketch with id {sketch_id} not found"+ )++ print(+ f"[EventEmitter] Fetching logs for sketch {sketch_id} (limit: {limit}, since: {since})"+ )+ query = (+ db.query(Log).filter(Log.sketch_id == sketch_id).order_by(Log.created_at.desc())+ )- print(f"[EventEmitter] Fetching logs for sketch {sketch_id} (limit: {limit}, since: {since})")- query = db.query(Log).filter(Log.sketch_id == sketch_id).order_by(Log.created_at.desc())-if since:query = query.filter(Log.created_at > since)else:# Default to last 24 hours if no since parameterquery = query.filter(Log.created_at > datetime.utcnow() - timedelta(days=1))-+logs = query.limit(limit).all()-+# Reverse to show chronologically (oldest to newest)logs = list(reversed(logs))-+results = []for log in logs:# Ensure payload is always a dictionary@@ -53,53 +60,58 @@ def get_logs_by_sketch(else:# Handle other types by converting to string and wrappingpayload = {"content": str(log.content)}-- results.append(Event(- id=str(log.id),- sketch_id=str(log.sketch_id) if log.sketch_id else None,- type=log.type,- payload=payload- ))-++ results.append(+ Event(+ id=str(log.id),+ sketch_id=str(log.sketch_id) if log.sketch_id else None,+ type=log.type,+ payload=payload,+ )+ )return results@router.get("/sketch/{sketch_id}/stream")-async def stream_events(request: Request, sketch_id: str,- db: Session = Depends(get_db),- # current_user: Profile = Depends(get_current_user)- ):+async def stream_events(+ request: Request,+ sketch_id: str,+ db: Session = Depends(get_db),+ # current_user: Profile = Depends(get_current_user)+):"""Stream events for a specific scan in real-time"""# Check if sketch existssketch = db.query(Sketch).filter(Sketch.id == sketch_id).first()if not sketch:- raise HTTPException(status_code=404, detail=f"Sketch with id {sketch_id} not found")+ raise HTTPException(+ status_code=404, detail=f"Sketch with id {sketch_id} not found"+ )async def event_generator():channel = sketch_idawait event_emitter.subscribe(channel)try:# Initial connection message- yield "data: {\"event\": \"connected\", \"data\": \"Connected to log stream\"}\n\n"+ yield 'data: {"event": "connected", "data": "Connected to log stream"}\n\n'while True:if await request.is_disconnected():break-+data = await event_emitter.get_message(channel)if data is None:- await asyncio.sleep(.1) # avoid tight loop on None+ await asyncio.sleep(0.1) # avoid tight loop on Nonecontinue-+# Handle different types of events- if isinstance(data, dict) and data.get('type') == 'scanner_complete':+ if isinstance(data, dict) and data.get("type") == "scanner_complete":# Send scanner completion event- yield json.dumps({'event': 'scanner_complete', 'data': data})+ yield json.dumps({"event": "scanner_complete", "data": data})else:# Send regular log event- yield json.dumps({'event': 'log', 'data': data})- await asyncio.sleep(.1)+ yield json.dumps({"event": "log", "data": data})+ await asyncio.sleep(0.1)except asyncio.CancelledError:print(f"[EventEmitter] Client disconnected from sketch_id: {sketch_id}")@@ -114,16 +126,17 @@ async def stream_events(request: Request, sketch_id: str,headers={"Cache-Control": "no-cache","Connection": "keep-alive",- "X-Accel-Buffering": "no"- }+ "X-Accel-Buffering": "no",+ },)+@router.delete("/sketch/{sketch_id}/logs")def delete_scan_logs(- sketch_id:str,+ sketch_id: str,db: Session = Depends(get_db),- current_user: Profile = Depends(get_current_user)- ):+ current_user: Profile = Depends(get_current_user),+):"""Delete all logs for a specific scan"""try:db.query(Log).filter(Log.sketch_id == sketch_id).delete()@@ -133,9 +146,9 @@ def delete_scan_logs(db.rollback()raise HTTPException(status_code=500, detail=f"Failed to delete logs: {str(e)}")+@router.get("/status/scan/{scan_id}/stream")-async def stream_status(request: Request, scan_id: str,- db: Session = Depends(get_db)):+async def stream_status(request: Request, scan_id: str, db: Session = Depends(get_db)):"""Stream status updates for a specific scan in real-time"""async def status_generator():@@ -143,7 +156,7 @@ async def stream_status(request: Request, scan_id: str,await event_emitter.subscribe(f"scan_{scan_id}_status")try:# Initial connection message- yield "data: {\"event\": \"connected\", \"data\": \"Connected to status stream\"}\n\n"+ yield 'data: {"event": "connected", "data": "Connected to status stream"}\n\n'while True:data = await event_emitter.get_message(f"scan_{scan_id}_status")@@ -154,7 +167,9 @@ async def stream_status(request: Request, scan_id: str,yield f"data: {data}\n\n"except asyncio.CancelledError:- print(f"[EventEmitter] Client disconnected from status stream for scan_id: {scan_id}")+ print(+ f"[EventEmitter] Client disconnected from status stream for scan_id: {scan_id}"+ )finally:await event_emitter.unsubscribe(f"scan_{scan_id}_status")@@ -164,6 +179,6 @@ async def stream_status(request: Request, scan_id: str,headers={"Cache-Control": "no-cache","Connection": "keep-alive",- "X-Accel-Buffering": "no"- }- )\ No newline at end of file+ "X-Accel-Buffering": "no",+ },+ )diff --git a/flowsint-api/app/api/routes/investigations.py b/flowsint-api/app/api/routes/investigations.pyindex 496e243b..8661e489 100644--- a/flowsint-api/app/api/routes/investigations.py+++ b/flowsint-api/app/api/routes/investigations.py@@ -6,21 +6,37 @@ from sqlalchemy.orm import Session, selectinloadfrom flowsint_core.core.postgre_db import get_dbfrom app.models.models import Analysis, Investigation, Profile, Sketchfrom app.api.deps import get_current_user-from app.api.schemas.investigation import InvestigationRead, InvestigationCreate, InvestigationUpdate+from app.api.schemas.investigation import (+ InvestigationRead,+ InvestigationCreate,+ InvestigationUpdate,+)from app.api.schemas.sketch import SketchReadfrom flowsint_core.core.graph_db import neo4j_connectionrouter = APIRouter()+# Get the list of all investigations@router.get("", response_model=List[InvestigationRead])-def get_investigations(db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):- investigations = db.query(Investigation).filter(Investigation.owner_id == current_user.id).all()+def get_investigations(+ db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)+):+ investigations = (+ db.query(Investigation).filter(Investigation.owner_id == current_user.id).all()+ )return investigations+# Create a new investigation[email protected]("/create", response_model=InvestigationRead, status_code=status.HTTP_201_CREATED)-def create_investigation(payload: InvestigationCreate, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):+ "/create", response_model=InvestigationRead, status_code=status.HTTP_201_CREATED+)+def create_investigation(+ payload: InvestigationCreate,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):new_investigation = Investigation(id=uuid4(),name=payload.name,@@ -35,33 +51,57 @@ def create_investigation(payload: InvestigationCreate, db: Session = Depends(getdb.refresh(new_investigation)return new_investigation+# Get a investigation by ID@router.get("/{investigation_id}", response_model=InvestigationRead)-def get_investigation_by_id(investigation_id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):- investigation = db.query(Investigation).options(selectinload(Investigation.sketches)).filter(Investigation.id == investigation_id).filter(Investigation.owner_id == current_user.id).first()+def get_investigation_by_id(+ investigation_id: UUID,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):+ investigation = (+ db.query(Investigation)+ .options(selectinload(Investigation.sketches))+ .filter(Investigation.id == investigation_id)+ .filter(Investigation.owner_id == current_user.id)+ .first()+ )if not investigation:raise HTTPException(status_code=404, detail="Investigation not found")return investigation+# Get a investigation by ID@router.get("/{investigation_id}/sketches", response_model=List[SketchRead])def get_sketches_by_investigation(investigation_id: UUID,db: Session = Depends(get_db),- current_user: Profile = Depends(get_current_user)+ current_user: Profile = Depends(get_current_user),):- sketches = db.query(Sketch).filter(Sketch.investigation_id == investigation_id).all()+ sketches = (+ db.query(Sketch).filter(Sketch.investigation_id == investigation_id).all()+ )if not sketches:- raise HTTPException(status_code=404, detail="No sketches found for this investigation")+ raise HTTPException(+ status_code=404, detail="No sketches found for this investigation"+ )return sketches+# Update a investigation by ID@router.put("/{investigation_id}", response_model=InvestigationRead)-def update_investigation(investigation_id: UUID, payload: InvestigationUpdate, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):- investigation = db.query(Investigation).filter(Investigation.id == investigation_id).first()+def update_investigation(+ investigation_id: UUID,+ payload: InvestigationUpdate,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):+ investigation = (+ db.query(Investigation).filter(Investigation.id == investigation_id).first()+ )if not investigation:raise HTTPException(status_code=404, detail="Investigation not found")-+investigation.name = payload.nameinvestigation.description = payload.descriptioninvestigation.status = payload.status@@ -71,18 +111,33 @@ def update_investigation(investigation_id: UUID, payload: InvestigationUpdate, ddb.refresh(investigation)return investigation+# Delete a investigation by ID@router.delete("/{investigation_id}", status_code=status.HTTP_204_NO_CONTENT)-def delete_investigation(investigation_id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):- investigation = db.query(Investigation).filter(Investigation.id == investigation_id, Investigation.owner_id == current_user.id).first()+def delete_investigation(+ investigation_id: UUID,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):+ investigation = (+ db.query(Investigation)+ .filter(+ Investigation.id == investigation_id,+ Investigation.owner_id == current_user.id,+ )+ .first()+ )if not investigation:raise HTTPException(status_code=404, detail="Investigation not found")-+# Get all sketches related to this investigation- sketches = db.query(Sketch).filter(Sketch.investigation_id == investigation_id).all()- analyses = db.query(Analysis).filter(Sketch.investigation_id == investigation_id).all()+ sketches = (+ db.query(Sketch).filter(Sketch.investigation_id == investigation_id).all()+ )+ analyses = (+ db.query(Analysis).filter(Sketch.investigation_id == investigation_id).all()+ )-# Delete all nodes and relationships for each sketch in Neo4jfor sketch in sketches:neo4j_query = """@@ -94,13 +149,13 @@ def delete_investigation(investigation_id: UUID, db: Session = Depends(get_db),except Exception as e:print(f"Neo4j cleanup error for sketch {sketch.id}: {e}")raise HTTPException(status_code=500, detail="Failed to clean up graph data")-+# Delete all sketches from PostgreSQLfor sketch in sketches:db.delete(sketch)for analysis in analyses:db.delete(analysis)-+# Finally delete the investigationdb.delete(investigation)db.commit()diff --git a/flowsint-api/app/api/routes/keys.py b/flowsint-api/app/api/routes/keys.pyindex 1ecf8d83..a57a2aba 100644--- a/flowsint-api/app/api/routes/keys.py+++ b/flowsint-api/app/api/routes/keys.py@@ -10,45 +10,62 @@ from datetime import datetimerouter = APIRouter()+def obfuscate_key(key: str) -> str:"""Obfuscate a key by showing only the last 4 characters, replacing others with asterisks."""if len(key) <= 4:return keyreturn "*" * (len(key) - 4) + key[-4:]+# Get the list of all keys for a user@router.get("", response_model=List[KeyRead])-def get_keys(db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):+def get_keys(+ db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)+):keys = db.query(Key).filter(Key.owner_id == current_user.id).all()- response_data = [KeyRead(- id=key.id,- owner_id=key.owner_id,- encrypted_key=obfuscate_key(key.encrypted_key),- name=key.name,- created_at=key.created_at- ) for key in keys]+ response_data = [+ KeyRead(+ id=key.id,+ owner_id=key.owner_id,+ encrypted_key=obfuscate_key(key.encrypted_key),+ name=key.name,+ created_at=key.created_at,+ )+ for key in keys+ ]return response_data+# Get a key by ID@router.get("/{id}", response_model=KeyRead)-def get_key_by_id(id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):+def get_key_by_id(+ id: UUID,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):key = db.query(Key).filter(Key.id == id, Key.owner_id == current_user.id).first()if not key:raise HTTPException(status_code=404, detail="Key not found")-+# Create a response with obfuscated keyresponse_data = KeyRead(id=key.id,owner_id=key.owner_id,encrypted_key=obfuscate_key(key.encrypted_key),name=key.name,- created_at=key.created_at+ created_at=key.created_at,)return response_data+# Create a new key@router.post("/create", response_model=KeyRead, status_code=status.HTTP_201_CREATED)-def create_key(payload: KeyCreate, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):+def create_key(+ payload: KeyCreate,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):new_key = Key(id=uuid4(),name=payload.name,@@ -61,12 +78,17 @@ def create_key(payload: KeyCreate, db: Session = Depends(get_db), current_user:db.refresh(new_key)return new_key+# Delete a key by ID@router.delete("/{id}", status_code=status.HTTP_204_NO_CONTENT)-def delete_key(id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):+def delete_key(+ id: UUID,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):key = db.query(Key).filter(Key.id == id, Key.owner_id == current_user.id).first()if not key:raise HTTPException(status_code=404, detail="Key not found")db.delete(key)db.commit()- return None\ No newline at end of file+ return Nonediff --git a/flowsint-api/app/api/routes/scan.py b/flowsint-api/app/api/routes/scan.pyindex 5436d62b..69577f97 100644--- a/flowsint-api/app/api/routes/scan.py+++ b/flowsint-api/app/api/routes/scan.py@@ -9,20 +9,32 @@ from app.api.schemas.scan import ScanReadrouter = APIRouter()+# Get the list of all scans[email protected]("", response_model=List[ScanRead],)-def get_scans(db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):+ "",+ response_model=List[ScanRead],+)+def get_scans(+ db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)+):scans = db.query(Scan).all()return scans+# Get a scan by ID@router.get("/{id}", response_model=ScanRead)-def get_scan_by_id(id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):+def get_scan_by_id(+ id: UUID,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):scan = db.query(Scan).filter(Scan.id == id).first()if not scan:raise HTTPException(status_code=404, detail="Transform not found")return scan+# Delete a scan by ID@router.delete("", status_code=status.HTTP_204_NO_CONTENT)def delete_scan(db: Session = Depends(get_db)):@@ -30,14 +42,21 @@ def delete_scan(db: Session = Depends(get_db)):db.commit()return None+# Delete a scan by ID@router.delete("/{id}", status_code=status.HTTP_204_NO_CONTENT)-def delete_scan(id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):- scan = db.query(Scan).filter(Scan.id == id, Scan.owner_id == current_user.id).first()+def delete_scan(+ id: UUID,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):+ scan = (+ db.query(Scan).filter(Scan.id == id, Scan.owner_id == current_user.id).first()+ )if not scan:raise HTTPException(status_code=404, detail="Scan not found")scan = db.query(Scan).filter(Scan.id == id).all()-+# Finally delete the scandb.delete(scan)db.commit()diff --git a/flowsint-api/app/api/routes/sketches.py b/flowsint-api/app/api/routes/sketches.pyindex 586cfdc1..71e7dd62 100644--- a/flowsint-api/app/api/routes/sketches.py+++ b/flowsint-api/app/api/routes/sketches.py@@ -16,14 +16,15 @@ from app.api.deps import get_current_userrouter = APIRouter()+@router.post("/create", response_model=SketchRead, status_code=status.HTTP_201_CREATED)def create_sketch(data: SketchCreate,db: Session = Depends(get_db),- current_user: Profile = Depends(get_current_user)+ current_user: Profile = Depends(get_current_user),):sketch_data = data.dict()- sketch_data['owner_id'] = current_user.id+ sketch_data["owner_id"] = current_user.idsketch = Sketch(**sketch_data)db.add(sketch)db.commit()@@ -32,18 +33,31 @@ def create_sketch(@router.get("", response_model=List[SketchRead])-def list_sketches(db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):+def list_sketches(+ db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)+):return db.query(Sketch).filter(Sketch.owner_id == current_user.id).all()+@router.get("/{sketch_id}")-def get_sketch_by_id(sketch_id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):+def get_sketch_by_id(+ sketch_id: UUID,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):sketch = db.query(Sketch).filter(Sketch.id == sketch_id).first()if not sketch:raise HTTPException(status_code=404, detail="Sketch not found")return sketch+@router.put("/{id}", response_model=SketchRead)-def update_sketch(id: UUID, data: SketchUpdate, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):+def update_sketch(+ id: UUID,+ data: SketchUpdate,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):sketch = db.query(Sketch).filter(Sketch.owner_id == current_user.id).get(id)if not sketch:raise HTTPException(status_code=404, detail="Sketch not found")@@ -53,12 +67,21 @@ def update_sketch(id: UUID, data: SketchUpdate, db: Session = Depends(get_db), cdb.refresh(sketch)return sketch+@router.delete("/{id}", status_code=204)-def delete_sketch(id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):- sketch = db.query(Sketch).filter(Sketch.id == id, Sketch.owner_id == current_user.id).first()+def delete_sketch(+ id: UUID,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):+ sketch = (+ db.query(Sketch)+ .filter(Sketch.id == id, Sketch.owner_id == current_user.id)+ .first()+ )if not sketch:raise HTTPException(status_code=404, detail="Sketch not found")-+# Delete all nodes and relationships in Neo4j firstneo4j_query = """MATCH (n {sketch_id: $sketch_id})@@ -69,16 +92,17 @@ def delete_sketch(id: UUID, db: Session = Depends(get_db), current_user: Profileexcept Exception as e:print(f"Neo4j cleanup error: {e}")raise HTTPException(status_code=500, detail="Failed to clean up graph data")-+# Then delete the sketch from PostgreSQLdb.delete(sketch)db.commit()+@router.get("/{id}/graph")async def get_sketch_nodes(- id: str,+ id: str,format: str = None,- db: Session = Depends(get_db),+ db: Session = Depends(get_db),# current_user: Profile = Depends(get_current_user)):"""@@ -94,12 +118,18 @@ async def get_sketch_nodes(rls: []Or if format=inline: List of inline relationship strings"""- sketch = db.query(Sketch).filter(Sketch.id == id,- # Sketch.owner_id == current_user.id- ).first()+ sketch = (+ db.query(Sketch)+ .filter(+ Sketch.id == id,+ # Sketch.owner_id == current_user.id+ )+ .first()+ )if not sketch:raise HTTPException(status_code=404, detail="Graph not found")import random+nodes_query = """MATCH (n)WHERE n.sketch_id = $sketch_id@@ -126,11 +156,11 @@ async def get_sketch_nodes("label": record["data"].get("label", "Node"),"type": "custom","caption": record["data"].get("label", "Node"),- "position":{+ "position": {"x": random.random() * 1000,"y": random.random() * 1000,},- "idx": idx+ "idx": idx,}for idx, record in enumerate(nodes_result)]@@ -150,10 +180,12 @@ async def get_sketch_nodes(if format == "inline":from flowsint_core.utils import get_inline_relationships+return get_inline_relationships(nodes, rels)return {"nds": nodes, "rls": rels}+class NodeData(BaseModel):label: str = Field(default="Node", description="Label/name of the node")color: str = Field(default="Node", description="Color of the node")@@ -163,17 +195,24 @@ class NodeData(BaseModel):class Config:extra = "allow" # Accept any additional fields+class NodeInput(BaseModel):type: str = Field(..., description="Type of the node")- data: NodeData = Field(default_factory=NodeData, description="Additional data for the node")+ data: NodeData = Field(+ default_factory=NodeData, description="Additional data for the node"+ )+def dict_to_cypher_props(props: dict, prefix: str = "") -> str:return ", ".join(f"{key}: ${prefix}{key}" for key in props)+@router.post("/{sketch_id}/nodes/add")-def add_node(sketch_id: str, node: NodeInput, current_user: Profile = Depends(get_current_user)):+def add_node(+ sketch_id: str, node: NodeInput, current_user: Profile = Depends(get_current_user)+):node_data = node.data.model_dump()-+node_type = node_data["type"]properties = {@@ -186,7 +225,6 @@ def add_node(sketch_id: str, node: NodeInput, current_user: Profile = Depends(geif node_data:flattened_data = flatten(node_data)properties.update(flattened_data)-cypher_props = dict_to_cypher_props(properties)@@ -202,15 +240,19 @@ def add_node(sketch_id: str, node: NodeInput, current_user: Profile = Depends(geraise HTTPException(status_code=500, detail=f"Database error: {str(e)}")if not create_result:- raise HTTPException(status_code=400, detail="Node creation failed - no result returned")+ raise HTTPException(+ status_code=400, detail="Node creation failed - no result returned"+ )try:new_node = create_result[0]["node"]new_node["id"] = create_result[0]["id"]except (IndexError, KeyError) as e:print(f"Error extracting node_id: {e}, result: {create_result}")- raise HTTPException(status_code=500, detail="Failed to extract node data from response")-+ raise HTTPException(+ status_code=500, detail="Failed to extract node data from response"+ )+new_node["data"] = node_datanew_node["data"]["id"] = new_node["id"]@@ -219,14 +261,20 @@ def add_node(sketch_id: str, node: NodeInput, current_user: Profile = Depends(ge"node": new_node,}+class RelationInput(BaseModel):source: Anytarget: Anytype: Literal["one-way", "two-way"]label: str = "RELATED_TO" # Optionnel : nom de la relation+@router.post("/{sketch_id}/relations/add")-def add_edge(sketch_id: str, relation: RelationInput, current_user: Profile = Depends(get_current_user)):+def add_edge(+ sketch_id: str,+ relation: RelationInput,+ current_user: Profile = Depends(get_current_user),+):query = f"""MATCH (a) WHERE elementId(a) = $from_id@@ -255,85 +303,87 @@ def add_edge(sketch_id: str, relation: RelationInput, current_user: Profile = De"edge": result[0]["r"],}+class NodeDeleteInput(BaseModel):nodeIds: List[str]+class NodeEditInput(BaseModel):nodeId: str- data: NodeData = Field(default_factory=NodeData, description="Updated data for the node")+ data: NodeData = Field(+ default_factory=NodeData, description="Updated data for the node"+ )+@router.put("/{sketch_id}/nodes/edit")def edit_node(sketch_id: str,node_edit: NodeEditInput,db: Session = Depends(get_db),- current_user: Profile = Depends(get_current_user)+ current_user: Profile = Depends(get_current_user),):# First verify the sketch exists and belongs to the usersketch = db.query(Sketch).filter(Sketch.id == sketch_id).first()if not sketch:raise HTTPException(status_code=404, detail="Sketch not found")-+node_data = node_edit.data.model_dump()node_type = node_data.get("type", "Node")-+# Prepare properties to updateproperties = {"type": node_type.lower(),"caption": node_data.get("label", "Node"),"label": node_data.get("label", "Node"),}-+# Add any additional data from the flattened node_dataif node_data:flattened_data = flatten(node_data)properties.update(flattened_data)-+# Build the SET clause for the Cypher queryset_clause = ", ".join(f"n.{key} = ${key}" for key in properties.keys())-+query = f"""MATCH (n)WHERE elementId(n) = $node_id AND n.sketch_id = $sketch_idSET {set_clause}RETURN n as node"""-- params = {- "node_id": node_edit.nodeId,- "sketch_id": sketch_id,- **properties- }-++ params = {"node_id": node_edit.nodeId, "sketch_id": sketch_id, **properties}+try:result = neo4j_connection.query(query, params)except Exception as e:print(f"Node update error: {e}")raise HTTPException(status_code=500, detail="Failed to update node")-+if not result:raise HTTPException(status_code=404, detail="Node not found or not accessible")-+updated_node = result[0]["node"]updated_node["data"] = node_data-+return {"status": "node updated","node": updated_node,}+@router.delete("/{sketch_id}/nodes")def delete_nodes(sketch_id: str,nodes: NodeDeleteInput,db: Session = Depends(get_db),- current_user: Profile = Depends(get_current_user)+ current_user: Profile = Depends(get_current_user),):# First verify the sketch exists and belongs to the usersketch = db.query(Sketch).filter(Sketch.id == sketch_id).first()if not sketch:raise HTTPException(status_code=404, detail="Sketch not found")-+# Delete nodes and their relationshipsquery = """UNWIND $node_ids AS node_id@@ -341,18 +391,18 @@ def delete_nodes(WHERE elementId(n) = node_id AND n.sketch_id = $sketch_idDETACH DELETE n"""-+try:- neo4j_connection.query(query, {- "node_ids": nodes.nodeIds,- "sketch_id": sketch_id- })+ neo4j_connection.query(+ query, {"node_ids": nodes.nodeIds, "sketch_id": sketch_id}+ )except Exception as e:print(f"Node deletion error: {e}")raise HTTPException(status_code=500, detail="Failed to delete nodes")-+return {"status": "nodes deleted", "count": len(nodes.nodeIds)}+@router.get("/{sketch_id}/nodes/{node_id}")def get_related_nodes(sketch_id: str,@@ -364,7 +414,7 @@ def get_related_nodes(# sketch = db.query(Sketch).filter(Sketch.id == sketch_id, Sketch.owner_id == current_user.id).first()# if not sketch:# raise HTTPException(status_code=404, detail="Sketch not found")-+# Query to get all direct relationships and connected nodes# First, let's get the center nodecenter_query = """@@ -372,19 +422,18 @@ def get_related_nodes(WHERE elementId(n) = $node_id AND n.sketch_id = $sketch_idRETURN elementId(n) as id, labels(n) as labels, properties(n) as data"""-+try:- center_result = neo4j_connection.query(center_query, {- "sketch_id": sketch_id,- "node_id": node_id- })+ center_result = neo4j_connection.query(+ center_query, {"sketch_id": sketch_id, "node_id": node_id}+ )except Exception as e:print(f"Center node query error: {e}")raise HTTPException(status_code=500, detail="Failed to retrieve center node")-+if not center_result:raise HTTPException(status_code=404, detail="Node not found")-+# Now get all relationships and connected nodesrelationships_query = """MATCH (n)@@ -415,16 +464,15 @@ def get_related_nodes(properties(other) as other_node_data,'incoming' as direction"""-+try:- result = neo4j_connection.query(relationships_query, {- "sketch_id": sketch_id,- "node_id": node_id- })+ result = neo4j_connection.query(+ relationships_query, {"sketch_id": sketch_id, "node_id": node_id}+ )except Exception as e:print(f"Related nodes query error: {e}")raise HTTPException(status_code=500, detail="Failed to retrieve related nodes")-+# Extract center node infocenter_record = center_result[0]center_node = {@@ -433,59 +481,61 @@ def get_related_nodes("data": center_record["data"],"label": center_record["data"].get("label", "Node"),"type": "custom",- "caption": center_record["data"].get("label", "Node")+ "caption": center_record["data"].get("label", "Node"),}-+# Collect all related nodes and relationshipsrelated_nodes = []relationships = []seen_nodes = set()seen_relationships = set()-+for record in result:# Skip if no relationship foundif not record["rel_id"]:continue-+# Add relationship if not seenif record["rel_id"] not in seen_relationships:if record["direction"] == "outgoing":- relationships.append({- "id": record["rel_id"],- "type": "straight",- "source": center_node["id"],- "target": record["other_node_id"],- "data": record["rel_data"],- "caption": record["rel_type"]- })+ relationships.append(+ {+ "id": record["rel_id"],+ "type": "straight",+ "source": center_node["id"],+ "target": record["other_node_id"],+ "data": record["rel_data"],+ "caption": record["rel_type"],+ }+ )else: # incoming- relationships.append({- "id": record["rel_id"],- "type": "straight",- "source": record["other_node_id"],- "target": center_node["id"],- "data": record["rel_data"],- "caption": record["rel_type"]- })+ relationships.append(+ {+ "id": record["rel_id"],+ "type": "straight",+ "source": record["other_node_id"],+ "target": center_node["id"],+ "data": record["rel_data"],+ "caption": record["rel_type"],+ }+ )seen_relationships.add(record["rel_id"])-+# Add related node if not seenif record["other_node_id"] and record["other_node_id"] not in seen_nodes:- related_nodes.append({- "id": record["other_node_id"],- "labels": record["other_node_labels"],- "data": record["other_node_data"],- "label": record["other_node_data"].get("label", "Node"),- "type": "custom",- "caption": record["other_node_data"].get("label", "Node")- })+ related_nodes.append(+ {+ "id": record["other_node_id"],+ "labels": record["other_node_labels"],+ "data": record["other_node_data"],+ "label": record["other_node_data"].get("label", "Node"),+ "type": "custom",+ "caption": record["other_node_data"].get("label", "Node"),+ }+ )seen_nodes.add(record["other_node_id"])-+# Combine center node with related nodesall_nodes = [center_node] + related_nodes-- return {- "nds": all_nodes,- "rls": relationships- }-++ return {"nds": all_nodes, "rls": relationships}diff --git a/flowsint-api/app/api/routes/transforms.py b/flowsint-api/app/api/routes/transforms.pyindex dd3fa27a..475e3fa3 100644--- a/flowsint-api/app/api/routes/transforms.py+++ b/flowsint-api/app/api/routes/transforms.py@@ -13,46 +13,62 @@ from flowsint_core.core.postgre_db import get_dbfrom app.models.models import Transform, Profilefrom app.api.deps import get_current_userfrom app.api.schemas.transform import TransformRead, TransformCreate, TransformUpdate-from flowsint_types import ASN, CIDR, CryptoWallet, CryptoWalletTransaction, CryptoNFT, Website, Individual+from flowsint_types import (+ ASN,+ CIDR,+ CryptoWallet,+ CryptoWalletTransaction,+ CryptoNFT,+ Website,+ Individual,+)+class FlowComputationRequest(BaseModel):nodes: List[Node]edges: List[Edge]inputType: Optional[str] = None+class FlowComputationResponse(BaseModel):transformBranches: List[FlowBranch]initialData: Any-++class StepSimulationRequest(BaseModel):transformBranches: List[FlowBranch]currentStepIndex: int+class LaunchTransformPayload(BaseModel):values: List[str]sketch_id: str+router = APIRouter()+# Get the list of all transforms@router.get("", response_model=List[TransformRead])def get_transforms(category: Optional[str] = Query(None),db: Session = Depends(get_db),- current_user: Profile = Depends(get_current_user)+ current_user: Profile = Depends(get_current_user),):query = db.query(Transform)-+if category is not None and category != "undefined":# Case-insensitive filtering by checking if any category matches (case-insensitive)transforms = query.all()return [- transform for transform in transforms+ transform+ for transform in transformsif any(cat.lower() == category.lower() for cat in transform.category)]-+return query.order_by(Transform.last_updated_at.desc()).all()+# Returns the "raw_materials" for the transform editor@router.get("/raw_materials")async def get_material_list():@@ -72,7 +88,7 @@ async def get_material_list():"params": scanner.get("params"),"params_schema": scanner.get("params_schema"),"required_params": scanner.get("required_params"),- "icon": scanner.get("icon")+ "icon": scanner.get("icon"),}for scanner in scanner_list]@@ -92,27 +108,33 @@ async def get_material_list():extract_input_schema_transform(Email),extract_input_schema_transform(CryptoWallet),extract_input_schema_transform(CryptoWalletTransaction),- extract_input_schema_transform(CryptoNFT)+ extract_input_schema_transform(CryptoNFT),]-+# Put types first, then add all scanner categoriesflattened_scanners = {"types": object_inputs}flattened_scanners.update(scanner_categories)return {"items": flattened_scanners}+# Returns the "raw_materials" for the transform editor@router.get("/input_type/{input_type}")-async def get_material_list(input_type:str):+async def get_material_list(input_type: str):scanners = ScannerRegistry.list_by_input_type(input_type)return {"items": scanners}-# Create a new transform[email protected]("/create", response_model=TransformRead, status_code=status.HTTP_201_CREATED)-def create_transform(payload: TransformCreate, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):-+ "/create", response_model=TransformRead, status_code=status.HTTP_201_CREATED+)+def create_transform(+ payload: TransformCreate,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):+new_transform = Transform(id=uuid4(),name=payload.name,@@ -127,28 +149,39 @@ def create_transform(payload: TransformCreate, db: Session = Depends(get_db), cudb.refresh(new_transform)return new_transform+# Get a transform by ID@router.get("/{transform_id}", response_model=TransformRead)-def get_transform_by_id(transform_id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):+def get_transform_by_id(+ transform_id: UUID,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):transform = db.query(Transform).filter(Transform.id == transform_id).first()if not transform:raise HTTPException(status_code=404, detail="Transform not found")return transform+# Update a transform by ID@router.put("/{transform_id}", response_model=TransformRead)-def update_transform(transform_id: UUID, payload: TransformUpdate, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):+def update_transform(+ transform_id: UUID,+ payload: TransformUpdate,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):transform = db.query(Transform).filter(Transform.id == transform_id).first()if not transform:raise HTTPException(status_code=404, detail="Transform not found")update_data = payload.dict(exclude_unset=True)for key, value in update_data.items():- print(f'only update {key}')+ print(f"only update {key}")if key == "category":if "SocialProfile" in value:value.append("Username")setattr(transform, key, value)-+transform.last_updated_at = datetime.utcnow()db.commit()@@ -158,7 +191,11 @@ def update_transform(transform_id: UUID, payload: TransformUpdate, db: Session =# Delete a transform by ID@router.delete("/{transform_id}", status_code=status.HTTP_204_NO_CONTENT)-def delete_transform(transform_id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):+def delete_transform(+ transform_id: UUID,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):transform = db.query(Transform).filter(Transform.id == transform_id).first()if not transform:raise HTTPException(status_code=404, detail="Transform not found")@@ -172,7 +209,7 @@ async def launch_transform(transform_id: str,payload: LaunchTransformPayload,db: Session = Depends(get_db),- current_user: Profile = Depends(get_current_user)+ current_user: Profile = Depends(get_current_user),):try:transform = db.query(Transform).filter(Transform.id == transform_id).first()@@ -180,32 +217,37 @@ async def launch_transform(raise HTTPException(status_code=404, detail="Transform not found")nodes = [Node(**node) for node in transform.transform_schema["nodes"]]edges = [Edge(**edge) for edge in transform.transform_schema["edges"]]- transform_branches = compute_transform_branches(- payload.values,- nodes,- edges- )+ transform_branches = compute_transform_branches(payload.values, nodes, edges)serializable_branches = [branch.model_dump() for branch in transform_branches]- task = celery.send_task("run_transform", args=[serializable_branches, payload.values, payload.sketch_id, str(current_user.id)])+ task = celery.send_task(+ "run_transform",+ args=[+ serializable_branches,+ payload.values,+ payload.sketch_id,+ str(current_user.id),+ ],+ )return {"id": task.id}except Exception as e:print(e)raise HTTPException(status_code=404, detail="Transform not found")+@router.post("/{transform_id}/compute", response_model=FlowComputationResponse)-def compute_transforms(request: FlowComputationRequest, current_user: Profile = Depends(get_current_user)):+def compute_transforms(+ request: FlowComputationRequest, current_user: Profile = Depends(get_current_user)+):initial_data = generate_sample_data(request.inputType or "string")transform_branches = compute_transform_branches(- initial_data,- request.nodes,- request.edges- )+ initial_data, request.nodes, request.edges+ )return FlowComputationResponse(- transformBranches=transform_branches,- initialData=initial_data+ transformBranches=transform_branches, initialData=initial_data)-++def generate_sample_data(type_str: str) -> Any:type_str = type_str.lower() if type_str else "string"if type_str == "string":@@ -229,7 +271,10 @@ def generate_sample_data(type_str: str) -> Any:else:return f"sample_{type_str}"-def compute_transform_branches(initial_value: Any, nodes: List[Node], edges: List[Edge]) -> List[FlowBranch]:++def compute_transform_branches(+ initial_value: Any, nodes: List[Node], edges: List[Edge]+) -> List[FlowBranch]:"""Computes flow branches based on nodes and edges with proper DFS traversal"""# Find input nodes (starting points)input_nodes = [node for node in nodes if node.data.get("type") == "type"]@@ -263,33 +308,38 @@ def compute_transform_branches(initial_value: Any, nodes: List[Node], edges: Lis"""Calculate the shortest possible path length from a node to any leaf"""if visited is None:visited = set()-+if start_node in visited:- return float('inf')-+ return float("inf")+visited.add(start_node)out_edges = [edge for edge in edges if edge.source == start_node]-+if not out_edges:return 1-- min_length = float('inf')++ min_length = float("inf")for edge in out_edges:length = calculate_path_length(edge.target, visited.copy())min_length = min(min_length, length)-+return 1 + min_lengthdef get_outgoing_edges(node_id: str) -> List[Edge]:"""Get outgoing edges sorted by the shortest possible path length"""out_edges = [edge for edge in edges if edge.source == node_id]# Sort edges by the length of the shortest possible path from their target- return sorted(- out_edges,- key=lambda e: calculate_path_length(e.target)- )-- def create_step(node_id: str, branch_id: str, depth: int, input_data: Dict[str, Any], is_input_node: bool, outputs: Dict[str, Any], node_params: Optional[Dict[str, Any]] = None) -> FlowStep:+ return sorted(out_edges, key=lambda e: calculate_path_length(e.target))++ def create_step(+ node_id: str,+ branch_id: str,+ depth: int,+ input_data: Dict[str, Any],+ is_input_node: bool,+ outputs: Dict[str, Any],+ node_params: Optional[Dict[str, Any]] = None,+ ) -> FlowStep:return FlowStep(nodeId=node_id,params=node_params,@@ -301,7 +351,17 @@ def compute_transform_branches(initial_value: Any, nodes: List[Node], edges: Lisdepth=depth,)- def explore_branch(current_node_id: str, branch_id: str, branch_name: str, depth: int, input_data: Dict[str, Any], path: List[str], branch_visited: set, steps: List[FlowStep], parent_outputs: Dict[str, Any] = None) -> None:+ def explore_branch(+ current_node_id: str,+ branch_id: str,+ branch_name: str,+ depth: int,+ input_data: Dict[str, Any],+ path: List[str],+ branch_visited: set,+ steps: List[FlowStep],+ parent_outputs: Dict[str, Any] = None,+ ) -> None:nonlocal branch_counter# Skip if node is already in current path (cycle detection)@@ -316,7 +376,9 @@ def compute_transform_branches(initial_value: Any, nodes: List[Node], edges: Lisis_input_node = current_node.data.get("type") == "type"if is_input_node:outputs_array = current_node.data["outputs"].get("properties", [])- first_output_name = outputs_array[0].get("name", "output") if outputs_array else "output"+ first_output_name = (+ outputs_array[0].get("name", "output") if outputs_array else "output"+ )current_outputs = {first_output_name: initial_value}else:# Check if we already have outputs for this scanner@@ -331,14 +393,22 @@ def compute_transform_branches(initial_value: Any, nodes: List[Node], edges: Lisnode_params = current_node.data.get("params", {})# Create and add current step- current_step = create_step(current_node_id, branch_id, depth, input_data, is_input_node, current_outputs, node_params)+ current_step = create_step(+ current_node_id,+ branch_id,+ depth,+ input_data,+ is_input_node,+ current_outputs,+ node_params,+ )steps.append(current_step)path.append(current_node_id)branch_visited.add(current_node_id)# Get all outgoing edges sorted by path lengthout_edges = get_outgoing_edges(current_node_id)-+if not out_edges:# Leaf node reached, save the branchbranches.append(FlowBranch(id=branch_id, name=branch_name, steps=steps[:]))@@ -347,15 +417,17 @@ def compute_transform_branches(initial_value: Any, nodes: List[Node], edges: Lisfor i, edge in enumerate(out_edges):if edge.target in path: # Skip if would create cyclecontinue-+# Prepare next node's inputoutput_key = edge.sourceHandleif not output_key and current_outputs:output_key = list(current_outputs.keys())[0]-+output_value = current_outputs.get(output_key) if output_key else Noneif output_value is None and parent_outputs:- output_value = parent_outputs.get(output_key) if output_key else None+ output_value = (+ parent_outputs.get(output_key) if output_key else None+ )next_input = {edge.targetHandle or "input": output_value}@@ -370,15 +442,17 @@ def compute_transform_branches(initial_value: Any, nodes: List[Node], edges: Lispath,branch_visited,steps,- current_outputs+ current_outputs,)else:# Create new branch starting from current nodebranch_counter += 1new_branch_id = f"{branch_id}-{branch_counter}"new_branch_name = f"{branch_name} (Branch {branch_counter})"- new_steps = steps[:len(steps)] # Copy steps up to current node- new_branch_visited = branch_visited.copy() # Create new visited set for the branch+ new_steps = steps[: len(steps)] # Copy steps up to current node+ new_branch_visited = (+ branch_visited.copy()+ ) # Create new visited set for the branchexplore_branch(edge.target,new_branch_id,@@ -388,7 +462,7 @@ def compute_transform_branches(initial_value: Any, nodes: List[Node], edges: Lispath[:], # Create new path copy for branchnew_branch_visited,new_steps,- current_outputs+ current_outputs,)# Backtrack: remove current node from path and remove its step@@ -408,70 +482,73 @@ def compute_transform_branches(initial_value: Any, nodes: List[Node], edges: Lis[], # Use list for path to maintain orderset(), # Use set for visited to check membership[],- None+ None,)# Sort branches by length (number of steps)branches.sort(key=lambda branch: len(branch.steps))return branches+def process_node_data(node: Node, inputs: Dict[str, Any]) -> Dict[str, Any]:"""Traite les données de nœud en fonction du type de nœud et des entrées"""outputs = {}output_types = node.data["outputs"].get("properties", [])-+for output in output_types:output_name = output.get("name", "output")class_name = node.data.get("class_name", "")# For simulation purposes, we'll return a placeholder value based on the scanner typeif class_name in ["ReverseResolveScanner", "ResolveScanner"]:# IP/Domain resolution scanners- outputs[output_name] = "192.168.1.1" if "ip" in output_name.lower() else "example.com"+ outputs[output_name] = (+ "192.168.1.1" if "ip" in output_name.lower() else "example.com"+ )elif class_name == "SubdomainScanner":# Subdomain scanneroutputs[output_name] = f"sub.{inputs.get('input', 'example.com')}"-+elif class_name == "WhoisScanner":# WHOIS scanneroutputs[output_name] = {"domain": inputs.get("input", "example.com"),"registrar": "Example Registrar",- "creation_date": "2020-01-01"+ "creation_date": "2020-01-01",}-+elif class_name == "GeolocationScanner":# Geolocation scanneroutputs[output_name] = {"country": "France","city": "Paris",- "coordinates": {"lat": 48.8566, "lon": 2.3522}+ "coordinates": {"lat": 48.8566, "lon": 2.3522},}-+elif class_name == "MaigretScanner":# Social media scanneroutputs[output_name] = {"username": inputs.get("input", "user123"),- "platforms": ["twitter", "github", "linkedin"]+ "platforms": ["twitter", "github", "linkedin"],}-+elif class_name == "HoleheScanner":# Email verification scanneroutputs[output_name] = {"email": inputs.get("input", "[email protected]"),"exists": True,- "platforms": ["gmail", "github"]+ "platforms": ["gmail", "github"],}-+elif class_name == "SireneScanner":# Organization scanneroutputs[output_name] = {"name": inputs.get("input", "Example Corp"),"siret": "12345678901234",- "address": "1 Example Street"+ "address": "1 Example Street",}-+else:# For unknown scanners, pass through the inputoutputs[output_name] = inputs.get("input") or f"transformed_{output_name}"- return outputs\ No newline at end of file+ return outputsdiff --git a/flowsint-api/app/api/routes/types.py b/flowsint-api/app/api/routes/types.pyindex 7bbd4786..e98e5f02 100644--- a/flowsint-api/app/api/routes/types.py+++ b/flowsint-api/app/api/routes/types.py@@ -3,18 +3,44 @@ from uuid import UUID, uuid4from fastapi import APIRouterfrom pydantic import BaseModel, TypeAdapterfrom flowsint_types import (- Domain, Ip, SocialProfile, Organization, Email, ASN, CIDR,- CryptoWallet, CryptoWalletTransaction, CryptoNFT, Website, Individual,- Phone, Leak, Username, Credential, Session,- DNSRecord, SSLCertificate, Device, Document, File, Message,- Malware, Weapon, BankAccount, CreditCard, WebTracker, Phrase+ Domain,+ Ip,+ SocialProfile,+ Organization,+ Email,+ ASN,+ CIDR,+ CryptoWallet,+ CryptoWalletTransaction,+ CryptoNFT,+ Website,+ Individual,+ Phone,+ Leak,+ Username,+ Credential,+ Session,+ DNSRecord,+ SSLCertificate,+ Device,+ Document,+ File,+ Message,+ Malware,+ Weapon,+ BankAccount,+ CreditCard,+ WebTracker,+ Phrase,)+# from flowsint_types.script import Script# from flowsint_types.reputation_score import ReputationScore# from flowsint_types.risk_profile import RiskProfilerouter = APIRouter()+# Returns the "types" for the sketches@router.get("/")async def get_types_list():@@ -39,9 +65,13 @@ async def get_types_list():"fields": [],"children": [extract_input_schema(Individual, label_key="full_name"),- extract_input_schema(SocialProfile, label_key="username", icon="socialprofile"),+ extract_input_schema(+ SocialProfile, label_key="username", icon="socialprofile"+ ),extract_input_schema(Organization, label_key="name"),- extract_input_schema(Username, label_key="username", icon="socialprofile"),+ extract_input_schema(+ Username, label_key="username", icon="socialprofile"+ ),# extract_input_schema(Alias, label_key="alias", icon="alias"),# extract_input_schema(Affiliation, label_key="organization", icon="affiliation"),],@@ -67,7 +97,9 @@ async def get_types_list():"children": [extract_input_schema(Phone, label_key="number"),extract_input_schema(Email, label_key="email"),- extract_input_schema(SocialProfile, label_key="username", icon="socialprofile"),+ extract_input_schema(+ SocialProfile, label_key="username", icon="socialprofile"+ ),extract_input_schema(Message, label_key="content", icon="message"),],},@@ -97,7 +129,9 @@ async def get_types_list():"label": "Security & Access","fields": [],"children": [- extract_input_schema(Credential, label_key="username", icon="credential"),+ extract_input_schema(+ Credential, label_key="username", icon="credential"+ ),extract_input_schema(Session, label_key="session_id", icon="session"),extract_input_schema(Device, label_key="device_id", icon="device"),extract_input_schema(Malware, label_key="name", icon="malware"),@@ -127,8 +161,12 @@ async def get_types_list():"label": "Financial Data","fields": [],"children": [- extract_input_schema(BankAccount, label_key="account_number", icon="creditcard"),- extract_input_schema(CreditCard, label_key="card_number", icon="creditcard"),+ extract_input_schema(+ BankAccount, label_key="account_number", icon="creditcard"+ ),+ extract_input_schema(+ CreditCard, label_key="card_number", icon="creditcard"+ ),],},{@@ -150,41 +188,47 @@ async def get_types_list():"label": "Crypto","fields": [],"children": [- extract_input_schema(CryptoWallet, label_key="address", icon="cryptowallet"),- extract_input_schema(CryptoWalletTransaction, label_key="hash", icon="cryptowallet"),+ extract_input_schema(+ CryptoWallet, label_key="address", icon="cryptowallet"+ ),+ extract_input_schema(+ CryptoWalletTransaction, label_key="hash", icon="cryptowallet"+ ),extract_input_schema(CryptoNFT, label_key="name", icon="cryptowallet"),],- }+ },]return types+def extract_input_schema(+ model: Type[BaseModel], label_key: str, icon: Optional[str] = None+) -> Dict[str, Any]:-def extract_input_schema(model: Type[BaseModel], label_key:str, icon: Optional[str]=None) -> Dict[str, Any]:-adapter = TypeAdapter(model)schema = adapter.json_schema()# Use the main schema properties, not the $defstype_name = model.__name__details = schemareturn {- "id": uuid4(),- "type": type_name,- "key": type_name.lower(),- "label_key": label_key,- "icon": icon or type_name.lower(),- "label": type_name,- "description": details.get("description",""),- "fields": [resolve_field(prop, details=info, schema=schema)- for prop, info in details.get("properties", {}).items()- ]+ "id": uuid4(),+ "type": type_name,+ "key": type_name.lower(),+ "label_key": label_key,+ "icon": icon or type_name.lower(),+ "label": type_name,+ "description": details.get("description", ""),+ "fields": [+ resolve_field(prop, details=info, schema=schema)+ for prop, info in details.get("properties", {}).items()+ ],}-def resolve_field(prop:str, details: dict, schema: dict = None) -> Dict:+def resolve_field(prop: str, details: dict, schema: dict = None) -> Dict:"""_summary_- The fields can sometimes contain nested complex objects, like:+ The fields can sometimes contain nested complex objects, like:- Organization having Individual[] as dirigeants, so we want to skip those.Args:details (dict): _description_@@ -193,28 +237,35 @@ def resolve_field(prop:str, details: dict, schema: dict = None) -> Dict:Returns:str: _description_"""- field = { "name": prop,"label": details["title"], "description": details["description"], "type":"text"}+ field = {+ "name": prop,+ "label": details["title"],+ "description": details["description"],+ "type": "text",+ }if has_enum(details):- field["type"] = "select"- field["options"]= [- { "label": label, "value": label } for label in get_enum_values(details)+ field["type"] = "select"+ field["options"] = [+ {"label": label, "value": label} for label in get_enum_values(details)]field["required"] = is_required(details)-+return fielddef has_enum(schema: dict) -> bool:- any_of = schema.get('anyOf', [])- return any(isinstance(entry, dict) and 'enum' in entry for entry in any_of)+ any_of = schema.get("anyOf", [])+ return any(isinstance(entry, dict) and "enum" in entry for entry in any_of)+def is_required(schema: dict) -> bool:- any_of = schema.get('anyOf', [])- return not any(entry == {'type': 'null'} for entry in any_of)-+ any_of = schema.get("anyOf", [])+ return not any(entry == {"type": "null"} for entry in any_of)++def get_enum_values(schema: dict) -> list:enum_values = []- for entry in schema.get('anyOf', []):- if isinstance(entry, dict) and 'enum' in entry:- enum_values.extend(entry['enum'])- return enum_values\ No newline at end of file+ for entry in schema.get("anyOf", []):+ if isinstance(entry, dict) and "enum" in entry:+ enum_values.extend(entry["enum"])+ return enum_valuesdiff --git a/flowsint-api/app/api/schemas/analysis.py b/flowsint-api/app/api/schemas/analysis.pyindex 0dbd8933..0ff2f57e 100644--- a/flowsint-api/app/api/schemas/analysis.py+++ b/flowsint-api/app/api/schemas/analysis.py@@ -3,6 +3,7 @@ from pydantic import UUID4, BaseModelfrom typing import Optional, Anyfrom datetime import datetime+class AnalysisCreate(BaseModel):title: strdescription: Optional[str] = None@@ -10,6 +11,7 @@ class AnalysisCreate(BaseModel):owner_id: Optional[UUID4] = Noneinvestigation_id: Optional[UUID4] = None+class AnalysisRead(ORMBase):id: UUID4title: str@@ -20,10 +22,11 @@ class AnalysisRead(ORMBase):owner_id: Optional[UUID4]investigation_id: Optional[UUID4]+class AnalysisUpdate(BaseModel):title: Optional[str] = Nonedescription: Optional[str] = Nonecontent: Optional[Any] = Nonelast_updated_at: Optional[datetime] = Noneowner_id: Optional[UUID4] = None- investigation_id: Optional[UUID4] = None\ No newline at end of file+ investigation_id: Optional[UUID4] = Nonediff --git a/flowsint-api/app/api/schemas/base.py b/flowsint-api/app/api/schemas/base.pyindex 972379c3..1823eb11 100644--- a/flowsint-api/app/api/schemas/base.py+++ b/flowsint-api/app/api/schemas/base.py@@ -1,5 +1,6 @@from pydantic import BaseModel+class ORMBase(BaseModel):class Config:from_attributes = Truediff --git a/flowsint-api/app/api/schemas/chat.py b/flowsint-api/app/api/schemas/chat.pyindex c99a33b2..62f49468 100644--- a/flowsint-api/app/api/schemas/chat.py+++ b/flowsint-api/app/api/schemas/chat.py@@ -3,6 +3,7 @@ from pydantic import UUID4, BaseModelfrom typing import List, Optional, Anyfrom datetime import datetime+class ChatMessageRead(BaseModel):id: UUID4content: Optional[Any] = None@@ -14,12 +15,14 @@ class ChatMessageRead(BaseModel):class Config:from_attributes = True+class ChatCreate(BaseModel):title: strdescription: Optional[str] = Noneowner_id: Optional[UUID4] = Noneinvestigation_id: Optional[UUID4] = None+class ChatRead(ORMBase):id: UUID4title: str@@ -29,4 +32,3 @@ class ChatRead(ORMBase):owner_id: Optional[UUID4]investigation_id: Optional[UUID4]messages: List[ChatMessageRead]-\ No newline at end of filediff --git a/flowsint-api/app/api/schemas/feedback.py b/flowsint-api/app/api/schemas/feedback.pyindex 32b783ab..36596a10 100644--- a/flowsint-api/app/api/schemas/feedback.py+++ b/flowsint-api/app/api/schemas/feedback.py@@ -3,10 +3,12 @@ from pydantic import UUID4, BaseModelfrom typing import Optionalfrom datetime import datetime+class FeedbackCreate(BaseModel):content: Optional[str] = Noneowner_id: Optional[UUID4] = None+class FeedbackRead(ORMBase):id: intcreated_at: datetimediff --git a/flowsint-api/app/api/schemas/investigation.py b/flowsint-api/app/api/schemas/investigation.pyindex 7680e45e..cccb1c06 100644--- a/flowsint-api/app/api/schemas/investigation.py+++ b/flowsint-api/app/api/schemas/investigation.py@@ -3,12 +3,15 @@ from pydantic import UUID4, BaseModelfrom typing import Optionalfrom datetime import datetimefrom .sketch import SketchRead++class InvestigationCreate(BaseModel):name: strdescription: strowner_id: Optional[UUID4] = Nonestatus: Optional[str] = "active"+class InvestigationRead(ORMBase):id: UUID4created_at: datetime@@ -19,11 +22,13 @@ class InvestigationRead(ORMBase):status: strsketches: list[SketchRead] = []+class InvestigationProfileCreate(BaseModel):investigation_id: UUID4profile_id: UUID4role: Optional[str] = "member"+class InvestigationProfileRead(ORMBase):id: intcreated_at: datetime@@ -31,6 +36,7 @@ class InvestigationProfileRead(ORMBase):profile_id: UUID4role: str+class InvestigationUpdate(BaseModel):name: strdescription: Optional[str] = Nonediff --git a/flowsint-api/app/api/schemas/investigation_profiles.py b/flowsint-api/app/api/schemas/investigation_profiles.pyindex a9e15346..fd56ca7e 100644--- a/flowsint-api/app/api/schemas/investigation_profiles.py+++ b/flowsint-api/app/api/schemas/investigation_profiles.py@@ -3,11 +3,13 @@ from pydantic import UUID4, BaseModelfrom typing import Optionalfrom datetime import datetime+class InvestigationProfileCreate(BaseModel):investigation_id: UUID4profile_id: UUID4role: Optional[str] = "member"+class InvestigationProfileRead(ORMBase):id: intcreated_at: datetimediff --git a/flowsint-api/app/api/schemas/key.py b/flowsint-api/app/api/schemas/key.pyindex acaf1f09..ed07be78 100644--- a/flowsint-api/app/api/schemas/key.py+++ b/flowsint-api/app/api/schemas/key.py@@ -2,10 +2,12 @@ from .base import ORMBasefrom pydantic import UUID4, BaseModelfrom datetime import datetime+class KeyCreate(BaseModel):key: strname: str-++class KeyRead(ORMBase):id: UUID4owner_id: UUID4diff --git a/flowsint-api/app/api/schemas/profile.py b/flowsint-api/app/api/schemas/profile.pyindex fbed722d..62c473fa 100644--- a/flowsint-api/app/api/schemas/profile.py+++ b/flowsint-api/app/api/schemas/profile.py@@ -2,13 +2,14 @@ from .base import ORMBasefrom pydantic import UUID4, BaseModel, EmailStrfrom typing import Optional+class ProfileCreate(BaseModel):email: EmailStrpassword: str+class ProfileRead(ORMBase):id: UUID4first_name: Optional[str]last_name: Optional[str]avatar_url: Optional[str]-diff --git a/flowsint-api/app/api/schemas/scan.py b/flowsint-api/app/api/schemas/scan.pyindex a1854fe4..b39d4aa1 100644--- a/flowsint-api/app/api/schemas/scan.py+++ b/flowsint-api/app/api/schemas/scan.py@@ -3,12 +3,14 @@ from .base import ORMBasefrom pydantic import UUID4, BaseModelfrom typing import Optional+class ScanCreate(BaseModel):values: Optional[List[str]] = Nonesketch_id: Optional[UUID4] = Nonestatus: Optional[str] = Noneresults: Optional[Any] = None+class ScanRead(ORMBase):id: UUID4sketch_id: Optional[UUID4]diff --git a/flowsint-api/app/api/schemas/sketch.py b/flowsint-api/app/api/schemas/sketch.pyindex 4fda7bb4..aeba81fd 100644--- a/flowsint-api/app/api/schemas/sketch.py+++ b/flowsint-api/app/api/schemas/sketch.py@@ -4,6 +4,7 @@ from typing import Optionalfrom datetime import datetimefrom uuid import UUID+class SketchCreate(BaseModel):title: strdescription: str@@ -11,6 +12,7 @@ class SketchCreate(BaseModel):investigation_id: UUID4status: Optional[str] = "active"+class SketchRead(ORMBase):id: UUID4title: str@@ -21,18 +23,21 @@ class SketchRead(ORMBase):last_updated_at: datetimestatus: str+class SketchProfileCreate(BaseModel):sketch_id: UUID4profile_id: UUID4role: Optional[str] = "editor"+class SketchProfileRead(ORMBase):id: intcreated_at: datetimesketch_id: UUID4profile_id: UUID4role: str-++class SketchUpdate(BaseModel):title: Optional[str] = Nonedescription: Optional[str] = None@@ -40,11 +45,10 @@ class SketchUpdate(BaseModel):status: Optional[str] = Noneinvestigation_id: Optional[UUID] = None-+class SketchIn(BaseModel):title: Optional[str] = Nonedescription: Optional[str] = Noneowner_id: Optional[UUID] = Nonestatus: Optional[str] = "active"investigation_id: UUID # requis-diff --git a/flowsint-api/app/api/schemas/transform.py b/flowsint-api/app/api/schemas/transform.pyindex 1f65384a..bc282506 100644--- a/flowsint-api/app/api/schemas/transform.py+++ b/flowsint-api/app/api/schemas/transform.py@@ -4,12 +4,14 @@ from typing import Optionalfrom datetime import datetimefrom typing import List, Optional, Dict, Any+class TransformCreate(BaseModel):name: strdescription: Optional[str] = Nonecategory: Optional[List[str]] = Nonetransform_schema: Optional[Dict[str, Any]] = None+class TransformRead(ORMBase):id: UUID4name: str@@ -19,6 +21,7 @@ class TransformRead(ORMBase):created_at: datetimelast_updated_at: datetime+class TransformUpdate(BaseModel):name: Optional[str] = Nonedescription: Optional[str] = Nonediff --git a/flowsint-api/app/main.py b/flowsint-api/app/main.pyindex 9c31c15d..8a91b89d 100644--- a/flowsint-api/app/main.py+++ b/flowsint-api/app/main.py@@ -4,6 +4,7 @@ import osfrom typing import Listfrom dotenv import load_dotenvfrom fastapi.middleware.cors import CORSMiddleware+# Routes to be includedfrom app.api.routes import authfrom app.api.routes import investigations@@ -53,11 +54,13 @@ app.add_middleware(app.include_router(auth.router, prefix="/api/auth", tags=["auth"])app.include_router(sketches.router, prefix="/api/sketches", tags=["sketches"])-app.include_router(investigations.router, prefix="/api/investigations", tags=["investigations"])+app.include_router(+ investigations.router, prefix="/api/investigations", tags=["investigations"]+)app.include_router(transforms.router, prefix="/api/transforms", tags=["transforms"])app.include_router(events.router, prefix="/api/events", tags=["events"])app.include_router(analysis.router, prefix="/api/analyses", tags=["analyses"])app.include_router(chat.router, prefix="/api/chats", tags=["chats"])app.include_router(scan.router, prefix="/api/scans", tags=["scans"])app.include_router(keys.router, prefix="/api/keys", tags=["keys"])-app.include_router(types.router, prefix="/api/types", tags=["types"])\ No newline at end of file+app.include_router(types.router, prefix="/api/types", tags=["types"])diff --git a/flowsint-api/app/models/models.py b/flowsint-api/app/models/models.pyindex b5bbca15..86cd56f7 100644--- a/flowsint-api/app/models/models.py+++ b/flowsint-api/app/models/models.py@@ -1,13 +1,21 @@import uuidfrom datetime import datetimefrom sqlalchemy import (- String, Text, DateTime, ForeignKey,- Index, func, JSON, Column, Enum as SQLEnum+ String,+ Text,+ DateTime,+ ForeignKey,+ Index,+ func,+ JSON,+ Column,+ Enum as SQLEnum,)from sqlalchemy.dialects.postgresql import UUID as PGUUID, ARRAY, JSONBfrom sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationshipfrom flowsint_core.core.enums import EventLevel+class Base(DeclarativeBase):pass@@ -18,19 +26,27 @@ class Feedback(Base):id: Mapped[uuid.UUID] = mapped_column(primary_key=True)created_at = mapped_column(DateTime(timezone=True), server_default=func.now())content = mapped_column(Text, nullable=True)- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id"), nullable=True)+ owner_id = mapped_column(+ PGUUID(as_uuid=True), ForeignKey("profiles.id"), nullable=True+ )class Investigation(Base):__tablename__ = "investigations"- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)+ id: Mapped[uuid.UUID] = mapped_column(+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4+ )created_at = mapped_column(DateTime(timezone=True), server_default=func.now())name = mapped_column(Text)description = mapped_column(Text)- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)+ owner_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),+ nullable=True,+ )last_updated_at = mapped_column(DateTime(timezone=True), server_default=func.now())- status = mapped_column(String, server_default='active')+ status = mapped_column(String, server_default="active")sketches = relationship("Sketch", back_populates="investigation")analyses = relationship("Analysis", back_populates="investigation")chats = relationship("Chat", back_populates="investigation")@@ -45,32 +61,51 @@ class InvestigationsProfiles(Base):id: Mapped[uuid.UUID] = mapped_column(primary_key=True)created_at = mapped_column(DateTime(timezone=True), server_default=func.now())- investigation_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"))- profile_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"))+ investigation_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"),+ )+ profile_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),+ )role = mapped_column(String, server_default="member")__table_args__ = (Index("idx_investigations_profiles_investigation_id", "investigation_id"),Index("idx_investigations_profiles_profile_id", "profile_id"),# Uniqueness constraint- Index("projects_profiles_unique_profile_project", "profile_id", "investigation_id", unique=True),+ Index(+ "projects_profiles_unique_profile_project",+ "profile_id",+ "investigation_id",+ unique=True,+ ),)class Log(Base):__tablename__ = "logs"- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)+ id: Mapped[uuid.UUID] = mapped_column(+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4+ )content = mapped_column(JSONB, nullable=True)created_at = mapped_column(DateTime(timezone=True), server_default=func.now())- sketch_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)+ sketch_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"),+ nullable=True,+ )type = Column(SQLEnum(EventLevel), default=EventLevel.INFO)class Profile(Base):__tablename__ = "profiles"- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)+ id: Mapped[uuid.UUID] = mapped_column(+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4+ )first_name = mapped_column(Text, nullable=True)last_name = mapped_column(Text, nullable=True)avatar_url = mapped_column(Text, nullable=True)@@ -82,17 +117,23 @@ class Profile(Base):class Scan(Base):__tablename__ = "scans"- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)- sketch_id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)+ id: Mapped[uuid.UUID] = mapped_column(+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4+ )+ sketch_id: Mapped[uuid.UUID] = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"),+ nullable=True,+ )status = Column(SQLEnum(EventLevel), default=EventLevel.PENDING)started_at = Column(DateTime, default=datetime.utcnow)completed_at = Column(DateTime, nullable=True)error = Column(Text, nullable=True)details = Column(JSON, nullable=True)-+# Relationshipssketch = relationship("Sketch", back_populates="scans")-+def __repr__(self):return f"<Scan(id={self.id}, status={self.status})>"@@ -100,13 +141,22 @@ class Scan(Base):class Sketch(Base):__tablename__ = "sketches"- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)+ id: Mapped[uuid.UUID] = mapped_column(+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4+ )title = mapped_column(Text)description = mapped_column(Text)created_at = mapped_column(DateTime(timezone=True), server_default=func.now())- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)+ owner_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),+ nullable=True,+ )status = mapped_column(String, server_default="active")- investigation_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"))+ investigation_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"),+ )investigation = relationship("Investigation", back_populates="sketches")last_updated_at = mapped_column(DateTime(timezone=True), server_default=func.now())scans = relationship("Scan", back_populates="sketch")@@ -122,21 +172,34 @@ class SketchesProfiles(Base):id: Mapped[uuid.UUID] = mapped_column(primary_key=True)created_at = mapped_column(DateTime(timezone=True), server_default=func.now())- profile_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"))- sketch_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"))+ profile_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),+ )+ sketch_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"),+ )role = mapped_column(String, server_default="editor")__table_args__ = (Index("idx_sketches_profiles_sketch_id", "sketch_id"),Index("idx_sketches_profiles_profile_id", "profile_id"),- Index("investigations_profiles_unique_profile_investigation", "profile_id", "sketch_id", unique=True),+ Index(+ "investigations_profiles_unique_profile_investigation",+ "profile_id",+ "sketch_id",+ unique=True,+ ),)class Transform(Base):__tablename__ = "transforms"- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)+ id: Mapped[uuid.UUID] = mapped_column(+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4+ )name = mapped_column(Text, nullable=False)description = mapped_column(Text, nullable=True)category = mapped_column(ARRAY(Text), nullable=True)@@ -148,14 +211,24 @@ class Transform(Base):class Analysis(Base):__tablename__ = "analyses"- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)+ id: Mapped[uuid.UUID] = mapped_column(+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4+ )title = mapped_column(Text, nullable=False)description = mapped_column(Text, nullable=True)content = mapped_column(JSON, nullable=True)created_at = mapped_column(DateTime(timezone=True), server_default=func.now())last_updated_at = mapped_column(DateTime(timezone=True), server_default=func.now())- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)- investigation_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)+ owner_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),+ nullable=True,+ )+ investigation_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"),+ nullable=True,+ )investigation = relationship("Investigation", back_populates="analyses")__table_args__ = (@@ -163,16 +236,27 @@ class Analysis(Base):Index("idx_analyses_investigation_id", "investigation_id"),)+class Chat(Base):__tablename__ = "chats"- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)+ id: Mapped[uuid.UUID] = mapped_column(+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4+ )title = mapped_column(Text, nullable=False)description = mapped_column(Text, nullable=True)created_at = mapped_column(DateTime(timezone=True), server_default=func.now())last_updated_at = mapped_column(DateTime(timezone=True), server_default=func.now())- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)- investigation_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)+ owner_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),+ nullable=True,+ )+ investigation_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"),+ nullable=True,+ )investigation = relationship("Investigation", back_populates="chats")messages = relationship("ChatMessage", back_populates="chat", cascade="all, delete")__table_args__ = (@@ -180,30 +264,42 @@ class Chat(Base):Index("idx_chats_investigation_id", "investigation_id"),)+class ChatMessage(Base):__tablename__ = "messages"- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)+ id: Mapped[uuid.UUID] = mapped_column(+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4+ )content = mapped_column(JSON, nullable=True)context = mapped_column(JSON, nullable=True)is_bot: Mapped[bool] = mapped_column(default=False)created_at = mapped_column(DateTime(timezone=True), server_default=func.now())- chat_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("chats.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=False)- chat = relationship("Chat", back_populates="messages")- __table_args__ = (- Index("idx_messages_chat_id", "chat_id"),+ chat_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("chats.id", onupdate="CASCADE", ondelete="CASCADE"),+ nullable=False,)+ chat = relationship("Chat", back_populates="messages")+ __table_args__ = (Index("idx_messages_chat_id", "chat_id"),)+class Key(Base):__tablename__ = "keys"- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)+ id: Mapped[uuid.UUID] = mapped_column(+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4+ )name = mapped_column(String, nullable=False)- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)+ owner_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),+ nullable=True,+ )encrypted_key = mapped_column(String, nullable=False)created_at = mapped_column(DateTime(timezone=True), server_default=func.now())__table_args__ = (Index("idx_keys_owner_id", "owner_id"),Index("idx_keys_service", "name"),- )\ No newline at end of file+ )diff --git a/flowsint-api/app/utils.py b/flowsint-api/app/utils.pyindex b1e02a8e..238eed7c 100644--- a/flowsint-api/app/utils.py+++ b/flowsint-api/app/utils.py@@ -13,6 +13,7 @@ import inspectfrom typing import Any, Dict, Typefrom pydantic import BaseModel, TypeAdapter+def is_valid_ip(address: str) -> bool:try:ipaddress.ip_address(address)@@ -20,11 +21,13 @@ def is_valid_ip(address: str) -> bool:except ValueError:return False+def is_valid_username(username: str) -> bool:if not re.match(r"^[a-zA-Z0-9_-]{3,30}$", username):return Falsereturn True+def is_valid_email(email: str) -> bool:if not re.match(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$", email):return False@@ -34,26 +37,29 @@ def is_valid_email(email: str) -> bool:def is_valid_domain(url_or_domain: str) -> str:try:- parsed = urlparse(url_or_domain if "://" in url_or_domain else "http://" + url_or_domain)+ parsed = urlparse(+ url_or_domain if "://" in url_or_domain else "http://" + url_or_domain+ )hostname = parsed.hostname or url_or_domain-+if not hostname or "." not in hostname:return False-+if not re.match(r"^[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$", hostname):return False-+return Trueexcept Exception as e:return False+def is_root_domain(domain: str) -> bool:"""Determine if a domain is a root domain or subdomain.-+Args:domain: The domain string to check-+Returns:True if it's a root domain (e.g., example.com), False if it's a subdomain (e.g., sub.example.com)"""@@ -62,20 +68,32 @@ def is_root_domain(domain: str) -> bool:if "://" in domain:parsed = urlparse(domain)domain = parsed.hostname or domain-+# Split by dots- parts = domain.split('.')-+ parts = domain.split(".")+# Handle common country code TLDs that have 2 parts (e.g., .co.uk, .com.au, .org.uk)- common_cc_tlds = ['.co.uk', '.com.au', '.org.uk', '.net.uk', '.gov.uk', '.ac.uk',- '.co.nz', '.com.sg', '.co.jp', '.co.kr', '.com.br', '.com.mx']-+ common_cc_tlds = [+ ".co.uk",+ ".com.au",+ ".org.uk",+ ".net.uk",+ ".gov.uk",+ ".ac.uk",+ ".co.nz",+ ".com.sg",+ ".co.jp",+ ".co.kr",+ ".com.br",+ ".com.mx",+ ]+# Check if the domain ends with a common country code TLDfor cc_tld in common_cc_tlds:if domain.endswith(cc_tld):# For country code TLDs, we need exactly 3 parts (e.g., example.co.uk)return len(parts) == 3-+# For regular TLDs, a root domain has 2 parts (e.g., example.com)# A subdomain has 3 or more parts (e.g., sub.example.com, www.sub.example.com)return len(parts) == 2@@ -83,6 +101,7 @@ def is_root_domain(domain: str) -> bool:# If we can't parse it, assume it's not a root domainreturn False+def is_valid_number(phone: str, region: str = "FR") -> None:"""Validates a phone number. Raises InvalidPhoneNumberError if invalid.@@ -94,19 +113,21 @@ def is_valid_number(phone: str, region: str = "FR") -> None:return Falseexcept NumberParseException:return False-++def parse_asn(asn: str) -> int:if not is_valid_asn(asn):raise ValueError(f"Invalid ASN format: {asn}")return int(re.sub(r"(?i)^AS", "", asn))-+def is_valid_asn(asn: str) -> bool:if not re.fullmatch(r"(AS)?\d+", asn, re.IGNORECASE):return Falseasn_num = int(re.sub(r"(?i)^AS", "", asn))return 0 <= asn_num <= 4294967295+def resolve_type(details: dict, schema_context: dict = None) -> str:if "anyOf" in details:types = []@@ -121,22 +142,23 @@ def resolve_type(details: dict, schema_context: dict = None) -> str:else:types.append(option.get("type", "unknown"))return " | ".join(types)-+if "type" in details:if details["type"] == "array":item_type = resolve_type(details.get("items", {}), schema_context)return f"{item_type}[]"return details["type"]-+# Handle $ref in array items or other contextsif "$ref" in details and schema_context:ref_path = details["$ref"]if ref_path.startswith("#/$defs/"):ref_name = ref_path.split("/")[-1]return ref_name-+return "any"+def extract_input_schema_transform(model: Type[BaseModel]) -> Dict[str, Any]:adapter = TypeAdapter(model)schema = adapter.json_schema()@@ -153,37 +175,30 @@ def extract_input_schema_transform(model: Type[BaseModel]) -> Dict[str, Any]:"outputs": {"type": type_name,"properties": [- {- "name": prop,- "type": resolve_type(info, schema)- }+ {"name": prop, "type": resolve_type(info, schema)}for prop, info in details.get("properties", {}).items()- ]- },- "inputs": {- "type": "",- "properties": []+ ],},+ "inputs": {"type": "", "properties": []},"type": "type","category": model.__name__,}-def get_domain_from_ssl(ip: str, port: int = 443) -> str | None:try:context = ssl.create_default_context()with socket.create_connection((ip, port), timeout=3) as sock:with context.wrap_socket(sock, server_hostname=ip) as ssock:cert = ssock.getpeercert()- subject = cert.get('subject', [])+ subject = cert.get("subject", [])for entry in subject:- if entry[0][0] == 'commonName':+ if entry[0][0] == "commonName":return entry[0][1]# Alternative: check subjectAltName- san = cert.get('subjectAltName', [])+ san = cert.get("subjectAltName", [])for typ, val in san:- if typ == 'DNS':+ if typ == "DNS":return valexcept Exception as e:print(f"SSL extraction failed for {ip}: {e}")@@ -208,12 +223,14 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:scanner_node = node_lookup.get(target_id)if scanner_node and scanner_node["data"]["type"] == "scanner":- scanners.append({- "scanner_name": scanner_node["data"]["name"],- "module": scanner_node["data"]["module"],- "input": source_handle,- "output": target_handle- })+ scanners.append(+ {+ "scanner_name": scanner_node["data"]["name"],+ "module": scanner_node["data"]["module"],+ "input": source_handle,+ "output": target_handle,+ }+ )return {"input": {@@ -221,25 +238,22 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:"outputs": input_output,},"scanners": scanners,- "scanner_names" : [scanner["scanner_name"] for scanner in scanners]-+ "scanner_names": [scanner["scanner_name"] for scanner in scanners],}-++def get_label_color(label: str) -> str:- color_map = {- 'subdomain': '#A5ABB6',- 'domain': '#68BDF6',- 'default': '#A5ABB6'- }-+ color_map = {"subdomain": "#A5ABB6", "domain": "#68BDF6", "default": "#A5ABB6"}+return color_map.get(label, color_map["default"])+def flatten(data_dict, prefix=""):"""Flattens a dictionary to contain only Neo4j-compatible property values.Neo4j supports primitive types (string, number, boolean) and arrays of those types.Args:- data_dict (dict): Dictionary to flatten+ data_dict (dict): Dictionary to flattenReturns:dict: Flattened dictionary with only Neo4j-compatible values"""@@ -250,13 +264,14 @@ def flatten(data_dict, prefix=""):if value is None:continueif isinstance(value, (str, int, float, bool)) or (- isinstance(value, list) and all(isinstance(item, (str, int, float, bool)) for item in value)+ isinstance(value, list)+ and all(isinstance(item, (str, int, float, bool)) for item in value)):key = f"{prefix}{key}"- flattened[key] = value+ flattened[key] = valuereturn flattened-+def get_inline_relationships(nodes: List[Any], edges: List[Any]) -> List[str]:"""Get the inline relationships for a list of nodes and edges.@@ -266,14 +281,15 @@ def get_inline_relationships(nodes: List[Any], edges: List[Any]) -> List[str]:source = next((node for node in nodes if node["id"] == edge["source"]), None)target = next((node for node in nodes if node["id"] == edge["target"]), None)if source and target:- relationships.append({"source":source, "edge":edge, "target":target})+ relationships.append({"source": source, "edge": edge, "target": target})return relationships+def to_json_serializable(obj):"""Convert any object to a JSON-serializable format."""import jsonfrom pydantic import BaseModel-+try:# Test if already JSON serializablejson.dumps(obj)@@ -282,11 +298,15 @@ def to_json_serializable(obj):# Handle common casesif isinstance(obj, BaseModel):# Use mode='json' to ensure all Pydantic types are properly serialized- return obj.model_dump(mode='json') if hasattr(obj, 'model_dump') else obj.dict()+ return (+ obj.model_dump(mode="json")+ if hasattr(obj, "model_dump")+ else obj.dict()+ )elif isinstance(obj, (list, tuple)):return [to_json_serializable(item) for item in obj]elif isinstance(obj, dict):return {key: to_json_serializable(value) for key, value in obj.items()}else:# Convert anything else to string- return str(obj)\ No newline at end of file+ return str(obj)diff --git a/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx b/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsxindex 38492352..e5e37a55 100644--- a/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx+++ b/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx@@ -71,19 +71,19 @@ export default function DetailsPanel({ node }: { node: GraphNode | null }) {)}<div className="flex-1 min-h-0"><ResizablePanelGroup direction="vertical" className="h-full">- <ResizablePanel defaultSize={30} minSize={20}>+ <ResizablePanel defaultSize={20} minSize={20}><div className="h-full overflow-auto"><KeyValueDisplay data={node.data} /></div></ResizablePanel><ResizableHandle />- <ResizablePanel defaultSize={40} minSize={25}>+ <ResizablePanel defaultSize={35} minSize={25}><div className="h-full p-3"><NeighborsGraph sketchId={sketchId as string} nodeId={node.id} /></div></ResizablePanel><ResizableHandle />- <ResizablePanel defaultSize={30} minSize={20}>+ <ResizablePanel defaultSize={45} minSize={20}><Relationships sketchId={sketchId as string} nodeId={node.id} /></ResizablePanel></ResizablePanelGroup>diff --git a/flowsint-core/src/flowsint_core/__init__.py b/flowsint-core/src/flowsint_core/__init__.pyindex c5f63f49..c6c2baa3 100644--- a/flowsint-core/src/flowsint_core/__init__.py+++ b/flowsint-core/src/flowsint_core/__init__.py@@ -16,4 +16,4 @@ from .core.vault import *from .utils import *__version__ = "0.1.0"-__author__ = "EliottElek <[email protected]>"\ No newline at end of file+__author__ = "EliottElek <[email protected]>"diff --git a/flowsint-core/src/flowsint_core/core/__init__.py b/flowsint-core/src/flowsint_core/core/__init__.pydeleted file mode 100644index 5e2b07a9..00000000--- a/flowsint-core/src/flowsint_core/core/__init__.py+++ /dev/null@@ -1,35 +0,0 @@-"""-Core module initialization.-"""--from .registry import ScannerRegistry--# Register scanner modules for discovery-ScannerRegistry.register_module("flowsint_transforms.domains.subdomains")-ScannerRegistry.register_module("flowsint_transforms.domains.whois")-ScannerRegistry.register_module("flowsint_transforms.domains.resolve")-ScannerRegistry.register_module("flowsint_transforms.ips.reverse_resolve")-ScannerRegistry.register_module("flowsint_transforms.ips.geolocation")-ScannerRegistry.register_module("flowsint_transforms.socials.maigret")-ScannerRegistry.register_module("flowsint_transforms.ips.ip_to_asn")-ScannerRegistry.register_module("flowsint_transforms.ips.asn_to_cidrs")-ScannerRegistry.register_module("flowsint_transforms.ips.cidr_to_ips")-ScannerRegistry.register_module("flowsint_transforms.organizations.org_to_asn")-ScannerRegistry.register_module("flowsint_transforms.domains.domain_to_asn")-ScannerRegistry.register_module("flowsint_transforms.crypto.wallet_to_transactions")-ScannerRegistry.register_module("flowsint_transforms.crypto.wallet_to_nfts")-ScannerRegistry.register_module("flowsint_transforms.domains.to_website")-ScannerRegistry.register_module("flowsint_transforms.websites.to_crawler")-ScannerRegistry.register_module("flowsint_transforms.websites.to_links")-ScannerRegistry.register_module("flowsint_transforms.websites.to_domain")-ScannerRegistry.register_module("flowsint_transforms.emails.to_gravatar")-ScannerRegistry.register_module("flowsint_transforms.emails.to_leaks")-ScannerRegistry.register_module("flowsint_transforms.individuals.to_org")-ScannerRegistry.register_module("flowsint_transforms.organizations.to_infos")-ScannerRegistry.register_module("flowsint_transforms.websites.to_webtrackers")-ScannerRegistry.register_module("flowsint_transforms.n8n.connector")--# Discover and register all scanners-ScannerRegistry.discover_scanners()--__all__ = ["ScannerRegistry"]diff --git a/flowsint-core/src/flowsint_core/core/auth.py b/flowsint-core/src/flowsint_core/core/auth.pyindex cedab67b..6b601bc2 100644--- a/flowsint-core/src/flowsint_core/core/auth.py+++ b/flowsint-core/src/flowsint_core/core/auth.py@@ -10,21 +10,28 @@ load_dotenv()# Get AUTH_SECRET after loading environment variablesAUTH_SECRET = os.getenv("AUTH_SECRET")if not AUTH_SECRET:- raise ValueError("AUTH_SECRET environment variable is not set. Please set it in your .env file.")+ raise ValueError(+ "AUTH_SECRET environment variable is not set. Please set it in your .env file."+ )ALGORITHM = "HS256"ACCESS_TOKEN_EXPIRE_MINUTES = 60 * 60pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")+def verify_password(plain_password: str, hashed_password: str) -> bool:return pwd_context.verify(plain_password, hashed_password)+def get_password_hash(password: str) -> str:return pwd_context.hash(password)+def create_access_token(data: dict, expires_delta: timedelta | None = None):to_encode = data.copy()- expire = datetime.utcnow() + (expires_delta or timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES))+ expire = datetime.utcnow() + (+ expires_delta or timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES)+ )to_encode.update({"exp": expire})encoded_jwt = jwt.encode(to_encode, AUTH_SECRET, algorithm=ALGORITHM)- return encoded_jwt\ No newline at end of file+ return encoded_jwtdiff --git a/flowsint-core/src/flowsint_core/core/celery.py b/flowsint-core/src/flowsint_core/core/celery.pyindex 58cd63b5..1da80106 100644--- a/flowsint-core/src/flowsint_core/core/celery.py+++ b/flowsint-core/src/flowsint_core/core/celery.py@@ -8,7 +8,7 @@ celery = Celery(include=["flowsint_core.tasks.event","flowsint_core.tasks.transform",- ]+ ],)celery.conf.update(@@ -20,5 +20,5 @@ celery.conf.update(task_track_started=True,task_time_limit=3600, # 1 hourworker_max_tasks_per_child=1000,- worker_prefetch_multiplier=1-)\ No newline at end of file+ worker_prefetch_multiplier=1,+)diff --git a/flowsint-core/src/flowsint_core/core/config.py b/flowsint-core/src/flowsint_core/core/config.pyindex e7a744be..bec1c203 100644--- a/flowsint-core/src/flowsint_core/core/config.py+++ b/flowsint-core/src/flowsint_core/core/config.py@@ -3,8 +3,10 @@ from dotenv import load_dotenvload_dotenv()+class Settings:CELERY_BROKER_URL = os.getenv("REDIS_URI", "redis://localhost:6379/0")CELERY_RESULT_BACKEND = os.getenv("REDIS_URI", "redis://localhost:6379/0")-settings = Settings()\ No newline at end of file++settings = Settings()diff --git a/flowsint-core/src/flowsint_core/core/enums.py b/flowsint-core/src/flowsint_core/core/enums.pyindex f0c37bb6..49471c73 100644--- a/flowsint-core/src/flowsint_core/core/enums.py+++ b/flowsint-core/src/flowsint_core/core/enums.py@@ -1,5 +1,6 @@from enum import Enum+class EventLevel(str, Enum):# Standard log levelsINFO = "INFO"@@ -21,4 +22,4 @@ class EventLevel(str, Enum):@propertydef lowercase(self) -> str:"""Get the lowercase version of the enum value"""- return self.value.lower()\ No newline at end of file+ return self.value.lower()diff --git a/flowsint-core/src/flowsint_core/core/events.py b/flowsint-core/src/flowsint_core/core/events.pyindex 744fca85..a2d9a88b 100644--- a/flowsint-core/src/flowsint_core/core/events.py+++ b/flowsint-core/src/flowsint_core/core/events.py@@ -7,6 +7,7 @@ from uuid import UUIDimport redis.asyncio as redisimport os+class EventEmitter:def __init__(self):self.id = uuid.uuid4()@@ -31,12 +32,14 @@ class EventEmitter:"""Get the next message from Redis for a specific channel"""if channel not in self.pubsubs:return None-- message = await self.pubsubs[channel].get_message(ignore_subscribe_messages=True)++ message = await self.pubsubs[channel].get_message(+ ignore_subscribe_messages=True+ )if message is None:await asyncio.sleep(0.1)return None-+if message:data = message["data"]if isinstance(data, bytes):@@ -57,8 +60,10 @@ class EventEmitter:except ValueError:return False+event_emitter = EventEmitter()+def init_events(app: FastAPI):"""Initialize the event system in the FastAPI app"""- print("[EventEmitter] Events initialized in FastAPI app")\ No newline at end of file+ print("[EventEmitter] Events initialized in FastAPI app")diff --git a/flowsint-core/src/flowsint_core/core/graph_db.py b/flowsint-core/src/flowsint_core/core/graph_db.pyindex 1d8d1503..84a256c2 100644--- a/flowsint-core/src/flowsint_core/core/graph_db.py+++ b/flowsint-core/src/flowsint_core/core/graph_db.py@@ -4,18 +4,21 @@ from dotenv import load_dotenvload_dotenv()+class Neo4jConnection:def __init__(self, uri: str, user: str, password: str):self.__uri = uriself.__user = userself.__password = password- self.__driver = GraphDatabase.driver(self.__uri, auth=(self.__user, self.__password))+ self.__driver = GraphDatabase.driver(+ self.__uri, auth=(self.__user, self.__password)+ )def query(self, query: str, parameters: dict = None):with self.__driver.session() as session:result = session.run(query, parameters)return result.data()-+URI = os.getenv("NEO4J_URI_BOLT")URI = "bolt://localhost:7687"@@ -25,7 +28,6 @@ PASSWORD = os.getenv("NEO4J_PASSWORD")neo4j_connection = Neo4jConnection(URI, USERNAME, PASSWORD)-# import os# from neo4j import GraphDatabase# from dotenv import load_dotenv@@ -43,7 +45,7 @@ neo4j_connection = Neo4jConnection(URI, USERNAME, PASSWORD)# with self.__driver.session() as session:# result = session.run(query, parameters)# return result.data()-+# URI = os.getenv("NEO4J_URI_BOLT")# USERNAME = os.getenv("NEO4J_USERNAME")diff --git a/flowsint-core/src/flowsint_core/core/logger.py b/flowsint-core/src/flowsint_core/core/logger.pyindex 88627814..eb6905b1 100644--- a/flowsint-core/src/flowsint_core/core/logger.py+++ b/flowsint-core/src/flowsint_core/core/logger.py@@ -8,17 +8,16 @@ import logginglogger = logging.getLogger(__name__)+class Logger:@staticmethod- def _create_log(sketch_id: Union[str, UUID], level: EventLevel, content: str) -> Log:+ def _create_log(+ sketch_id: Union[str, UUID], level: EventLevel, content: str+ ) -> Log:"""Create a log entry in the database"""db = next(get_db())try:- log = Log(- sketch_id=str(sketch_id),- type=level.value,- content=content- )+ log = Log(sketch_id=str(sketch_id), type=level.value, content=content)db.add(log)db.commit()db.refresh(log)@@ -30,46 +29,62 @@ class Logger:def info(sketch_id: Union[str, UUID], message: Dict):"""Log an info message"""log = Logger._create_log(sketch_id, EventLevel.INFO, message)- emit_event_task.apply(args=[str(log.id), str(sketch_id), EventLevel.INFO, message])+ emit_event_task.apply(+ args=[str(log.id), str(sketch_id), EventLevel.INFO, message]+ )@staticmethoddef error(sketch_id: Union[str, UUID], message: Dict):"""Log an error message"""log = Logger._create_log(sketch_id, EventLevel.FAILED, message)- emit_event_task.apply(args=[str(log.id), str(sketch_id), EventLevel.FAILED, message])+ emit_event_task.apply(+ args=[str(log.id), str(sketch_id), EventLevel.FAILED, message]+ )@staticmethoddef warn(sketch_id: Union[str, UUID], message: Dict):"""Log a warning message"""log = Logger._create_log(sketch_id, EventLevel.WARNING, message)- emit_event_task.apply(args=[str(log.id), str(sketch_id), EventLevel.WARNING, message])+ emit_event_task.apply(+ args=[str(log.id), str(sketch_id), EventLevel.WARNING, message]+ )@staticmethoddef debug(sketch_id: Union[str, UUID], message: Dict):"""Log a debug message"""log = Logger._create_log(sketch_id, EventLevel.DEBUG, message)- emit_event_task.apply(args=[str(log.id), str(sketch_id), EventLevel.DEBUG, message])-+ emit_event_task.apply(+ args=[str(log.id), str(sketch_id), EventLevel.DEBUG, message]+ )+@staticmethoddef graph_append(sketch_id: Union[str, UUID], message: Dict):"""Log an insert event to the graph"""log = Logger._create_log(sketch_id, EventLevel.GRAPH_APPEND, message)- emit_event_task.apply(args=[str(log.id), str(sketch_id), EventLevel.GRAPH_APPEND, message])+ emit_event_task.apply(+ args=[str(log.id), str(sketch_id), EventLevel.GRAPH_APPEND, message]+ )@staticmethoddef success(sketch_id: Union[str, UUID], message: Dict):"""Log a success message"""log = Logger._create_log(sketch_id, EventLevel.SUCCESS, message)- emit_event_task.apply(args=[str(log.id), str(sketch_id), EventLevel.SUCCESS, message])-+ emit_event_task.apply(+ args=[str(log.id), str(sketch_id), EventLevel.SUCCESS, message]+ )+@staticmethoddef completed(sketch_id: Union[str, UUID], message: Dict):"""Log a completed message"""log = Logger._create_log(sketch_id, EventLevel.COMPLETED, message)- emit_event_task.apply(args=[str(log.id), str(sketch_id), EventLevel.COMPLETED, message])-+ emit_event_task.apply(+ args=[str(log.id), str(sketch_id), EventLevel.COMPLETED, message]+ )+@staticmethoddef pending(sketch_id: Union[str, UUID], message: Dict):"""Log a pending message"""log = Logger._create_log(sketch_id, EventLevel.PENDING, message)- emit_event_task.apply(args=[str(log.id), str(sketch_id), EventLevel.PENDING, message])+ emit_event_task.apply(+ args=[str(log.id), str(sketch_id), EventLevel.PENDING, message]+ )diff --git a/flowsint-core/src/flowsint_core/core/models.py b/flowsint-core/src/flowsint_core/core/models.pyindex b385335a..f12a8a18 100644--- a/flowsint-core/src/flowsint_core/core/models.py+++ b/flowsint-core/src/flowsint_core/core/models.py@@ -1,13 +1,21 @@import uuidfrom datetime import datetimefrom sqlalchemy import (- String, Text, DateTime, ForeignKey,- Index, func, JSON, Column, Enum as SQLEnum+ String,+ Text,+ DateTime,+ ForeignKey,+ Index,+ func,+ JSON,+ Column,+ Enum as SQLEnum,)from sqlalchemy.dialects.postgresql import UUID as PGUUID, ARRAY, JSONBfrom sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationshipfrom .enums import EventLevel+class Base(DeclarativeBase):pass@@ -18,19 +26,27 @@ class Feedback(Base):id: Mapped[uuid.UUID] = mapped_column(primary_key=True)created_at = mapped_column(DateTime(timezone=True), server_default=func.now())content = mapped_column(Text, nullable=True)- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id"), nullable=True)+ owner_id = mapped_column(+ PGUUID(as_uuid=True), ForeignKey("profiles.id"), nullable=True+ )class Investigation(Base):__tablename__ = "investigations"- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)+ id: Mapped[uuid.UUID] = mapped_column(+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4+ )created_at = mapped_column(DateTime(timezone=True), server_default=func.now())name = mapped_column(Text)description = mapped_column(Text)- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)+ owner_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),+ nullable=True,+ )last_updated_at = mapped_column(DateTime(timezone=True), server_default=func.now())- status = mapped_column(String, server_default='active')+ status = mapped_column(String, server_default="active")sketches = relationship("Sketch", back_populates="investigation")analyses = relationship("Analysis", back_populates="investigation")chats = relationship("Chat", back_populates="investigation")@@ -45,32 +61,51 @@ class InvestigationsProfiles(Base):id: Mapped[uuid.UUID] = mapped_column(primary_key=True)created_at = mapped_column(DateTime(timezone=True), server_default=func.now())- investigation_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"))- profile_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"))+ investigation_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"),+ )+ profile_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),+ )role = mapped_column(String, server_default="member")__table_args__ = (Index("idx_investigations_profiles_investigation_id", "investigation_id"),Index("idx_investigations_profiles_profile_id", "profile_id"),# Uniqueness constraint- Index("projects_profiles_unique_profile_project", "profile_id", "investigation_id", unique=True),+ Index(+ "projects_profiles_unique_profile_project",+ "profile_id",+ "investigation_id",+ unique=True,+ ),)class Log(Base):__tablename__ = "logs"- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)+ id: Mapped[uuid.UUID] = mapped_column(+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4+ )content = mapped_column(JSONB, nullable=True)created_at = mapped_column(DateTime(timezone=True), server_default=func.now())- sketch_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)+ sketch_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"),+ nullable=True,+ )type = Column(SQLEnum(EventLevel), default=EventLevel.INFO)class Profile(Base):__tablename__ = "profiles"- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)+ id: Mapped[uuid.UUID] = mapped_column(+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4+ )first_name = mapped_column(Text, nullable=True)last_name = mapped_column(Text, nullable=True)avatar_url = mapped_column(Text, nullable=True)@@ -82,17 +117,23 @@ class Profile(Base):class Scan(Base):__tablename__ = "scans"- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)- sketch_id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)+ id: Mapped[uuid.UUID] = mapped_column(+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4+ )+ sketch_id: Mapped[uuid.UUID] = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"),+ nullable=True,+ )status = Column(SQLEnum(EventLevel), default=EventLevel.PENDING)started_at = Column(DateTime, default=datetime.utcnow)completed_at = Column(DateTime, nullable=True)error = Column(Text, nullable=True)details = Column(JSON, nullable=True)-+# Relationshipssketch = relationship("Sketch", back_populates="scans")-+def __repr__(self):return f"<Scan(id={self.id}, status={self.status})>"@@ -100,13 +141,22 @@ class Scan(Base):class Sketch(Base):__tablename__ = "sketches"- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)+ id: Mapped[uuid.UUID] = mapped_column(+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4+ )title = mapped_column(Text)description = mapped_column(Text)created_at = mapped_column(DateTime(timezone=True), server_default=func.now())- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)+ owner_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),+ nullable=True,+ )status = mapped_column(String, server_default="active")- investigation_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"))+ investigation_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"),+ )investigation = relationship("Investigation", back_populates="sketches")last_updated_at = mapped_column(DateTime(timezone=True), server_default=func.now())scans = relationship("Scan", back_populates="sketch")@@ -122,21 +172,34 @@ class SketchesProfiles(Base):id: Mapped[uuid.UUID] = mapped_column(primary_key=True)created_at = mapped_column(DateTime(timezone=True), server_default=func.now())- profile_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"))- sketch_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"))+ profile_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),+ )+ sketch_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("sketches.id", onupdate="CASCADE", ondelete="CASCADE"),+ )role = mapped_column(String, server_default="editor")__table_args__ = (Index("idx_sketches_profiles_sketch_id", "sketch_id"),Index("idx_sketches_profiles_profile_id", "profile_id"),- Index("investigations_profiles_unique_profile_investigation", "profile_id", "sketch_id", unique=True),+ Index(+ "investigations_profiles_unique_profile_investigation",+ "profile_id",+ "sketch_id",+ unique=True,+ ),)class Transform(Base):__tablename__ = "transforms"- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)+ id: Mapped[uuid.UUID] = mapped_column(+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4+ )name = mapped_column(Text, nullable=False)description = mapped_column(Text, nullable=True)category = mapped_column(ARRAY(Text), nullable=True)@@ -148,14 +211,24 @@ class Transform(Base):class Analysis(Base):__tablename__ = "analyses"- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)+ id: Mapped[uuid.UUID] = mapped_column(+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4+ )title = mapped_column(Text, nullable=False)description = mapped_column(Text, nullable=True)content = mapped_column(JSON, nullable=True)created_at = mapped_column(DateTime(timezone=True), server_default=func.now())last_updated_at = mapped_column(DateTime(timezone=True), server_default=func.now())- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)- investigation_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)+ owner_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),+ nullable=True,+ )+ investigation_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"),+ nullable=True,+ )investigation = relationship("Investigation", back_populates="analyses")__table_args__ = (@@ -163,16 +236,27 @@ class Analysis(Base):Index("idx_analyses_investigation_id", "investigation_id"),)+class Chat(Base):__tablename__ = "chats"- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)+ id: Mapped[uuid.UUID] = mapped_column(+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4+ )title = mapped_column(Text, nullable=False)description = mapped_column(Text, nullable=True)created_at = mapped_column(DateTime(timezone=True), server_default=func.now())last_updated_at = mapped_column(DateTime(timezone=True), server_default=func.now())- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)- investigation_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)+ owner_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),+ nullable=True,+ )+ investigation_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("investigations.id", onupdate="CASCADE", ondelete="CASCADE"),+ nullable=True,+ )investigation = relationship("Investigation", back_populates="chats")messages = relationship("ChatMessage", back_populates="chat", cascade="all, delete")__table_args__ = (@@ -180,30 +264,42 @@ class Chat(Base):Index("idx_chats_investigation_id", "investigation_id"),)+class ChatMessage(Base):__tablename__ = "messages"- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)+ id: Mapped[uuid.UUID] = mapped_column(+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4+ )content = mapped_column(JSON, nullable=True)context = mapped_column(JSON, nullable=True)is_bot: Mapped[bool] = mapped_column(default=False)created_at = mapped_column(DateTime(timezone=True), server_default=func.now())- chat_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("chats.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=False)- chat = relationship("Chat", back_populates="messages")- __table_args__ = (- Index("idx_messages_chat_id", "chat_id"),+ chat_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("chats.id", onupdate="CASCADE", ondelete="CASCADE"),+ nullable=False,)+ chat = relationship("Chat", back_populates="messages")+ __table_args__ = (Index("idx_messages_chat_id", "chat_id"),)+class Key(Base):__tablename__ = "keys"- id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)+ id: Mapped[uuid.UUID] = mapped_column(+ PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4+ )name = mapped_column(String, nullable=False)- owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)+ owner_id = mapped_column(+ PGUUID(as_uuid=True),+ ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"),+ nullable=True,+ )encrypted_key = mapped_column(String, nullable=False)created_at = mapped_column(DateTime(timezone=True), server_default=func.now())__table_args__ = (Index("idx_keys_owner_id", "owner_id"),Index("idx_keys_service", "name"),- )\ No newline at end of file+ )diff --git a/flowsint-core/src/flowsint_core/core/orchestrator.py b/flowsint-core/src/flowsint_core/core/orchestrator.pyindex fd584f14..ba8f8bc6 100644--- a/flowsint-core/src/flowsint_core/core/orchestrator.py+++ b/flowsint-core/src/flowsint_core/core/orchestrator.py@@ -11,11 +11,20 @@ import asyncioimport jsonimport os+class TransformOrchestrator(Scanner):"""Orchestrator for running a list of scanners."""- def __init__(self, sketch_id: str, scan_id: str, transform_branches: List[FlowBranch], neo4j_conn=None, vault=None):++ def __init__(+ self,+ sketch_id: str,+ scan_id: str,+ transform_branches: List[FlowBranch],+ neo4j_conn=None,+ vault=None,+ ):super().__init__(sketch_id, scan_id, neo4j_conn=neo4j_conn, vault=vault)self.transform_branches = transform_branchesself.scanners = {} # Map of nodeId -> scanner instance@@ -31,14 +40,14 @@ class TransformOrchestrator(Scanner):# Create a directory for storing transform files if it doesn't existtransform_dir = "transform_logs"os.makedirs(transform_dir, exist_ok=True)-+# Create filename with sketch_id and scan_idfilename = f"transform_execution_{self.sketch_id}_{self.scan_id}.json"self.execution_log_file = os.path.join(transform_dir, filename)-+# Serialize the transform branchesserialized_branches = to_json_serializable(self.transform_branches)-+# Create initial log structureinitial_log = {"sketch_id": self.sketch_id,@@ -52,11 +61,11 @@ class TransformOrchestrator(Scanner):"total_steps": 0,"completed_steps": 0,"failed_steps": 0,- "total_execution_time_ms": 0+ "total_execution_time_ms": 0,},- "final_results": {}+ "final_results": {},}-+# Count total stepstotal_steps = 0for branch in self.transform_branches:@@ -64,55 +73,68 @@ class TransformOrchestrator(Scanner):if step.type != "type":total_steps += 1initial_log["summary"]["total_steps"] = total_steps-+# Save initial log file- with open(self.execution_log_file, 'w', encoding='utf-8') as f:+ with open(self.execution_log_file, "w", encoding="utf-8") as f:json.dump(initial_log, f, indent=2, ensure_ascii=False)-- Logger.info(self.sketch_id, {"message": f"Transform execution log created at {self.execution_log_file}"})-++ Logger.info(+ self.sketch_id,+ {+ "message": f"Transform execution log created at {self.execution_log_file}"+ },+ )+except Exception as e:- Logger.error(self.sketch_id, {"message": f"Failed to create execution log: {str(e)}"})+ Logger.error(+ self.sketch_id, {"message": f"Failed to create execution log: {str(e)}"}+ )self.execution_log_file = None- def _update_execution_log(self, step_entry: Dict[str, Any], status: str = None) -> None:+ def _update_execution_log(+ self, step_entry: Dict[str, Any], status: str = None+ ) -> None:"""Update the execution log file with a new step entry or status update."""if not self.execution_log_file:return-+try:# Read current log- with open(self.execution_log_file, 'r', encoding='utf-8') as f:+ with open(self.execution_log_file, "r", encoding="utf-8") as f:log_data = json.load(f)-+# Update timestamplog_data["updated_at"] = datetime.now().isoformat()-+# Update status if providedif status:log_data["status"] = status-+# Add step entry if providedif step_entry:log_data["execution_log"].append(step_entry)-+# Update summaryif step_entry["status"] == "completed":log_data["summary"]["completed_steps"] += 1elif step_entry["status"] == "error":log_data["summary"]["failed_steps"] += 1-+if "execution_time_ms" in step_entry:- log_data["summary"]["total_execution_time_ms"] += step_entry["execution_time_ms"]-+ log_data["summary"]["total_execution_time_ms"] += step_entry[+ "execution_time_ms"+ ]+# Write updated log- with open(self.execution_log_file, 'w', encoding='utf-8') as f:+ with open(self.execution_log_file, "w", encoding="utf-8") as f:json.dump(log_data, f, indent=2, ensure_ascii=False)-+except Exception as e:- Logger.error(self.sketch_id, {"message": f"Failed to update execution log: {str(e)}"})+ Logger.error(+ self.sketch_id, {"message": f"Failed to update execution log: {str(e)}"}+ )def _finalize_execution_log(self, final_results: Dict[str, Any]) -> None:"""@@ -120,25 +142,33 @@ class TransformOrchestrator(Scanner):"""if not self.execution_log_file:return-+try:# Read current log- with open(self.execution_log_file, 'r', encoding='utf-8') as f:+ with open(self.execution_log_file, "r", encoding="utf-8") as f:log_data = json.load(f)-+# Update final datalog_data["updated_at"] = datetime.now().isoformat()log_data["status"] = "completed"log_data["final_results"] = to_json_serializable(final_results)-+# Write final log- with open(self.execution_log_file, 'w', encoding='utf-8') as f:+ with open(self.execution_log_file, "w", encoding="utf-8") as f:json.dump(log_data, f, indent=2, ensure_ascii=False)-- Logger.info(self.sketch_id, {"message": f"Transform execution log finalized at {self.execution_log_file}"})-++ Logger.info(+ self.sketch_id,+ {+ "message": f"Transform execution log finalized at {self.execution_log_file}"+ },+ )+except Exception as e:- Logger.error(self.sketch_id, {"message": f"Failed to finalize execution log: {str(e)}"})+ Logger.error(+ self.sketch_id,+ {"message": f"Failed to finalize execution log: {str(e)}"},+ )def _save_transform_branches(self) -> None:"""@@ -148,27 +178,37 @@ class TransformOrchestrator(Scanner):# Create a directory for storing transform files if it doesn't existtransform_dir = "transform_logs"os.makedirs(transform_dir, exist_ok=True)-+# Create filename with sketch_id and scan_idfilename = f"transform_branches_{self.sketch_id}_{self.scan_id}.json"filepath = os.path.join(transform_dir, filename)-+# Serialize the transform branchesserialized_branches = to_json_serializable(self.transform_branches)-+# Save to JSON file- with open(filepath, 'w', encoding='utf-8') as f:- json.dump({- "sketch_id": self.sketch_id,- "scan_id": self.scan_id,- "timestamp": datetime.now().isoformat(),- "transform_branches": serialized_branches- }, f, indent=2, ensure_ascii=False)-- Logger.info(self.sketch_id, {"message": f"Transform branches saved to {filepath}"})-+ with open(filepath, "w", encoding="utf-8") as f:+ json.dump(+ {+ "sketch_id": self.sketch_id,+ "scan_id": self.scan_id,+ "timestamp": datetime.now().isoformat(),+ "transform_branches": serialized_branches,+ },+ f,+ indent=2,+ ensure_ascii=False,+ )++ Logger.info(+ self.sketch_id, {"message": f"Transform branches saved to {filepath}"}+ )+except Exception as e:- Logger.error(self.sketch_id, {"message": f"Failed to save transform branches: {str(e)}"})+ Logger.error(+ self.sketch_id,+ {"message": f"Failed to save transform branches: {str(e)}"},+ )def _load_scanners(self) -> None:if not self.transform_branches:@@ -181,29 +221,31 @@ class TransformOrchestrator(Scanner):if step.type == "type":continuescanner_nodes.append(step)-+if not scanner_nodes:raise ValueError("No scanner nodes found in transform branches")-+# Create scanner instances for each nodefor node in scanner_nodes:node_id = node.nodeId-+# Extract scanner name from nodeId (assuming format like "scanner_name-1234567890")- scanner_name = node_id.split('-')[0]-+ scanner_name = node_id.split("-")[0]+if not ScannerRegistry.scanner_exists(scanner_name):raise ValueError(f"Scanner '{scanner_name}' not found in registry")# Pass the step params to the scanner instance- scanner_params = node.params if hasattr(node, 'params') and node.params else {}+ scanner_params = (+ node.params if hasattr(node, "params") and node.params else {}+ )scanner = ScannerRegistry.get_scanner(- scanner_name,- self.sketch_id,- self.scan_id,+ scanner_name,+ self.sketch_id,+ self.scan_id,neo4j_conn=self.neo4j_conn,vault=self.vault,- params=scanner_params+ params=scanner_params,)self.scanners[node_id] = scanner@@ -216,13 +258,15 @@ class TransformOrchestrator(Scanner):return results_mapping[ref_value]return None- def prepare_scanner_inputs(self, step: FlowStep, results_mapping: Dict[str, Any], initial_values: List[str]) -> List[Any]:+ def prepare_scanner_inputs(+ self, step: FlowStep, results_mapping: Dict[str, Any], initial_values: List[str]+ ) -> List[Any]:"""Prepare the inputs for a scanner based on the references and previous results.Handles single references, lists, and direct values."""inputs = {}-+for input_key, input_ref in step.inputs.items():# Cas 1 : une seule référence (string)if isinstance(input_ref, str):@@ -251,10 +295,15 @@ class TransformOrchestrator(Scanner):primary_key = scanner.key()return {primary_key: initial_values}else:- scanner = self.scanners.get(step.nodeId)+ scanner = self.scanners.get(step.nodeId)return inputs[input_key]- def update_results_mapping(self, outputs: Dict[str, Any], step_outputs: Dict[str, str], results_mapping: Dict[str, Any]) -> None:+ def update_results_mapping(+ self,+ outputs: Dict[str, Any],+ step_outputs: Dict[str, str],+ results_mapping: Dict[str, Any],+ ) -> None:"""Update the results mapping with new outputs from a scanner."""@@ -280,10 +329,7 @@ class TransformOrchestrator(Scanner):@classmethoddef output_schema(cls) -> Dict[str, str]:- return {- "branches": "array",- "results": "dict"- }+ return {"branches": "array", "results": "dict"}def scan(self, values: List[str]) -> Dict[str, Any]:"""@@ -303,54 +349,49 @@ class TransformOrchestrator(Scanner):"""# Update execution log to indicate scan has startedself._update_execution_log(None, "running")-- results = {- "initial_values": values,- "branches": [],- "results": {}- }++ results = {"initial_values": values, "branches": [], "results": {}}Logger.pending(self.sketch_id, {"message": "Starting transform..."})-+# Global mapping of output references to actual valuesresults_mapping = {}# Cache for scanner results to avoid recomputationscanner_results_cache = {}-+total_steps = sum(len(branch.steps) for branch in self.transform_branches)completed_steps = 0-+# Process each branchfor branch in self.transform_branches:branch_id = branch.idbranch_name = branch.name- branch_results = {- "id": branch_id,- "name": branch_name,- "steps": []- }-+ branch_results = {"id": branch_id, "name": branch_name, "steps": []}+# Process each step in the branchscanner_inputs = valuesfor step in branch.steps:if step.type == "type":continue-+node_id = step.nodeIdscanner = self.scanners.get(node_id)-+if not scanner:- Logger.error(self.sketch_id, {"message": f"Scanner not found for node {node_id}"})+ Logger.error(+ self.sketch_id,+ {"message": f"Scanner not found for node {node_id}"},+ )continue-+scanner_name = scanner.name()step_start_time = time.time()-+step_result = {"nodeId": node_id,"scanner": scanner_name,- "status": "error" # Default to error, will update on success+ "status": "error", # Default to error, will update on success}-+# Create execution log entrylog_entry = {"step_id": f"{branch_id}_{node_id}",@@ -363,9 +404,9 @@ class TransformOrchestrator(Scanner):"status": "running","error": None,"timestamp": datetime.now().isoformat(),- "execution_time_ms": 0+ "execution_time_ms": 0,}-+try:# Update status for current stepcompleted_steps += 1@@ -374,11 +415,13 @@ class TransformOrchestrator(Scanner):step_result["error"] = error_msglog_entry["status"] = "error"log_entry["error"] = error_msg- log_entry["execution_time_ms"] = int((time.time() - step_start_time) * 1000)+ log_entry["execution_time_ms"] = int(+ (time.time() - step_start_time) * 1000+ )self._update_execution_log(log_entry)branch_results["steps"].append(step_result)continue-+# Check if we already have results for this scanner with these inputscache_key = f"{node_id}:{str(scanner_inputs)}"if cache_key in scanner_results_cache:@@ -388,20 +431,24 @@ class TransformOrchestrator(Scanner):# Execute the scanneroutputs = await scanner.execute(scanner_inputs)if not isinstance(outputs, (dict, list)):- raise ValueError(f"Scanner '{scanner_name}' returned unsupported output format")+ raise ValueError(+ f"Scanner '{scanner_name}' returned unsupported output format"+ )# Cache the resultsscanner_results_cache[cache_key] = outputslog_entry["cache_hit"] = False-+# Store the outputs in the step result (serialize to avoid JSON issues)step_result["outputs"] = to_json_serializable(outputs)step_result["status"] = "completed"-+# Update log entry with successlog_entry["outputs"] = to_json_serializable(outputs)log_entry["status"] = "completed"- log_entry["execution_time_ms"] = int((time.time() - step_start_time) * 1000)-+ log_entry["execution_time_ms"] = int(+ (time.time() - step_start_time) * 1000+ )+# Update the global results mapping with the outputsself.update_results_mapping(outputs, step.outputs, results_mapping)# Also store the raw outputs in the main results@@ -414,35 +461,40 @@ class TransformOrchestrator(Scanner):step_result["error"] = error_msglog_entry["status"] = "error"log_entry["error"] = error_msg- log_entry["execution_time_ms"] = int((time.time() - step_start_time) * 1000)+ log_entry["execution_time_ms"] = int(+ (time.time() - step_start_time) * 1000+ )results["results"][node_id] = {"error": error_msg}self._update_execution_log(log_entry)return results-+except Exception as e:error_msg = f"Error during scan: {str(e)}"- Logger.error(self.sketch_id, {"message":error_msg})+ Logger.error(self.sketch_id, {"message": error_msg})step_result["error"] = error_msglog_entry["status"] = "error"log_entry["error"] = error_msg- log_entry["execution_time_ms"] = int((time.time() - step_start_time) * 1000)+ log_entry["execution_time_ms"] = int(+ (time.time() - step_start_time) * 1000+ )results["results"][node_id] = {"error": error_msg}self._update_execution_log(log_entry)return results-+# Update execution log with this stepself._update_execution_log(log_entry)branch_results["steps"].append(step_result)-+results["branches"].append(branch_results)-- Logger.completed(self.sketch_id, {"message": "Transform completed successfully."})-++ Logger.completed(+ self.sketch_id, {"message": "Transform completed successfully."}+ )+# Include the final reference mapping for debuggingresults["reference_mapping"] = results_mapping-+# Finalize execution logself._finalize_execution_log(results)-+return results-\ No newline at end of filediff --git a/flowsint-core/src/flowsint_core/core/postgre_db.py b/flowsint-core/src/flowsint_core/core/postgre_db.pyindex bb4bf25e..516dea1a 100644--- a/flowsint-core/src/flowsint_core/core/postgre_db.py+++ b/flowsint-core/src/flowsint_core/core/postgre_db.py@@ -19,4 +19,4 @@ def get_db() -> Session:try:yield dbfinally:- db.close()\ No newline at end of file+ db.close()diff --git a/flowsint-core/src/flowsint_core/core/registry.py b/flowsint-core/src/flowsint_core/core/registry.pyindex 8ac501b8..eaa053bf 100644--- a/flowsint-core/src/flowsint_core/core/registry.py+++ b/flowsint-core/src/flowsint_core/core/registry.py@@ -1,61 +1,47 @@import inspect-from typing import Dict, Type, List, Optional-from .scanner_base import Scanner+from typing import Dict, Type, List+from flowsint_core.core.scanner_base import Scanner+from flowsint_transforms.domains.subdomains import SubdomainScanner+from flowsint_transforms.domains.whois import WhoisScanner+from flowsint_transforms.domains.resolve import ResolveScanner+from flowsint_transforms.ips.reverse_resolve import ReverseResolveScanner+from flowsint_transforms.ips.geolocation import GeolocationScanner+from flowsint_transforms.socials.maigret import MaigretScanner+from flowsint_transforms.ips.ip_to_asn import IpToAsnScanner+from flowsint_transforms.ips.asn_to_cidrs import AsnToCidrsScanner+from flowsint_transforms.ips.cidr_to_ips import CidrToIpsScanner+from flowsint_transforms.organizations.org_to_asn import OrgToAsnScanner+from flowsint_transforms.domains.domain_to_asn import DomainToAsnScanner+from flowsint_transforms.crypto.wallet_to_transactions import (+ CryptoWalletAddressToTransactions,+)+from flowsint_transforms.crypto.wallet_to_nfts import CryptoWalletAddressToNFTs+from flowsint_transforms.domains.to_website import DomainToWebsiteScanner+from flowsint_transforms.websites.to_crawler import WebsiteToCrawler+from flowsint_transforms.websites.to_links import WebsiteToLinks+from flowsint_transforms.websites.to_domain import WebsiteToDomainScanner+from flowsint_transforms.emails.to_gravatar import EmailToGravatarScanner+from flowsint_transforms.emails.to_leaks import EmailToBreachesScanner+from flowsint_transforms.individuals.to_org import IndividualToOrgScanner+from flowsint_transforms.organizations.to_infos import OrgToInfosScanner+from flowsint_transforms.websites.to_webtrackers import WebsiteToWebtrackersScanner+from flowsint_transforms.n8n.connector import N8nConnectorclass ScannerRegistry:- """- Registry for managing scanner discovery and instantiation.- Uses a dynamic discovery system to avoid circular dependencies.- """-+_scanners: Dict[str, Type[Scanner]] = {}- _scanner_modules: List[str] = []-+@classmethoddef register(cls, scanner_class: Type[Scanner]) -> None:- """Register a scanner class in the registry."""- if not issubclass(scanner_class, Scanner):- raise ValueError(f"Class {scanner_class.__name__} must inherit from Scanner")-- scanner_name = scanner_class.name()- cls._scanners[scanner_name] = scanner_class-- @classmethod- def register_module(cls, module_path: str) -> None:- """Register a module path for dynamic scanner discovery."""- if module_path not in cls._scanner_modules:- cls._scanner_modules.append(module_path)-- @classmethod- def discover_scanners(cls) -> None:- """Dynamically discover and register scanners from registered modules."""- import importlib- import pkgutil-- for module_path in cls._scanner_modules:- try:- module = importlib.import_module(module_path)-- # Walk through the module to find scanner classes- for name, obj in inspect.getmembers(module):- if (inspect.isclass(obj) and- issubclass(obj, Scanner) and- obj != Scanner and- hasattr(obj, 'name')):- cls.register(obj)-- except ImportError as e:- print(f"Warning: Could not import module {module_path}: {e}")-+ cls._scanners[scanner_class.name()] = scanner_class+@classmethoddef scanner_exists(cls, name: str) -> bool:- """Check if a scanner with the given name exists."""return name in cls._scanners-+@classmethoddef get_scanner(cls, name: str, sketch_id: str, scan_id: str, **kwargs) -> Scanner:- """Get a scanner instance by name."""if name not in cls._scanners:raise Exception(f"Scanner '{name}' not found")return cls._scanners[name](sketch_id=sketch_id, scan_id=scan_id, **kwargs)@@ -80,15 +66,13 @@ class ScannerRegistry:@classmethoddef list(cls) -> Dict[str, Dict[str, str]]:- """List all registered scanners with their metadata."""return {name: cls._create_scanner_metadata(scanner)for name, scanner in cls._scanners.items()}-+@classmethoddef list_by_categories(cls) -> Dict[str, List[Dict[str, str]]]:- """List scanners grouped by category."""scanners_by_category = {}for _, scanner in cls._scanners.items():category = scanner.category()@@ -96,23 +80,45 @@ class ScannerRegistry:scanners_by_category[category] = []scanners_by_category[category].append(cls._create_scanner_metadata(scanner))return scanners_by_category-+@classmethoddef list_by_input_type(cls, input_type: str) -> List[Dict[str, str]]:- """List scanners that accept a specific input type."""input_type_lower = input_type.lower()-+if input_type_lower == "any":- return [cls._create_scanner_metadata(scanner) for scanner in cls._scanners.values()]-+ return [+ cls._create_scanner_metadata(scanner)+ for scanner in cls._scanners.values()+ ]+return [cls._create_scanner_metadata(scanner)for scanner in cls._scanners.values()if scanner.input_schema()["type"].lower() in ["any", input_type_lower]]-- @classmethod- def clear(cls) -> None:- """Clear all registered scanners (mainly for testing)."""- cls._scanners.clear()- cls._scanner_modules.clear()+++# Register all scanners+ScannerRegistry.register(ReverseResolveScanner)+ScannerRegistry.register(ResolveScanner)+ScannerRegistry.register(SubdomainScanner)+ScannerRegistry.register(WhoisScanner)+ScannerRegistry.register(GeolocationScanner)+ScannerRegistry.register(MaigretScanner)+ScannerRegistry.register(IpToAsnScanner)+ScannerRegistry.register(AsnToCidrsScanner)+ScannerRegistry.register(CidrToIpsScanner)+ScannerRegistry.register(OrgToAsnScanner)+ScannerRegistry.register(DomainToAsnScanner)+ScannerRegistry.register(CryptoWalletAddressToTransactions)+ScannerRegistry.register(CryptoWalletAddressToNFTs)+ScannerRegistry.register(DomainToWebsiteScanner)+ScannerRegistry.register(WebsiteToCrawler)+ScannerRegistry.register(WebsiteToLinks)+ScannerRegistry.register(WebsiteToDomainScanner)+ScannerRegistry.register(EmailToGravatarScanner)+ScannerRegistry.register(EmailToBreachesScanner)+ScannerRegistry.register(IndividualToOrgScanner)+ScannerRegistry.register(OrgToInfosScanner)+ScannerRegistry.register(WebsiteToWebtrackersScanner)+ScannerRegistry.register(N8nConnector)diff --git a/flowsint-core/src/flowsint_core/core/scanner_base.py b/flowsint-core/src/flowsint_core/core/scanner_base.pyindex d8b1b56f..013fc53e 100644--- a/flowsint-core/src/flowsint_core/core/scanner_base.py+++ b/flowsint-core/src/flowsint_core/core/scanner_base.py@@ -7,9 +7,11 @@ from .logger import Loggerfrom .vault import VaultProtocolfrom ..utils import resolve_type+class InvalidScannerParams(Exception):pass+def build_params_model(params_schema: list) -> BaseModel:"""Build a strict Pydantic model from a params_schema.@@ -22,75 +24,75 @@ def build_params_model(params_schema: list) -> BaseModel:type = str # You can later enhance this to support int, bool, etc.required = param.get("required", False)default = ... if required else param.get("default")- fields[name] = (Optional[type], Field(default=default, description=param.get("description", "")))+ fields[name] = (+ Optional[type],+ Field(default=default, description=param.get("description", "")),+ )- model = create_model(- "ParamsModel",- __config__=ConfigDict(extra="forbid"),- **fields- )+ model = create_model("ParamsModel", __config__=ConfigDict(extra="forbid"), **fields)return model+class Scanner(ABC):"""Abstract base class for all scanners.-+## InputType and OutputType Pattern-+Scanners only need to define InputType and OutputType as class attributes.The base class automatically handles schema generation:-+```pythonfrom typing import Listfrom flowsint_types import Domainfrom flowsint_types import Ip-+class MyScanner(Scanner):# Define types as class attributesInputType = List[Domain]OutputType = List[Ip]-+@classmethoddef name(cls):return "my_scanner"-+@classmethoddef category(cls):return "Domain"-+@classmethoddef key(cls):return "domain"-+def preprocess(self, data: InputType) -> InputType:cleaned: InputType = []# ... implementationreturn cleaned-+async def scan(self, data: InputType) -> OutputType:results: OutputType = []# ... implementationreturn results-+# Make types available at module level for easy accessInputType = MyScanner.InputTypeOutputType = MyScanner.OutputType```-+The base class automatically provides:- input_schema() method using InputType- output_schema() method using OutputType- Error handling for missing type definitions- Consistent schema generation across all scanners-+Subclasses can override input_schema() or output_schema() if needed for special cases."""-+# Abstract type aliases that must be defined in subclasses for runtime useInputType = NotImplementedOutputType = NotImplemented-+def __init__(self,sketch_id: Optional[str] = None,@@ -98,7 +100,7 @@ class Scanner(ABC):neo4j_conn: Optional[Neo4jConnection] = None,params_schema: Optional[List[Dict[str, Any]]] = None,vault: Optional[VaultProtocol] = None,- params: Optional[Dict[str, Any]] = None+ params: Optional[Dict[str, Any]] = None,):self.scan_id = scan_id or "default"self.sketch_id = sketch_id or "system"@@ -107,7 +109,7 @@ class Scanner(ABC):self.params_schema = params_schema or []self.ParamsModel = build_params_model(self.params_schema)self.params: Dict[str, Any] = params or {}- # Params is filled synchronously by the constructor. This params is generally constructed of+ # Params is filled synchronously by the constructor. This params is generally constructed of# vaultSecret references, not the key directly. The idea is that the real key values are resolved after calling# async_init(), right before the execution.@@ -116,7 +118,9 @@ class Scanner(ABC):# Resolve parameters (e.g. replace vaultSecret by real secrets)if self.params:resolved_params = self.resolve_params()- Logger.debug(self.sketch_id, {"message": f"Resolved params: {str(resolved_params)}"})+ Logger.debug(+ self.sketch_id, {"message": f"Resolved params: {str(resolved_params)}"}+ )else:resolved_params = {}# Strict validation after resolution@@ -124,13 +128,20 @@ class Scanner(ABC):validated = self.ParamsModel(**resolved_params)self.params = validated.model_dump()except ValidationError as e:- raise InvalidScannerParams(f"Scanner '{self.name()}' received invalid parameters: {e}")+ raise InvalidScannerParams(+ f"Scanner '{self.name()}' received invalid parameters: {e}"+ )def resolve_params(self) -> Dict[str, Any]:resolved = {}- Logger.warn(self.sketch_id, {"message": f"Params schema: {str(self.params_schema)}"})+ Logger.warn(+ self.sketch_id, {"message": f"Params schema: {str(self.params_schema)}"}+ )Logger.warn(self.sketch_id, {"message": f"Params: {str(self.params)}"})- Logger.warn(self.sketch_id, {"message": f"Params schema length: {len(self.params_schema)}"})+ Logger.warn(+ self.sketch_id,+ {"message": f"Params schema length: {len(self.params_schema)}"},+ )i = 1for param in self.params_schema:Logger.warn(self.sketch_id, {"message": f"Param {i}: {str(param)}"})@@ -160,16 +171,15 @@ class Scanner(ABC):continuereturn resolved-@classmethoddef required_params(self) -> bool:- return False+ return False@classmethod@abstractmethoddef name(cls) -> str:pass-+@classmethoddef icon(cls) -> str | None:return None@@ -193,6 +203,7 @@ class Scanner(ABC):Falls back to cleaned docstring if not overridden."""import inspect+return inspect.cleandoc(cls.__doc__ or "No documentation available.")@classmethod@@ -215,21 +226,21 @@ class Scanner(ABC):Subclasses don't need to override this unless they have special requirements."""return cls.generate_output_schema()-+@classmethoddef generate_input_schema(cls) -> Dict[str, Any]:"""Helper method to generate input schema from InputType class attribute.-+Raises:NotImplementedError: If InputType is not defined in the subclass"""if cls.InputType is NotImplemented:raise NotImplementedError(f"InputType must be defined in {cls.__name__}")-+adapter = TypeAdapter(cls.InputType)schema = adapter.json_schema()-+# Handle different schema structuresif "$defs" in schema and schema["$defs"]:# Follow the $ref in items to get the correct type (not just the first one)@@ -241,35 +252,35 @@ class Scanner(ABC):else:# Fallback: get the first type definition (for backward compatibility)type_name, details = list(schema["$defs"].items())[0]-+return {"type": type_name,"properties": [{"name": prop, "type": resolve_type(info, schema)}for prop, info in details["properties"].items()- ]+ ],}else:# Handle simpler schemasreturn {"type": schema.get("title", "Any"),- "properties": [{"name": "value", "type": "object"}]+ "properties": [{"name": "value", "type": "object"}],}-- @classmethod++ @classmethoddef generate_output_schema(cls) -> Dict[str, Any]:"""Helper method to generate output schema from OutputType class attribute.-+Raises:NotImplementedError: If OutputType is not defined in the subclass"""if cls.OutputType is NotImplemented:raise NotImplementedError(f"OutputType must be defined in {cls.__name__}")-+adapter = TypeAdapter(cls.OutputType)schema = adapter.json_schema()-+# Handle different schema structuresif "$defs" in schema and schema["$defs"]:# Follow the $ref in items to get the correct type (not just the first one)@@ -281,19 +292,19 @@ class Scanner(ABC):else:# Fallback: get the first type definition (for backward compatibility)type_name, details = list(schema["$defs"].items())[0]-+return {"type": type_name,"properties": [{"name": prop, "type": resolve_type(info, schema)}for prop, info in details["properties"].items()- ]+ ],}else:# Handle simpler schemasreturn {"type": schema.get("title", "Any"),- "properties": [{"name": "value", "type": "object"}]+ "properties": [{"name": "value", "type": "object"}],}@abstractmethod@@ -309,7 +320,9 @@ class Scanner(ABC):def preprocess(self, values: List[str]) -> List[str]:return values- def postprocess(self, results: List[Dict[str, Any]], input_data: List[str] = None) -> List[Dict[str, Any]]:+ def postprocess(+ self, results: List[Dict[str, Any]], input_data: List[str] = None+ ) -> List[Dict[str, Any]]:return resultsasync def execute(self, values: List[str]) -> List[Dict[str, Any]]:@@ -322,51 +335,68 @@ class Scanner(ABC):processed = self.postprocess(results, preprocessed)if self.name() != "transform_orchestrator":- Logger.completed(self.sketch_id, {"message": f"Scanner {self.name()} finished."})+ Logger.completed(+ self.sketch_id, {"message": f"Scanner {self.name()} finished."}+ )return processedexcept Exception as e:if self.name() != "transform_orchestrator":- Logger.error(self.sketch_id, {"message": f"Scanner {self.name()} errored: '{str(e)}'."})+ Logger.error(+ self.sketch_id,+ {"message": f"Scanner {self.name()} errored: '{str(e)}'."},+ )return []- def create_node(self, node_type: str, key_prop: str, key_value: str, **properties) -> None:+ def create_node(+ self, node_type: str, key_prop: str, key_value: str, **properties+ ) -> None:"""Simple helper to create a single Neo4j node."""if not self.neo4j_conn:return-- properties['sketch_id'] = self.sketch_id- properties['label'] = properties.get('label', key_value)-++ properties["sketch_id"] = self.sketch_id+ properties["label"] = properties.get("label", key_value)+set_clauses = [f"n.{prop} = ${prop}" for prop in properties.keys()]params = {key_prop: key_value, **properties}-+query = f"""MERGE (n:{node_type} {{{key_prop}: ${key_prop}}})SET {', '.join(set_clauses)}"""self.neo4j_conn.query(query, params)-- def create_relationship(self, from_type: str, from_key: str, from_value: str,- to_type: str, to_key: str, to_value: str, rel_type: str) -> None:++ def create_relationship(+ self,+ from_type: str,+ from_key: str,+ from_value: str,+ to_type: str,+ to_key: str,+ to_value: str,+ rel_type: str,+ ) -> None:"""Simple helper to create a relationship between two nodes."""if not self.neo4j_conn:return-+query = f"""MATCH (from:{from_type} {{{from_key}: $from_value}})MATCH (to:{to_type} {{{to_key}: $to_value}})MERGE (from)-[:{rel_type} {{sketch_id: $sketch_id}}]->(to)"""-- self.neo4j_conn.query(query, {- 'from_value': from_value,- 'to_value': to_value,- 'sketch_id': self.sketch_id- })-++ self.neo4j_conn.query(+ query,+ {+ "from_value": from_value,+ "to_value": to_value,+ "sketch_id": self.sketch_id,+ },+ )+def log_graph_message(self, message: str) -> None:"""Simple helper to log a graph message."""Logger.graph_append(self.sketch_id, {"message": message})-diff --git a/flowsint-core/src/flowsint_core/core/types.py b/flowsint-core/src/flowsint_core/core/types.pyindex b221df47..c32c0fe7 100644--- a/flowsint-core/src/flowsint_core/core/types.py+++ b/flowsint-core/src/flowsint_core/core/types.py@@ -4,42 +4,89 @@ from datetime import datetimefrom typing import Dict, List, Any, Optional, Literalfrom .enums import EventLevel+class Event(BaseModel):id: str = Field(..., description="Unique identifier for the event")sketch_id: str = Field(..., description="ID of the sketch")type: EventLevel = Field(..., description="Type of event")payload: Dict[str, Any] = Field(..., description="Payload of the event")- created_at: datetime = Field(default_factory=datetime.now, description="Timestamp when the event was created")+ created_at: datetime = Field(+ default_factory=datetime.now, description="Timestamp when the event was created"+ )+class Node(BaseModel):"""Represents a node in a transformation flow with position and data."""+id: str = Field(..., description="Unique identifier for the node", title="Node ID")- data: Dict[str, Any] = Field(..., description="Data payload associated with the node", title="Node Data")- position: Optional[Dict[str, float]] = Field(None, description="X and Y coordinates for node positioning", title="Position")- type: Optional[str] = Field(None, description="Type of node (e.g., input, output, process)", title="Node Type")-+ data: Dict[str, Any] = Field(+ ..., description="Data payload associated with the node", title="Node Data"+ )+ position: Optional[Dict[str, float]] = Field(+ None, description="X and Y coordinates for node positioning", title="Position"+ )+ type: Optional[str] = Field(+ None,+ description="Type of node (e.g., input, output, process)",+ title="Node Type",+ )++class Edge(BaseModel):"""Represents an edge connecting two nodes in a transformation flow."""+id: str = Field(..., description="Unique identifier for the edge", title="Edge ID")- data: Optional[Dict[str, Any]] = Field(None, description="Data payload associated with the edge", title="Edge Data")+ data: Optional[Dict[str, Any]] = Field(+ None, description="Data payload associated with the edge", title="Edge Data"+ )source: str = Field(..., description="ID of the source node", title="Source Node")target: str = Field(..., description="ID of the target node", title="Target Node")- sourceHandle: Optional[str] = Field(None, description="Handle ID on the source node", title="Source Handle")- targetHandle: Optional[str] = Field(None, description="Handle ID on the target node", title="Target Handle")+ sourceHandle: Optional[str] = Field(+ None, description="Handle ID on the source node", title="Source Handle"+ )+ targetHandle: Optional[str] = Field(+ None, description="Handle ID on the target node", title="Target Handle"+ )+class FlowStep(BaseModel):"""Represents a single step in a transformation flow with execution status."""+nodeId: str = Field(..., description="ID of the associated node", title="Node ID")- params: Optional[Dict[str, Any]] = Field(None, description="Parameters for the step", title="Parameters")- type: Literal["type", "scanner"] = Field(..., description="Type of step - either type transformation or scanner", title="Step Type")- inputs: Dict[str, Any] = Field(..., description="Input data for this step", title="Inputs")- outputs: Dict[str, Any] = Field(..., description="Output data from this step", title="Outputs")- status: Literal["pending", "processing", "completed", "error"] = Field(..., description="Current execution status of the step", title="Status")- branchId: str = Field(..., description="ID of the branch this step belongs to", title="Branch ID")- depth: int = Field(..., description="Depth level of this step in the flow", title="Depth")+ params: Optional[Dict[str, Any]] = Field(+ None, description="Parameters for the step", title="Parameters"+ )+ type: Literal["type", "scanner"] = Field(+ ...,+ description="Type of step - either type transformation or scanner",+ title="Step Type",+ )+ inputs: Dict[str, Any] = Field(+ ..., description="Input data for this step", title="Inputs"+ )+ outputs: Dict[str, Any] = Field(+ ..., description="Output data from this step", title="Outputs"+ )+ status: Literal["pending", "processing", "completed", "error"] = Field(+ ..., description="Current execution status of the step", title="Status"+ )+ branchId: str = Field(+ ..., description="ID of the branch this step belongs to", title="Branch ID"+ )+ depth: int = Field(+ ..., description="Depth level of this step in the flow", title="Depth"+ )+class FlowBranch(BaseModel):"""Represents a branch containing a sequence of transformation steps."""- id: str = Field(..., description="Unique identifier for the branch", title="Branch ID")- name: str = Field(..., description="Human-readable name of the branch", title="Branch Name")- steps: List[FlowStep] = Field(..., description="List of steps in this branch", title="Steps")\ No newline at end of file++ id: str = Field(+ ..., description="Unique identifier for the branch", title="Branch ID"+ )+ name: str = Field(+ ..., description="Human-readable name of the branch", title="Branch Name"+ )+ steps: List[FlowStep] = Field(+ ..., description="List of steps in this branch", title="Steps"+ )diff --git a/flowsint-core/src/flowsint_core/core/vault.py b/flowsint-core/src/flowsint_core/core/vault.pyindex fad08029..10f51e91 100644--- a/flowsint-core/src/flowsint_core/core/vault.py+++ b/flowsint-core/src/flowsint_core/core/vault.py@@ -4,9 +4,10 @@ from sqlalchemy.orm import Sessionfrom sqlalchemy import selectfrom .models import Key+class VaultProtocol(Protocol):- def get_secret(self, vault_ref: str) -> Optional[str]:- ...+ def get_secret(self, vault_ref: str) -> Optional[str]: ...+class Vault(VaultProtocol):def __init__(self, db: Session, owner_id: uuid.UUID):diff --git a/flowsint-core/src/flowsint_core/tasks/event.py b/flowsint-core/src/flowsint_core/tasks/event.pyindex 767f1d9e..2a2a6bb3 100644--- a/flowsint-core/src/flowsint_core/tasks/event.py+++ b/flowsint-core/src/flowsint_core/tasks/event.py@@ -9,17 +9,17 @@ from ..core.types import Eventlogger = logging.getLogger(__name__)+@celery.task(name="emit_event")def emit_event_task(log_id: str, sketch_id: str, log_type: EventLevel, content: Dict):"""Celery task to emit a log event"""try:event = Event(- id=log_id,- sketch_id=sketch_id,- type=log_type,- payload=content+ id=log_id, sketch_id=sketch_id, type=log_type, payload=content).model_dump_json()- redis_client = redis.from_url(os.getenv("REDIS_URI", "redis://127.0.0.1:6379/0"))+ redis_client = redis.from_url(+ os.getenv("REDIS_URI", "redis://127.0.0.1:6379/0")+ )redis_client.publish(sketch_id, event)except Exception as e:raisediff --git a/flowsint-core/src/flowsint_core/tasks/transform.py b/flowsint-core/src/flowsint_core/tasks/transform.pyindex b81ed304..600f1fda 100644--- a/flowsint-core/src/flowsint_core/tasks/transform.py+++ b/flowsint-core/src/flowsint_core/tasks/transform.py@@ -14,10 +14,11 @@ from sqlalchemy.orm import Sessionfrom ..core.logger import Loggerfrom ..core.enums import EventLevelfrom flowsint_core.utils import to_json_serializable+load_dotenv()URI = os.getenv("NEO4J_URI_BOLT")-URI="bolt://localhost:7687"+URI = "bolt://localhost:7687"USERNAME = os.getenv("NEO4J_USERNAME")PASSWORD = os.getenv("NEO4J_PASSWORD")@@ -25,8 +26,15 @@ neo4j_connection = Neo4jConnection(URI, USERNAME, PASSWORD)db: Session = next(get_db())logger = Logger()+@celery.task(name="run_transform", bind=True)-def run_transform(self, transform_branches, values: List[str], sketch_id: str | None, owner_id: Optional[str] = None):+def run_transform(+ self,+ transform_branches,+ values: List[str],+ sketch_id: str | None,+ owner_id: Optional[str] = None,+):session = SessionLocal()try:@@ -49,7 +57,9 @@ def run_transform(self, transform_branches, values: List[str], sketch_id: str |try:vault = Vault(session, uuid.UUID(owner_id))except Exception as e:- Logger.error(sketch_id, {"message": f"Failed to create vault: {str(e)}"})+ Logger.error(+ sketch_id, {"message": f"Failed to create vault: {str(e)}"}+ )transform_branches = [FlowBranch(**branch) for branch in transform_branches]scanner = TransformOrchestrator(@@ -59,7 +69,7 @@ def run_transform(self, transform_branches, values: List[str], sketch_id: str |neo4j_conn=neo4j_connection,vault=vault,)-+# Use the synchronous scan method which internally handles the async operationsresults = scanner.scan(values=values)diff --git a/flowsint-core/src/flowsint_core/tests/fixtures/database.py b/flowsint-core/src/flowsint_core/tests/fixtures/database.pyindex 0bd300f7..dac66630 100644--- a/flowsint-core/src/flowsint_core/tests/fixtures/database.py+++ b/flowsint-core/src/flowsint_core/tests/fixtures/database.py@@ -5,8 +5,6 @@ from sqlalchemy import create_enginefrom sqlalchemy.orm import sessionmakerfrom app.models.models import Profile, Keyfrom flowsint_core.core.vault import Vault-import tempfile-import os@pytest.fixture(scope="function")@@ -14,17 +12,17 @@ def test_db_session():"""Create an in-memory SQLite database for testing"""# Create in-memory SQLite databaseengine = create_engine("sqlite:///:memory:", echo=False)-+# Only create specific tables we need for testing (to avoid ARRAY type issues)Profile.__table__.create(engine)Key.__table__.create(engine)-+# Create sessionTestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)session = TestingSessionLocal()-+yield session-+# Clean upsession.close()engine.dispose()@@ -45,13 +43,13 @@ def test_profile(test_db_session, test_owner_id):hashed_password="hashed_test_password",is_active=True,first_name="Test",- last_name="User"+ last_name="User",)-+test_db_session.add(profile)test_db_session.commit()test_db_session.refresh(profile)-+return profile@@ -62,13 +60,13 @@ def test_api_key(test_db_session, test_owner_id):id=uuid.uuid4(),name="test_etherscan_key",owner_id=test_owner_id,- encrypted_key="test_api_key_12345"+ encrypted_key="test_api_key_12345",)-+test_db_session.add(key)test_db_session.commit()test_db_session.refresh(key)-+return key@@ -90,46 +88,45 @@ def mock_logger():return logger[email protected](scope="function")[email protected](scope="function")def test_sketch_scan_ids():"""Generate test sketch and scan IDs"""- return {- "sketch_id": str(uuid.uuid4()),- "scan_id": str(uuid.uuid4())- }+ return {"sketch_id": str(uuid.uuid4()), "scan_id": str(uuid.uuid4())}@pytest.fixture(scope="function")def multiple_test_keys(test_db_session, test_owner_id):"""Create multiple test API keys for comprehensive testing"""keys = []-+# Create keys with different nameskey_configs = [{"name": "etherscan_api_key", "value": "test_etherscan_key_123"},{"name": "another_service_key", "value": "test_another_key_456"},- {"name": "vaultKey1", "value": "test_vault_key_789"}+ {"name": "vaultKey1", "value": "test_vault_key_789"},]-+for config in key_configs:key = Key(id=uuid.uuid4(),name=config["name"],owner_id=test_owner_id,- encrypted_key=config["value"]+ encrypted_key=config["value"],)test_db_session.add(key)keys.append(key)-+test_db_session.commit()-+for key in keys:test_db_session.refresh(key)-+return keys@pytest.fixture(scope="function")-def test_vault_with_multiple_keys(test_db_session, test_owner_id, test_profile, multiple_test_keys):+def test_vault_with_multiple_keys(+ test_db_session, test_owner_id, test_profile, multiple_test_keys+):"""Create a test vault with multiple keys for comprehensive testing"""- return Vault(db=test_db_session, owner_id=test_owner_id)\ No newline at end of file+ return Vault(db=test_db_session, owner_id=test_owner_id)diff --git a/flowsint-core/src/flowsint_core/tests/orchestrator.py b/flowsint-core/src/flowsint_core/tests/orchestrator.pyindex 5bf09abe..97deb6ed 100644--- a/flowsint-core/src/flowsint_core/tests/orchestrator.py+++ b/flowsint-core/src/flowsint_core/tests/orchestrator.py@@ -4,76 +4,108 @@ from flowsint_types.whois import Whoisimport pytest+def test_preprocess_valid_domains():- scanner = TransformOrchestrator("123",scanner_names= ["domain_resolve_scanner", "domain_whois_scanner"])+ scanner = TransformOrchestrator(+ "123", scanner_names=["domain_resolve_scanner", "domain_whois_scanner"]+ )assert isinstance(scanner.scanners, list)assert len(scanner.scanners) == 2-++def test_preprocess_valid_domains():- with pytest.raises(ValueError, match="Scanner 'this_scan_is_wrong' not found in registry"):- TransformOrchestrator("123",scanner_names= ["domain_resolve_scanner", "this_scan_is_wrong"])+ with pytest.raises(+ ValueError, match="Scanner 'this_scan_is_wrong' not found in registry"+ ):+ TransformOrchestrator(+ "123", scanner_names=["domain_resolve_scanner", "this_scan_is_wrong"]+ )+def test_execute_domain_subdomains_scanner():domains = ["example.com"]- scanner = TransformOrchestrator("123", scanner_names = ["domain_subdomains_scanner"])+ scanner = TransformOrchestrator("123", scanner_names=["domain_subdomains_scanner"])results = scanner.execute(values=domains)assert results["initial_values"] == domains- assert results['scanners'] == ['domain_subdomains_scanner']- assert isinstance(results['results']['domain_subdomains_scanner'], list)- assert isinstance(results['results']['domain_subdomains_scanner'][0], Domain)+ assert results["scanners"] == ["domain_subdomains_scanner"]+ assert isinstance(results["results"]["domain_subdomains_scanner"], list)+ assert isinstance(results["results"]["domain_subdomains_scanner"][0], Domain)def test_execute_domain_whois_scanner():domains = ["example.com"]- scanner = TransformOrchestrator("123", scanner_names = ["domain_whois_scanner"])+ scanner = TransformOrchestrator("123", scanner_names=["domain_whois_scanner"])results = scanner.execute(values=domains)assert results["initial_values"] == domains- assert results['scanners'] == ['domain_whois_scanner']- assert isinstance(results['results']['domain_whois_scanner'], list)- assert isinstance(results['results']['domain_whois_scanner'][0], Whois)-+ assert results["scanners"] == ["domain_whois_scanner"]+ assert isinstance(results["results"]["domain_whois_scanner"], list)+ assert isinstance(results["results"]["domain_whois_scanner"][0], Whois)++def test_execute_ip_resolve():- ips = ['91.199.212.73']- scanner = TransformOrchestrator("123", scanner_names = ["ip_reverse_resolve_scanner"])+ ips = ["91.199.212.73"]+ scanner = TransformOrchestrator("123", scanner_names=["ip_reverse_resolve_scanner"])results = scanner.execute(values=ips)assert results["initial_values"] == ips- assert results['scanners'] == ['ip_reverse_resolve_scanner']- assert isinstance(results['results']['ip_reverse_resolve_scanner'], list)- if(len(results['results']['ip_reverse_resolve_scanner'])!= 0):- assert isinstance(results['results']['ip_reverse_resolve_scanner'][0], Domain)-+ assert results["scanners"] == ["ip_reverse_resolve_scanner"]+ assert isinstance(results["results"]["ip_reverse_resolve_scanner"], list)+ if len(results["results"]["ip_reverse_resolve_scanner"]) != 0:+ assert isinstance(results["results"]["ip_reverse_resolve_scanner"][0], Domain)++def test_execute_ip_resolve_and_whois():- ips = ['91.199.212.73']- scanner = TransformOrchestrator("123", scanner_names = ["ip_reverse_resolve_scanner", "domain_whois_scanner"])+ ips = ["91.199.212.73"]+ scanner = TransformOrchestrator(+ "123", scanner_names=["ip_reverse_resolve_scanner", "domain_whois_scanner"]+ )results = scanner.execute(values=ips)assert results["initial_values"] == ips- assert results['scanners'] == ['ip_reverse_resolve_scanner', 'domain_whois_scanner']-+ assert results["scanners"] == ["ip_reverse_resolve_scanner", "domain_whois_scanner"]++def test_execute_ip_resolve_and_whois_multiple():- ips = ['91.199.212.73', '76.76.21.21']- scanner = TransformOrchestrator("123", scanner_names = ["ip_reverse_resolve_scanner", "domain_whois_scanner"])+ ips = ["91.199.212.73", "76.76.21.21"]+ scanner = TransformOrchestrator(+ "123", scanner_names=["ip_reverse_resolve_scanner", "domain_whois_scanner"]+ )results = scanner.execute(values=ips)assert results["initial_values"] == ips- assert results['scanners'] == ['ip_reverse_resolve_scanner', 'domain_whois_scanner']+ assert results["scanners"] == ["ip_reverse_resolve_scanner", "domain_whois_scanner"]print(results)+def test_execute_ip_resolve_and_whois_multiple():- ips = ['162.19.81.222']- scanner = TransformOrchestrator("123", scanner_names = ["ip_reverse_resolve_scanner", "domain_whois_scanner", "domain_subdomains_scanner"])+ ips = ["162.19.81.222"]+ scanner = TransformOrchestrator(+ "123",+ scanner_names=[+ "ip_reverse_resolve_scanner",+ "domain_whois_scanner",+ "domain_subdomains_scanner",+ ],+ )results = scanner.execute(values=ips)assert results["initial_values"] == ips- assert results['scanners'] == ['ip_reverse_resolve_scanner', 'domain_whois_scanner', "domain_subdomains_scanner"]-+ assert results["scanners"] == [+ "ip_reverse_resolve_scanner",+ "domain_whois_scanner",+ "domain_subdomains_scanner",+ ]++def test_execute_domain_whois_and_subdomains():domains = ["alliage.io"]- scanner = TransformOrchestrator("123", scanner_names = ["domain_whois_scanner", "domain_subdomains_scanner"])+ scanner = TransformOrchestrator(+ "123", scanner_names=["domain_whois_scanner", "domain_subdomains_scanner"]+ )results = scanner.execute(values=domains)assert results["initial_values"] == domains- assert results['scanners'] == ['domain_whois_scanner', "domain_subdomains_scanner"]+ assert results["scanners"] == ["domain_whois_scanner", "domain_subdomains_scanner"]+def test_execute_domain_subdomains():domains = ["alliage.io"]- scanner = TransformOrchestrator("123", scanner_names = ["domain_subdomains_scanner"])+ scanner = TransformOrchestrator("123", scanner_names=["domain_subdomains_scanner"])results = scanner.execute(values=domains)assert results["initial_values"] == domains- assert results['scanners'] == ["domain_subdomains_scanner"]+ assert results["scanners"] == ["domain_subdomains_scanner"]diff --git a/flowsint-core/src/flowsint_core/tests/test_base_scanner.py b/flowsint-core/src/flowsint_core/tests/test_base_scanner.pyindex 0eb04761..7d2078c3 100644--- a/flowsint-core/src/flowsint_core/tests/test_base_scanner.py+++ b/flowsint-core/src/flowsint_core/tests/test_base_scanner.py@@ -7,31 +7,31 @@ from flowsint_types.ip import Ipclass MockScanner(Scanner):"""Mock scanner for testing base functionality"""-+# Define InputType and OutputTypeInputType = List[Domain]OutputType = List[Ip]-+@classmethoddef name(cls) -> str:return "mock_scanner"-+@classmethoddef category(cls) -> str:return "Test"-+@classmethoddef key(cls) -> str:return "domain"-+@classmethoddef input_schema(cls) -> Dict[str, Any]:return cls.generate_input_schema()-+@classmethoddef output_schema(cls) -> Dict[str, Any]:return cls.generate_output_schema()-+async def scan(self, values: List[str]) -> List[Dict[str, Any]]:# Mock implementationreturn [{"address": "1.2.3.4"}]@@ -39,165 +39,165 @@ class MockScanner(Scanner):class IncompleteScanner(Scanner):"""Scanner without InputType/OutputType for testing error cases"""-+@classmethoddef name(cls) -> str:return "incomplete_scanner"-+@classmethoddef category(cls) -> str:return "Test"-+@classmethoddef key(cls) -> str:return "test"-+@classmethoddef input_schema(cls) -> Dict[str, Any]:return cls.generate_input_schema()-+@classmethoddef output_schema(cls) -> Dict[str, Any]:return cls.generate_output_schema()-+async def scan(self, values: List[str]) -> List[Dict[str, Any]]:return []class TestBaseScannerInputOutputTypes:"""Test suite for Scanner InputType/OutputType functionality"""-+def test_input_type_output_type_class_attributes(self):"""Test that InputType and OutputType are properly set as class attributes"""- assert hasattr(MockScanner, 'InputType')- assert hasattr(MockScanner, 'OutputType')+ assert hasattr(MockScanner, "InputType")+ assert hasattr(MockScanner, "OutputType")assert MockScanner.InputType == List[Domain]assert MockScanner.OutputType == List[Ip]-+def test_generate_input_schema_success(self):"""Test that generate_input_schema works correctly with valid InputType"""schema = MockScanner.generate_input_schema()-+assert isinstance(schema, dict)assert "type" in schemaassert "properties" in schemaassert schema["type"] == "Domain"-+# Check that properties are correctly extractedproperties = schema["properties"]assert isinstance(properties, list)-+# Should have domain propertydomain_prop = next((p for p in properties if p["name"] == "domain"), None)assert domain_prop is not Noneassert domain_prop["type"] == "string"-+def test_generate_output_schema_success(self):"""Test that generate_output_schema works correctly with valid OutputType"""schema = MockScanner.generate_output_schema()-+assert isinstance(schema, dict)assert "type" in schemaassert "properties" in schemaassert schema["type"] == "Ip"-+# Check that properties are correctly extractedproperties = schema["properties"]assert isinstance(properties, list)-+# Should have address propertyaddress_prop = next((p for p in properties if p["name"] == "address"), None)assert address_prop is not Noneassert address_prop["type"] == "string"-+def test_generate_input_schema_not_implemented_error(self):"""Test that generate_input_schema raises error when InputType is not defined"""with pytest.raises(NotImplementedError) as exc_info:IncompleteScanner.generate_input_schema()-+assert "InputType must be defined" in str(exc_info.value)assert "IncompleteScanner" in str(exc_info.value)-+def test_generate_output_schema_not_implemented_error(self):"""Test that generate_output_schema raises error when OutputType is not defined"""with pytest.raises(NotImplementedError) as exc_info:IncompleteScanner.generate_output_schema()-+assert "OutputType must be defined" in str(exc_info.value)assert "IncompleteScanner" in str(exc_info.value)-+def test_input_output_schema_methods_use_generate_methods(self):"""Test that the schema methods properly use the generate methods"""input_schema = MockScanner.input_schema()output_schema = MockScanner.output_schema()-+# These should be identical to calling generate methods directlyassert input_schema == MockScanner.generate_input_schema()assert output_schema == MockScanner.generate_output_schema()-+def test_base_scanner_has_not_implemented_defaults(self):"""Test that base Scanner class has NotImplemented defaults"""assert Scanner.InputType is NotImplementedassert Scanner.OutputType is NotImplemented-+def test_inheritance_preserves_input_output_types(self):"""Test that InputType and OutputType are properly inherited"""-+class ChildScanner(MockScanner):pass-+# Child should inherit the types from MockScannerassert ChildScanner.InputType == List[Domain]assert ChildScanner.OutputType == List[Ip]-+# And schema generation should workinput_schema = ChildScanner.generate_input_schema()output_schema = ChildScanner.generate_output_schema()-+assert input_schema["type"] == "Domain"assert output_schema["type"] == "Ip"-+def test_scanner_instance_can_access_class_types(self):"""Test that scanner instances can access InputType and OutputType"""scanner = MockScanner("test_sketch", "test_scan")-+assert scanner.InputType == List[Domain]assert scanner.OutputType == List[Ip]-+# Instance should be able to call generate methodsinput_schema = scanner.generate_input_schema()output_schema = scanner.generate_output_schema()-+assert input_schema["type"] == "Domain"assert output_schema["type"] == "Ip"class TestScannerFunctionality:"""Test other Scanner base functionality"""-+def test_scanner_initialization(self):"""Test that Scanner initializes correctly"""scanner = MockScanner("test_sketch", "test_scan")-+assert scanner.sketch_id == "test_sketch"assert scanner.scan_id == "test_scan"assert scanner.params == {}assert scanner.params_schema == []-+def test_scanner_initialization_with_defaults(self):"""Test Scanner initialization with default values"""scanner = MockScanner()-+assert scanner.sketch_id == "system"assert scanner.scan_id == "default"-+@pytest.mark.asyncioasync def test_scanner_execute_method(self):"""Test the execute method workflow"""scanner = MockScanner("test_sketch", "test_scan")-+result = await scanner.execute(["test.com"])-+assert isinstance(result, list)assert len(result) == 1- assert result[0]["address"] == "1.2.3.4"\ No newline at end of file+ assert result[0]["address"] == "1.2.3.4"diff --git a/flowsint-core/src/flowsint_core/tests/test_crypto_with_vault.py b/flowsint-core/src/flowsint_core/tests/test_crypto_with_vault.pyindex c226ea31..03b5551d 100644--- a/flowsint-core/src/flowsint_core/tests/test_crypto_with_vault.py+++ b/flowsint-core/src/flowsint_core/tests/test_crypto_with_vault.py@@ -1,12 +1,19 @@import pytestimport uuid-from flowsint_transforms.crypto.wallet_to_transactions import CryptoWalletAddressToTransactions+from flowsint_transforms.crypto.wallet_to_transactions import (+ CryptoWalletAddressToTransactions,+)from flowsint_transforms.crypto.wallet_to_nfts import CryptoWalletAddressToNFTsfrom ..core.vault import Vaultfrom .fixtures.database import (- test_db_session, test_owner_id, test_profile, test_api_key,- test_vault, test_vault_with_multiple_keys, multiple_test_keys,- test_sketch_scan_ids+ test_db_session,+ test_owner_id,+ test_profile,+ test_api_key,+ test_vault,+ test_vault_with_multiple_keys,+ multiple_test_keys,+ test_sketch_scan_ids,)@@ -28,7 +35,9 @@ class TestVaultIntegration:secret = test_vault_with_multiple_keys.get_secret("vaultKey1")assert secret == "test_vault_key_789"- def test_vault_get_secret_by_uuid(self, test_vault_with_multiple_keys, multiple_test_keys):+ def test_vault_get_secret_by_uuid(+ self, test_vault_with_multiple_keys, multiple_test_keys+ ):"""Test retrieving secret by key UUID"""# Get the UUID of the first keykey_uuid = str(multiple_test_keys[0].id)@@ -59,10 +68,10 @@ class TestCryptoWalletAddressToTransactions:vault=test_vault,params={"ETHERSCAN_API_KEY": "test_etherscan_key",- "ETHERSCAN_API_URL": "https://api.etherscan.io/api"- }+ "ETHERSCAN_API_URL": "https://api.etherscan.io/api",+ },)-+assert scanner.sketch_id == test_sketch_scan_ids["sketch_id"]assert scanner.scan_id == test_sketch_scan_ids["scan_id"]assert scanner.vault == test_vault@@ -74,14 +83,16 @@ class TestCryptoWalletAddressToTransactions:scan_id=test_sketch_scan_ids["scan_id"],params={"ETHERSCAN_API_KEY": "direct_api_key",- "ETHERSCAN_API_URL": "https://api.etherscan.io/api"- }+ "ETHERSCAN_API_URL": "https://api.etherscan.io/api",+ },)-+assert scanner.vault is None@pytest.mark.asyncio- async def test_async_init_with_vault_key_exists(self, test_vault_with_multiple_keys, test_sketch_scan_ids):+ async def test_async_init_with_vault_key_exists(+ self, test_vault_with_multiple_keys, test_sketch_scan_ids+ ):"""Test async_init when vault key exists"""scanner = CryptoWalletAddressToTransactions(sketch_id=test_sketch_scan_ids["sketch_id"],@@ -89,23 +100,25 @@ class TestCryptoWalletAddressToTransactions:vault=test_vault_with_multiple_keys,params={"ETHERSCAN_API_KEY": "vaultKey1", # This key exists in vault- "ETHERSCAN_API_URL": "https://api.etherscan.io/api"- }+ "ETHERSCAN_API_URL": "https://api.etherscan.io/api",+ },)-+# Get params before async_initparams_before = scanner.get_params()assert params_before["ETHERSCAN_API_KEY"] == "vaultKey1"-+# Initialize scannerawait scanner.async_init()-+# Get params after async_init - should be resolved from vaultparams_after = scanner.get_params()assert params_after["ETHERSCAN_API_KEY"] == "test_vault_key_789"@pytest.mark.asyncio- async def test_async_init_with_vault_key_not_exists(self, test_vault, test_sketch_scan_ids):+ async def test_async_init_with_vault_key_not_exists(+ self, test_vault, test_sketch_scan_ids+ ):"""Test async_init when vault key doesn't exist"""scanner = CryptoWalletAddressToTransactions(sketch_id=test_sketch_scan_ids["sketch_id"],@@ -113,17 +126,17 @@ class TestCryptoWalletAddressToTransactions:vault=test_vault,params={"ETHERSCAN_API_KEY": "nonexistent_key", # This key doesn't exist- "ETHERSCAN_API_URL": "https://api.etherscan.io/api"- }+ "ETHERSCAN_API_URL": "https://api.etherscan.io/api",+ },)-+# Get params before async_initparams_before = scanner.get_params()assert params_before["ETHERSCAN_API_KEY"] == "nonexistent_key"-+# Initialize scannerawait scanner.async_init()-+# Get params after async_init - should remain unchanged since key not foundparams_after = scanner.get_params()assert params_after["ETHERSCAN_API_KEY"] == "nonexistent_key"@@ -136,17 +149,17 @@ class TestCryptoWalletAddressToTransactions:scan_id=test_sketch_scan_ids["scan_id"],params={"ETHERSCAN_API_KEY": "direct_api_key",- "ETHERSCAN_API_URL": "https://api.etherscan.io/api"- }+ "ETHERSCAN_API_URL": "https://api.etherscan.io/api",+ },)-+# Get params before async_initparams_before = scanner.get_params()assert params_before["ETHERSCAN_API_KEY"] == "direct_api_key"-+# Initialize scannerawait scanner.async_init()-+# Get params after async_init - should remain unchangedparams_after = scanner.get_params()assert params_after["ETHERSCAN_API_KEY"] == "direct_api_key"@@ -169,16 +182,18 @@ class TestCryptoWalletAddressToNFTs:vault=test_vault,params={"ETHERSCAN_API_KEY": "test_etherscan_key",- "ETHERSCAN_API_URL": "https://api.etherscan.io/api"- }+ "ETHERSCAN_API_URL": "https://api.etherscan.io/api",+ },)-+assert scanner.sketch_id == test_sketch_scan_ids["sketch_id"]assert scanner.scan_id == test_sketch_scan_ids["scan_id"]assert scanner.vault == test_vault@pytest.mark.asyncio- async def test_async_init_with_vault_multiple_keys(self, test_vault_with_multiple_keys, test_sketch_scan_ids):+ async def test_async_init_with_vault_multiple_keys(+ self, test_vault_with_multiple_keys, test_sketch_scan_ids+ ):"""Test async_init with multiple vault keys"""scanner = CryptoWalletAddressToNFTs(sketch_id=test_sketch_scan_ids["sketch_id"],@@ -186,13 +201,13 @@ class TestCryptoWalletAddressToNFTs:vault=test_vault_with_multiple_keys,params={"ETHERSCAN_API_KEY": "etherscan_api_key", # This key exists in vault- "ETHERSCAN_API_URL": "https://api.etherscan.io/api"- }+ "ETHERSCAN_API_URL": "https://api.etherscan.io/api",+ },)-+# Initialize scannerawait scanner.async_init()-+# Get params after async_init - should be resolved from vaultparams_after = scanner.get_params()assert params_after["ETHERSCAN_API_KEY"] == "test_etherscan_key_123"@@ -208,7 +223,9 @@ class TestBothCryptoScannersIntegration:"""Test suite for testing both crypto scanners together"""@pytest.mark.asyncio- async def test_multiple_scanners_same_vault(self, test_vault_with_multiple_keys, test_sketch_scan_ids):+ async def test_multiple_scanners_same_vault(+ self, test_vault_with_multiple_keys, test_sketch_scan_ids+ ):"""Test multiple scanners using the same vault"""scanner1 = CryptoWalletAddressToTransactions(sketch_id=test_sketch_scan_ids["sketch_id"],@@ -216,33 +233,35 @@ class TestBothCryptoScannersIntegration:vault=test_vault_with_multiple_keys,params={"ETHERSCAN_API_KEY": "vaultKey1",- "ETHERSCAN_API_URL": "https://api.etherscan.io/api"- }+ "ETHERSCAN_API_URL": "https://api.etherscan.io/api",+ },)-+scanner2 = CryptoWalletAddressToNFTs(sketch_id=test_sketch_scan_ids["sketch_id"],scan_id=test_sketch_scan_ids["scan_id"],vault=test_vault_with_multiple_keys,params={"ETHERSCAN_API_KEY": "vaultKey1",- "ETHERSCAN_API_URL": "https://api.etherscan.io/api"- }+ "ETHERSCAN_API_URL": "https://api.etherscan.io/api",+ },)-+# Initialize both scannersawait scanner1.async_init()await scanner2.async_init()-+# Both should have resolved the same keyparams1 = scanner1.get_params()params2 = scanner2.get_params()-+assert params1["ETHERSCAN_API_KEY"] == "test_vault_key_789"assert params2["ETHERSCAN_API_KEY"] == "test_vault_key_789"@pytest.mark.asyncio- async def test_multiple_scanners_different_keys(self, test_vault_with_multiple_keys, test_sketch_scan_ids):+ async def test_multiple_scanners_different_keys(+ self, test_vault_with_multiple_keys, test_sketch_scan_ids+ ):"""Test multiple scanners using different vault keys"""scanner1 = CryptoWalletAddressToTransactions(sketch_id=test_sketch_scan_ids["sketch_id"],@@ -250,28 +269,28 @@ class TestBothCryptoScannersIntegration:vault=test_vault_with_multiple_keys,params={"ETHERSCAN_API_KEY": "vaultKey1",- "ETHERSCAN_API_URL": "https://api.etherscan.io/api"- }+ "ETHERSCAN_API_URL": "https://api.etherscan.io/api",+ },)-+scanner2 = CryptoWalletAddressToNFTs(sketch_id=test_sketch_scan_ids["sketch_id"],scan_id=test_sketch_scan_ids["scan_id"],vault=test_vault_with_multiple_keys,params={"ETHERSCAN_API_KEY": "etherscan_api_key",- "ETHERSCAN_API_URL": "https://api.etherscan.io/api"- }+ "ETHERSCAN_API_URL": "https://api.etherscan.io/api",+ },)-+# Initialize both scannersawait scanner1.async_init()await scanner2.async_init()-+# Should have resolved different keysparams1 = scanner1.get_params()params2 = scanner2.get_params()-+assert params1["ETHERSCAN_API_KEY"] == "test_vault_key_789"assert params2["ETHERSCAN_API_KEY"] == "test_etherscan_key_123"@@ -279,24 +298,28 @@ class TestBothCryptoScannersIntegration:"""Test that both crypto scanners have consistent schemas"""transactions_schema = CryptoWalletAddressToTransactions.input_schema()nfts_schema = CryptoWalletAddressToNFTs.input_schema()-+# Both should accept the same input type (crypto wallet)assert transactions_schema["type"] == nfts_schema["type"]-+# Both should have param schemas with ETHERSCAN_API_KEYtransactions_params = CryptoWalletAddressToTransactions.get_params_schema()nfts_params = CryptoWalletAddressToNFTs.get_params_schema()-+# Check that both have ETHERSCAN_API_KEY parametertransactions_api_key = next(- (param for param in transactions_params if param["name"] == "ETHERSCAN_API_KEY"),- None+ (+ param+ for param in transactions_params+ if param["name"] == "ETHERSCAN_API_KEY"+ ),+ None,)nfts_api_key = next(- (param for param in nfts_params if param["name"] == "ETHERSCAN_API_KEY"),- None+ (param for param in nfts_params if param["name"] == "ETHERSCAN_API_KEY"),+ None,)-+assert transactions_api_key is not Noneassert nfts_api_key is not None- assert transactions_api_key["required"] == nfts_api_key["required"]\ No newline at end of file+ assert transactions_api_key["required"] == nfts_api_key["required"]diff --git a/flowsint-core/src/flowsint_core/tests/test_input_output_migration.py b/flowsint-core/src/flowsint_core/tests/test_input_output_migration.pyindex 0143393b..62d223fd 100644--- a/flowsint-core/src/flowsint_core/tests/test_input_output_migration.py+++ b/flowsint-core/src/flowsint_core/tests/test_input_output_migration.py@@ -3,6 +3,7 @@ Test migration from old TypeAlias pattern to new InputType/OutputType class attrThis test demonstrates the benefits and proper usage of the new pattern."""+import pytestfrom typing import List, Dict, Any, TypeAliasfrom pydantic import TypeAdapter@@ -15,19 +16,19 @@ from app.utils import resolve_typeclass OldPatternScanner(Scanner):"""Example of old pattern using module-level TypeAlias"""-+@classmethoddef name(cls) -> str:return "old_pattern_scanner"-+@classmethoddef category(cls) -> str:return "Test"-+@classmethoddef key(cls) -> str:return "domain"-+@classmethoddef input_schema(cls) -> Dict[str, Any]:# Old pattern: manually defining TypeAlias and building schema@@ -40,9 +41,9 @@ class OldPatternScanner(Scanner):"properties": [{"name": prop, "type": resolve_type(info, schema)}for prop, info in details["properties"].items()- ]+ ],}-+@classmethoddef output_schema(cls) -> Dict[str, Any]:# Old pattern: manually defining TypeAlias and building schema@@ -55,34 +56,34 @@ class OldPatternScanner(Scanner):"properties": [{"name": prop, "type": resolve_type(info, schema)}for prop, info in details["properties"].items()- ]+ ],}-+async def scan(self, values: List[str]) -> List[Dict[str, Any]]:return [{"address": "1.2.3.4"}]class NewPatternScanner(Scanner):"""Example of new pattern using class attributes with automatic schema generation"""-+# New pattern: just define class attributes - base class handles the rest!InputType = List[Domain]OutputType = List[Ip]-+@classmethoddef name(cls) -> str:return "new_pattern_scanner"-+@classmethoddef category(cls) -> str:return "Test"-+@classmethoddef key(cls) -> str:return "domain"-+# No need to implement input_schema() or output_schema() - base class does it automatically!-+# Methods can use InputType/OutputType directly (once made available at module level)def preprocess(self, data: List[Domain]) -> List[Domain]:# Using concrete type for clarity in test, but would use InputType in real implementation@@ -91,7 +92,7 @@ class NewPatternScanner(Scanner):if isinstance(item, Domain):cleaned.append(item)return cleaned-+async def scan(self, values: List[Domain]) -> List[Ip]:# Using concrete type for clarity in test, but would use InputType/OutputType in real implementationresults: List[Ip] = []@@ -99,6 +100,7 @@ class NewPatternScanner(Scanner):results.append(Ip(address="1.2.3.4"))return results+# Make types available at module level (this is what enables clean usage)NewPatternInputType = NewPatternScanner.InputTypeNewPatternOutputType = NewPatternScanner.OutputType@@ -106,230 +108,262 @@ NewPatternOutputType = NewPatternScanner.OutputTypeclass AdvancedNewPatternScanner(Scanner):"""Example showing advanced usage with different types"""-+InputType = List[Email]OutputType = List[Domain]-+@classmethoddef name(cls) -> str:return "advanced_pattern_scanner"-+@classmethoddef category(cls) -> str:return "Test"-+@classmethoddef key(cls) -> str:return "email"-+# Schema methods automatic!-+async def scan(self, values: List[str]) -> List[Dict[str, Any]]:return [{"domain": "example.com"}]class TestInputOutputMigrationPattern:"""Test migration from old to new pattern"""-+def test_both_patterns_produce_same_schema(self):"""Test that old and new patterns produce identical schemas"""old_input_schema = OldPatternScanner.input_schema()new_input_schema = NewPatternScanner.input_schema()-+old_output_schema = OldPatternScanner.output_schema()new_output_schema = NewPatternScanner.output_schema()-+# Schemas should be identicalassert old_input_schema == new_input_schemaassert old_output_schema == new_output_schema-+def test_new_pattern_benefits_code_reuse(self):"""Test that new pattern reduces code duplication"""# With the new pattern, multiple scanners can easily reuse the same logic-+class Scanner1(Scanner):InputType = List[Domain]OutputType = List[Ip]-+@classmethod- def name(cls): return "scanner1"+ def name(cls):+ return "scanner1"+@classmethod- def category(cls): return "Test"+ def category(cls):+ return "Test"+@classmethod- def key(cls): return "domain"+ def key(cls):+ return "domain"+# No need for input_schema() or output_schema() - automatic!- async def scan(self, values): return []-+ async def scan(self, values):+ return []+class Scanner2(Scanner):InputType = List[Domain]OutputType = List[Ip]-+@classmethod- def name(cls): return "scanner2"+ def name(cls):+ return "scanner2"+@classmethod- def category(cls): return "Test"+ def category(cls):+ return "Test"+@classmethod- def key(cls): return "domain"+ def key(cls):+ return "domain"+# No need for input_schema() or output_schema() - automatic!- async def scan(self, values): return []-+ async def scan(self, values):+ return []+# Both should produce identical schemas with minimal codeassert Scanner1.input_schema() == Scanner2.input_schema()assert Scanner1.output_schema() == Scanner2.output_schema()-+def test_new_pattern_type_introspection(self):"""Test that new pattern allows for better type introspection"""# Can easily check what types a scanner usesassert NewPatternScanner.InputType == List[Domain]assert NewPatternScanner.OutputType == List[Ip]-+assert AdvancedNewPatternScanner.InputType == List[Email]assert AdvancedNewPatternScanner.OutputType == List[Domain]-+# This wasn't easily possible with the old pattern-+def test_new_pattern_inheritance_works(self):"""Test that new pattern works well with inheritance"""-+class BaseDomainScanner(Scanner):InputType = List[Domain]OutputType = List[Ip]-+@classmethod- def name(cls): return "base_domain"+ def name(cls):+ return "base_domain"+@classmethod- def category(cls): return "Test"+ def category(cls):+ return "Test"+@classmethod- def key(cls): return "domain"+ def key(cls):+ return "domain"+# Schema methods automatic!- async def scan(self, values): return []-+ async def scan(self, values):+ return []+class SpecializedDomainScanner(BaseDomainScanner):@classmethod- def name(cls): return "specialized_domain"+ def name(cls):+ return "specialized_domain"+# Inherits InputType and OutputType-+# Child should inherit the types and schemasassert SpecializedDomainScanner.InputType == List[Domain]assert SpecializedDomainScanner.OutputType == List[Ip]-+specialized_input = SpecializedDomainScanner.input_schema()specialized_output = SpecializedDomainScanner.output_schema()-+assert specialized_input["type"] == "Domain"assert specialized_output["type"] == "Ip"-+def test_new_pattern_error_handling(self):"""Test that new pattern provides better error handling"""-+class IncompleteDomainScanner(Scanner):# Forgot to define InputType and OutputType@classmethod- def name(cls): return "incomplete"+ def name(cls):+ return "incomplete"+@classmethod- def category(cls): return "Test"+ def category(cls):+ return "Test"+@classmethod- def key(cls): return "domain"+ def key(cls):+ return "domain"+# Base class will try to generate schemas automatically and fail appropriately- async def scan(self, values): return []-+ async def scan(self, values):+ return []+# Should get clear error messageswith pytest.raises(NotImplementedError) as exc_info:IncompleteDomainScanner.input_schema()assert "InputType must be defined" in str(exc_info.value)assert "IncompleteDomainScanner" in str(exc_info.value)-+with pytest.raises(NotImplementedError) as exc_info:IncompleteDomainScanner.output_schema()assert "OutputType must be defined" in str(exc_info.value)assert "IncompleteDomainScanner" in str(exc_info.value)-+def test_new_pattern_runtime_accessibility(self):"""Test that types are accessible at runtime for dynamic operations"""-+# Can build registries or perform operations based on typesscanners = [NewPatternScanner, AdvancedNewPatternScanner]-+domain_input_scanners = [- scanner for scanner in scanners- if hasattr(scanner, 'InputType') and scanner.InputType == List[Domain]+ scanner+ for scanner in scanners+ if hasattr(scanner, "InputType") and scanner.InputType == List[Domain]]-+email_input_scanners = [- scanner for scanner in scanners- if hasattr(scanner, 'InputType') and scanner.InputType == List[Email]+ scanner+ for scanner in scanners+ if hasattr(scanner, "InputType") and scanner.InputType == List[Email]]-+assert len(domain_input_scanners) == 1assert domain_input_scanners[0] == NewPatternScanner-+assert len(email_input_scanners) == 1assert email_input_scanners[0] == AdvancedNewPatternScanner-+def test_new_pattern_with_clean_type_usage(self):"""Test that the new pattern allows clean type usage without quotes"""scanner = NewPatternScanner("test", "test")-+# Test that we can create data of the expected typestest_domains = [Domain(domain="example.com"), Domain(domain="test.com")]-+# Preprocess should work with clean type annotationsresult = scanner.preprocess(test_domains)assert len(result) == 2assert all(isinstance(d, Domain) for d in result)-+@pytest.mark.asyncioasync def test_new_pattern_async_methods(self):"""Test that async methods work correctly with clean type annotations"""scanner = NewPatternScanner("test", "test")-+test_domains = [Domain(domain="example.com")]result = await scanner.scan(test_domains)-+assert len(result) == 1assert isinstance(result[0], Ip)assert result[0].address == "1.2.3.4"-+def test_module_level_type_access(self):"""Test that types are properly accessible at module level"""# These should be available after the class definitionassert NewPatternInputType == List[Domain]assert NewPatternOutputType == List[Ip]-+# And they should match the class attributesassert NewPatternInputType == NewPatternScanner.InputTypeassert NewPatternOutputType == NewPatternScanner.OutputType-+def test_migration_checklist(self):"""Test that demonstrates a complete migration checklist"""-+# Migration steps:# 1. Define InputType and OutputType as class attributes# 2. Remove input_schema() and output_schema() method implementations (base class handles automatically)# 3. Add module-level assignments: InputType = MyScanner.InputType (optional, for clean usage)-+# Verify the new pattern is simpler and more maintainablenew_scanner_benefits = ["Just define InputType and OutputType class attributes",- "Automatic schema generation by base class",+ "Automatic schema generation by base class","No boilerplate schema methods needed","Consistent schema generation across all scanners",- "Clean type usage throughout class methods"+ "Clean type usage throughout class methods",]-+assert len(new_scanner_benefits) == 5-+# Verify functionality is preserved and automaticschema = NewPatternScanner.input_schema()assert schema["type"] == "Domain"assert any(prop["name"] == "domain" for prop in schema["properties"])-+# Verify schemas are generated automatically without manual implementation- assert hasattr(NewPatternScanner, 'input_schema')- assert hasattr(NewPatternScanner, 'output_schema')-+ assert hasattr(NewPatternScanner, "input_schema")+ assert hasattr(NewPatternScanner, "output_schema")+# The base class should be handling the schema generationinput_schema = NewPatternScanner.input_schema()output_schema = NewPatternScanner.output_schema()assert input_schema is not None- assert output_schema is not None\ No newline at end of file+ assert output_schema is not Nonediff --git a/flowsint-core/src/flowsint_core/tests/test_registry.py b/flowsint-core/src/flowsint_core/tests/test_registry.pyindex 5388ef1c..ef0c7a7a 100644--- a/flowsint-core/src/flowsint_core/tests/test_registry.py+++ b/flowsint-core/src/flowsint_core/tests/test_registry.py@@ -5,40 +5,41 @@ from flowsint_types.domain import Domainfrom flowsint_types.ip import Ipfrom typing import List+class MockScanner(Scanner):"""Mock scanner for testing registry functionality"""-+InputType = List[Domain]OutputType = List[Ip]-+@classmethoddef name(cls) -> str:return "mock_scanner"-+@classmethoddef category(cls) -> str:return "Test"-+@classmethoddef key(cls) -> str:return "domain"-+@classmethoddef documentation(cls) -> str:return "Mock scanner for testing"-+@classmethoddef get_params_schema(cls) -> List[dict]:return []-+@classmethoddef required_params(cls) -> bool:return False-+@classmethoddef icon(cls) -> str:return "test-icon"-+async def scan(self, data: InputType) -> OutputType:return []@@ -54,29 +55,28 @@ class TestScannerRegistry:"""Test registering a scanner"""ScannerRegistry.register(MockScanner)assert ScannerRegistry.scanner_exists("mock_scanner")-+scanners = ScannerRegistry.list()assert "mock_scanner" in scannersassert scanners["mock_scanner"]["class_name"] == "MockScanner"def test_register_invalid_scanner(self):"""Test that registering a non-scanner class raises error"""+class NotAScanner:pass-+with pytest.raises(ValueError, match="must inherit from Scanner"):ScannerRegistry.register(NotAScanner)def test_get_scanner_instance(self):"""Test getting a scanner instance"""ScannerRegistry.register(MockScanner)-+scanner = ScannerRegistry.get_scanner(- "mock_scanner",- sketch_id="test_sketch",- scan_id="test_scan"+ "mock_scanner", sketch_id="test_sketch", scan_id="test_scan")-+assert isinstance(scanner, MockScanner)assert scanner.sketch_id == "test_sketch"assert scanner.scan_id == "test_scan"@@ -85,15 +85,13 @@ class TestScannerRegistry:"""Test that getting a non-existent scanner raises exception"""with pytest.raises(Exception, match="Scanner 'nonexistent' not found"):ScannerRegistry.get_scanner(- "nonexistent",- sketch_id="test_sketch",- scan_id="test_scan"+ "nonexistent", sketch_id="test_sketch", scan_id="test_scan")def test_list_by_categories(self):"""Test listing scanners by category"""ScannerRegistry.register(MockScanner)-+by_category = ScannerRegistry.list_by_categories()assert "Test" in by_categoryassert len(by_category["Test"]) == 1@@ -102,11 +100,11 @@ class TestScannerRegistry:def test_list_by_input_type(self):"""Test listing scanners by input type"""ScannerRegistry.register(MockScanner)-+domain_scanners = ScannerRegistry.list_by_input_type("Domain")assert len(domain_scanners) == 1assert domain_scanners[0]["name"] == "mock_scanner"-+# Test with "any" input typeany_scanners = ScannerRegistry.list_by_input_type("any")assert len(any_scanners) == 1@@ -116,7 +114,7 @@ class TestScannerRegistry:"""Test registering a module path"""ScannerRegistry.register_module("test.module.path")assert "test.module.path" in ScannerRegistry._scanner_modules-+# Registering the same module again should not duplicateScannerRegistry.register_module("test.module.path")assert ScannerRegistry._scanner_modules.count("test.module.path") == 1diff --git a/flowsint-core/src/flowsint_core/tests/test_schema_generation.py b/flowsint-core/src/flowsint_core/tests/test_schema_generation.pyindex cb2bcd8b..bb09fd8b 100644--- a/flowsint-core/src/flowsint_core/tests/test_schema_generation.py+++ b/flowsint-core/src/flowsint_core/tests/test_schema_generation.py@@ -1,6 +1,7 @@"""Test schema generation for scanners with various InputType and OutputType combinations."""+from typing import Listfrom flowsint_types.website import Websitefrom flowsint_types.domain import Domain@@ -12,90 +13,107 @@ from flowsint_transforms.domains.to_website import DomainToWebsiteScannerclass TestSchemaGeneration:"""Test that schema generation correctly identifies InputType and OutputType."""-+def test_website_to_domain_scanner_schemas(self):"""Test that WebsiteToDomainScanner correctly shows Website as input type."""scanner = WebsiteToDomainScanner-+# Test InputType attributeassert scanner.InputType == List[Website]-+# Test input schema generationinput_schema = scanner.input_schema()- assert input_schema["type"] == "Website", f"Expected 'Website', got '{input_schema['type']}'"-- # Test output schema generation+ assert (+ input_schema["type"] == "Website"+ ), f"Expected 'Website', got '{input_schema['type']}'"++ # Test output schema generationoutput_schema = scanner.output_schema()- assert output_schema["type"] == "Domain", f"Expected 'Domain', got '{output_schema['type']}'"-+ assert (+ output_schema["type"] == "Domain"+ ), f"Expected 'Domain', got '{output_schema['type']}'"+def test_website_to_webtrackers_scanner_schemas(self):"""Test that WebsiteToWebtrackersScanner correctly shows Website as input type."""scanner = WebsiteToWebtrackersScanner-+# Test InputType attributeassert scanner.InputType == List[Website]-+# Test input schema generationinput_schema = scanner.input_schema()- assert input_schema["type"] == "Website", f"Expected 'Website', got '{input_schema['type']}'"-+ assert (+ input_schema["type"] == "Website"+ ), f"Expected 'Website', got '{input_schema['type']}'"+# Test output schema generationoutput_schema = scanner.output_schema()- assert output_schema["type"] == "WebTracker", f"Expected 'WebTracker', got '{output_schema['type']}'"-+ assert (+ output_schema["type"] == "WebTracker"+ ), f"Expected 'WebTracker', got '{output_schema['type']}'"+def test_website_to_crawler_scanner_schemas(self):"""Test that WebsiteToCrawler correctly shows Website as input type."""scanner = WebsiteToCrawler-+# Test InputType attributeassert scanner.InputType == List[Website]-+# Test input schema generationinput_schema = scanner.input_schema()- assert input_schema["type"] == "Website", f"Expected 'Website', got '{input_schema['type']}'"-+ assert (+ input_schema["type"] == "Website"+ ), f"Expected 'Website', got '{input_schema['type']}'"+def test_domain_to_website_scanner_schemas(self):"""Test that DomainToWebsiteScanner correctly shows Domain as input and Website as output."""scanner = DomainToWebsiteScanner-+# Test InputType attributeassert scanner.InputType == List[Domain]-+# Test input schema generationinput_schema = scanner.input_schema()- assert input_schema["type"] == "Domain", f"Expected 'Domain', got '{input_schema['type']}'"-+ assert (+ input_schema["type"] == "Domain"+ ), f"Expected 'Domain', got '{input_schema['type']}'"+# Test output schema generationoutput_schema = scanner.output_schema()- assert output_schema["type"] == "Website", f"Expected 'Website', got '{output_schema['type']}'"-+ assert (+ output_schema["type"] == "Website"+ ), f"Expected 'Website', got '{output_schema['type']}'"+def test_all_website_scanners_have_correct_input_types(self):"""Test that all scanners taking Website input show Website in schema."""website_input_scanners = [(WebsiteToDomainScanner, "Website", "Domain"),- (WebsiteToWebtrackersScanner, "Website", "WebTracker"),+ (WebsiteToWebtrackersScanner, "Website", "WebTracker"),(WebsiteToCrawler, "Website", None), # Unknown output type]-+for scanner_class, expected_input, expected_output in website_input_scanners:input_schema = scanner_class.input_schema()- assert input_schema["type"] == expected_input, \- f"{scanner_class.__name__}: Expected input '{expected_input}', got '{input_schema['type']}'"-+ assert (+ input_schema["type"] == expected_input+ ), f"{scanner_class.__name__}: Expected input '{expected_input}', got '{input_schema['type']}'"+if expected_output:output_schema = scanner_class.output_schema()- assert output_schema["type"] == expected_output, \- f"{scanner_class.__name__}: Expected output '{expected_output}', got '{output_schema['type']}'"+ assert (+ output_schema["type"] == expected_output+ ), f"{scanner_class.__name__}: Expected output '{expected_output}', got '{output_schema['type']}'"def test_schema_generation_debug_info(self):"""Debug test to see what's actually in the schemas."""scanner = WebsiteToDomainScanner-+# Get the raw TypeAdapter schemafrom pydantic import TypeAdapter+adapter = TypeAdapter(scanner.InputType)raw_schema = adapter.json_schema()-+print(f"\n=== Debug Info for {scanner.__name__} ===")print(f"InputType: {scanner.InputType}")print(f"Raw schema keys: {list(raw_schema.keys())}")@@ -103,43 +121,45 @@ class TestSchemaGeneration:print(f"$defs keys: {list(raw_schema['$defs'].keys())}")print(f"Schema items: {raw_schema.get('items', 'No items')}")print(f"Generated input schema: {scanner.input_schema()}")-+# This test always passes, it's just for debuggingassert True-+def test_schema_generation_follows_ref_not_first_def(self):"""Regression test for the schema generation bug.-+Before the fix: generate_input_schema() picked the first definition in $defs(alphabetically "Domain" came before "Website"), so Website scanners incorrectlyshowed "Domain" as their input type.-- After the fix: generate_input_schema() follows the $ref in items to get the++ After the fix: generate_input_schema() follows the $ref in items to get thecorrect type name."""scanner = WebsiteToDomainScanner-+# Get the raw schema to understand the structurefrom pydantic import TypeAdapter+adapter = TypeAdapter(scanner.InputType)raw_schema = adapter.json_schema()-+# Verify the raw schema structure that caused the bugassert "$defs" in raw_schemaassert "Domain" in raw_schema["$defs"]assert "Website" in raw_schema["$defs"]assert raw_schema["items"]["$ref"] == "#/$defs/Website"-+# Before fix: list(raw_schema["$defs"].items())[0][0] would be "Domain" (first alphabetically)first_def_name = list(raw_schema["$defs"].items())[0][0]assert first_def_name == "Domain" # This would have been the bug-+# After fix: We follow the $ref to get "Website"ref_type = raw_schema["items"]["$ref"].split("/")[-1]assert ref_type == "Website" # This is what we should use-+# Verify our fix works correctlyinput_schema = scanner.input_schema()- assert input_schema["type"] == "Website", \- f"Schema generation should follow $ref, not pick first def. Got '{input_schema['type']}'"\ No newline at end of file+ assert (+ input_schema["type"] == "Website"+ ), f"Schema generation should follow $ref, not pick first def. Got '{input_schema['type']}'"diff --git a/flowsint-core/src/flowsint_core/utils.py b/flowsint-core/src/flowsint_core/utils.pyindex b1e02a8e..238eed7c 100644--- a/flowsint-core/src/flowsint_core/utils.py+++ b/flowsint-core/src/flowsint_core/utils.py@@ -13,6 +13,7 @@ import inspectfrom typing import Any, Dict, Typefrom pydantic import BaseModel, TypeAdapter+def is_valid_ip(address: str) -> bool:try:ipaddress.ip_address(address)@@ -20,11 +21,13 @@ def is_valid_ip(address: str) -> bool:except ValueError:return False+def is_valid_username(username: str) -> bool:if not re.match(r"^[a-zA-Z0-9_-]{3,30}$", username):return Falsereturn True+def is_valid_email(email: str) -> bool:if not re.match(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$", email):return False@@ -34,26 +37,29 @@ def is_valid_email(email: str) -> bool:def is_valid_domain(url_or_domain: str) -> str:try:- parsed = urlparse(url_or_domain if "://" in url_or_domain else "http://" + url_or_domain)+ parsed = urlparse(+ url_or_domain if "://" in url_or_domain else "http://" + url_or_domain+ )hostname = parsed.hostname or url_or_domain-+if not hostname or "." not in hostname:return False-+if not re.match(r"^[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$", hostname):return False-+return Trueexcept Exception as e:return False+def is_root_domain(domain: str) -> bool:"""Determine if a domain is a root domain or subdomain.-+Args:domain: The domain string to check-+Returns:True if it's a root domain (e.g., example.com), False if it's a subdomain (e.g., sub.example.com)"""@@ -62,20 +68,32 @@ def is_root_domain(domain: str) -> bool:if "://" in domain:parsed = urlparse(domain)domain = parsed.hostname or domain-+# Split by dots- parts = domain.split('.')-+ parts = domain.split(".")+# Handle common country code TLDs that have 2 parts (e.g., .co.uk, .com.au, .org.uk)- common_cc_tlds = ['.co.uk', '.com.au', '.org.uk', '.net.uk', '.gov.uk', '.ac.uk',- '.co.nz', '.com.sg', '.co.jp', '.co.kr', '.com.br', '.com.mx']-+ common_cc_tlds = [+ ".co.uk",+ ".com.au",+ ".org.uk",+ ".net.uk",+ ".gov.uk",+ ".ac.uk",+ ".co.nz",+ ".com.sg",+ ".co.jp",+ ".co.kr",+ ".com.br",+ ".com.mx",+ ]+# Check if the domain ends with a common country code TLDfor cc_tld in common_cc_tlds:if domain.endswith(cc_tld):# For country code TLDs, we need exactly 3 parts (e.g., example.co.uk)return len(parts) == 3-+# For regular TLDs, a root domain has 2 parts (e.g., example.com)# A subdomain has 3 or more parts (e.g., sub.example.com, www.sub.example.com)return len(parts) == 2@@ -83,6 +101,7 @@ def is_root_domain(domain: str) -> bool:# If we can't parse it, assume it's not a root domainreturn False+def is_valid_number(phone: str, region: str = "FR") -> None:"""Validates a phone number. Raises InvalidPhoneNumberError if invalid.@@ -94,19 +113,21 @@ def is_valid_number(phone: str, region: str = "FR") -> None:return Falseexcept NumberParseException:return False-++def parse_asn(asn: str) -> int:if not is_valid_asn(asn):raise ValueError(f"Invalid ASN format: {asn}")return int(re.sub(r"(?i)^AS", "", asn))-+def is_valid_asn(asn: str) -> bool:if not re.fullmatch(r"(AS)?\d+", asn, re.IGNORECASE):return Falseasn_num = int(re.sub(r"(?i)^AS", "", asn))return 0 <= asn_num <= 4294967295+def resolve_type(details: dict, schema_context: dict = None) -> str:if "anyOf" in details:types = []@@ -121,22 +142,23 @@ def resolve_type(details: dict, schema_context: dict = None) -> str:else:types.append(option.get("type", "unknown"))return " | ".join(types)-+if "type" in details:if details["type"] == "array":item_type = resolve_type(details.get("items", {}), schema_context)return f"{item_type}[]"return details["type"]-+# Handle $ref in array items or other contextsif "$ref" in details and schema_context:ref_path = details["$ref"]if ref_path.startswith("#/$defs/"):ref_name = ref_path.split("/")[-1]return ref_name-+return "any"+def extract_input_schema_transform(model: Type[BaseModel]) -> Dict[str, Any]:adapter = TypeAdapter(model)schema = adapter.json_schema()@@ -153,37 +175,30 @@ def extract_input_schema_transform(model: Type[BaseModel]) -> Dict[str, Any]:"outputs": {"type": type_name,"properties": [- {- "name": prop,- "type": resolve_type(info, schema)- }+ {"name": prop, "type": resolve_type(info, schema)}for prop, info in details.get("properties", {}).items()- ]- },- "inputs": {- "type": "",- "properties": []+ ],},+ "inputs": {"type": "", "properties": []},"type": "type","category": model.__name__,}-def get_domain_from_ssl(ip: str, port: int = 443) -> str | None:try:context = ssl.create_default_context()with socket.create_connection((ip, port), timeout=3) as sock:with context.wrap_socket(sock, server_hostname=ip) as ssock:cert = ssock.getpeercert()- subject = cert.get('subject', [])+ subject = cert.get("subject", [])for entry in subject:- if entry[0][0] == 'commonName':+ if entry[0][0] == "commonName":return entry[0][1]# Alternative: check subjectAltName- san = cert.get('subjectAltName', [])+ san = cert.get("subjectAltName", [])for typ, val in san:- if typ == 'DNS':+ if typ == "DNS":return valexcept Exception as e:print(f"SSL extraction failed for {ip}: {e}")@@ -208,12 +223,14 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:scanner_node = node_lookup.get(target_id)if scanner_node and scanner_node["data"]["type"] == "scanner":- scanners.append({- "scanner_name": scanner_node["data"]["name"],- "module": scanner_node["data"]["module"],- "input": source_handle,- "output": target_handle- })+ scanners.append(+ {+ "scanner_name": scanner_node["data"]["name"],+ "module": scanner_node["data"]["module"],+ "input": source_handle,+ "output": target_handle,+ }+ )return {"input": {@@ -221,25 +238,22 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:"outputs": input_output,},"scanners": scanners,- "scanner_names" : [scanner["scanner_name"] for scanner in scanners]-+ "scanner_names": [scanner["scanner_name"] for scanner in scanners],}-++def get_label_color(label: str) -> str:- color_map = {- 'subdomain': '#A5ABB6',- 'domain': '#68BDF6',- 'default': '#A5ABB6'- }-+ color_map = {"subdomain": "#A5ABB6", "domain": "#68BDF6", "default": "#A5ABB6"}+return color_map.get(label, color_map["default"])+def flatten(data_dict, prefix=""):"""Flattens a dictionary to contain only Neo4j-compatible property values.Neo4j supports primitive types (string, number, boolean) and arrays of those types.Args:- data_dict (dict): Dictionary to flatten+ data_dict (dict): Dictionary to flattenReturns:dict: Flattened dictionary with only Neo4j-compatible values"""@@ -250,13 +264,14 @@ def flatten(data_dict, prefix=""):if value is None:continueif isinstance(value, (str, int, float, bool)) or (- isinstance(value, list) and all(isinstance(item, (str, int, float, bool)) for item in value)+ isinstance(value, list)+ and all(isinstance(item, (str, int, float, bool)) for item in value)):key = f"{prefix}{key}"- flattened[key] = value+ flattened[key] = valuereturn flattened-+def get_inline_relationships(nodes: List[Any], edges: List[Any]) -> List[str]:"""Get the inline relationships for a list of nodes and edges.@@ -266,14 +281,15 @@ def get_inline_relationships(nodes: List[Any], edges: List[Any]) -> List[str]:source = next((node for node in nodes if node["id"] == edge["source"]), None)target = next((node for node in nodes if node["id"] == edge["target"]), None)if source and target:- relationships.append({"source":source, "edge":edge, "target":target})+ relationships.append({"source": source, "edge": edge, "target": target})return relationships+def to_json_serializable(obj):"""Convert any object to a JSON-serializable format."""import jsonfrom pydantic import BaseModel-+try:# Test if already JSON serializablejson.dumps(obj)@@ -282,11 +298,15 @@ def to_json_serializable(obj):# Handle common casesif isinstance(obj, BaseModel):# Use mode='json' to ensure all Pydantic types are properly serialized- return obj.model_dump(mode='json') if hasattr(obj, 'model_dump') else obj.dict()+ return (+ obj.model_dump(mode="json")+ if hasattr(obj, "model_dump")+ else obj.dict()+ )elif isinstance(obj, (list, tuple)):return [to_json_serializable(item) for item in obj]elif isinstance(obj, dict):return {key: to_json_serializable(value) for key, value in obj.items()}else:# Convert anything else to string- return str(obj)\ No newline at end of file+ return str(obj)diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_nfts.py b/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_nfts.pyindex bbdd45b4..014cdf39 100644--- a/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_nfts.py+++ b/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_nfts.py@@ -8,38 +8,39 @@ from flowsint_core.core.graph_db import Neo4jConnectionETHERSCAN_API_URL = os.getenv("ETHERSCAN_API_URL")+class CryptoWalletAddressToNFTs(Scanner):"""Resolve NFTs for a wallet address (ETH)."""-+# Define types as class attributes - base class handles schema generation automaticallyInputType = List[CryptoWallet]OutputType = List[CryptoNFT]-+def __init__(- self,- sketch_id: Optional[str] = None,- scan_id: Optional[str] = None,- neo4j_conn: Optional[Neo4jConnection] = None,- vault=None,- params: Optional[Dict[str, Any]] = None- ):- super().__init__(- sketch_id=sketch_id,- scan_id=scan_id,- neo4j_conn=neo4j_conn,- params_schema=self.get_params_schema(),- vault=vault,- params=params- )+ self,+ sketch_id: Optional[str] = None,+ scan_id: Optional[str] = None,+ neo4j_conn: Optional[Neo4jConnection] = None,+ vault=None,+ params: Optional[Dict[str, Any]] = None,+ ):+ super().__init__(+ sketch_id=sketch_id,+ scan_id=scan_id,+ neo4j_conn=neo4j_conn,+ params_schema=self.get_params_schema(),+ vault=vault,+ params=params,+ )@classmethoddef required_params(cls) -> bool:return True-+@classmethoddef icon(cls) -> str | None:return "cryptowallet"-+@classmethoddef get_params_schema(cls) -> List[Dict[str, Any]]:"""Declare required parameters for this scanner"""@@ -48,17 +49,17 @@ class CryptoWalletAddressToNFTs(Scanner):"name": "ETHERSCAN_API_KEY","type": "vaultSecret","description": "The Etherscan API key to use for the transaction lookup.",- "required": True+ "required": True,},{"name": "ETHERSCAN_API_URL","type": "url","description": "The Etherscan API URL to use for the transaction lookup.","required": False,- "default": ETHERSCAN_API_URL- }+ "default": ETHERSCAN_API_URL,+ },]-+@classmethoddef name(cls) -> str:return "wallet_to_nfts"@@ -66,7 +67,7 @@ class CryptoWalletAddressToNFTs(Scanner):@classmethoddef category(cls) -> str:return "CryptoWallet"-+@classmethoddef key(cls) -> str:return "address"@@ -101,36 +102,37 @@ class CryptoWalletAddressToNFTs(Scanner):except Exception as e:print(f"Error resolving nfts for {d.address}: {e}")return results-+def _get_nfts(self, address: str, api_key: str, api_url: str) -> List[CryptoNFT]:nfts = []"""Get nfts for a wallet address."""params = {- "module": "account",- "action": "tokennfttx",- "address": address,- "startblock": 0,- "endblock": 99999999,- "page": 1,- "offset": 10000,- "sort": "asc",- "apikey": api_key- }+ "module": "account",+ "action": "tokennfttx",+ "address": address,+ "startblock": 0,+ "endblock": 99999999,+ "page": 1,+ "offset": 10000,+ "sort": "asc",+ "apikey": api_key,+ }response = requests.get(api_url, params=params)data = response.json()results = data["result"]for tx in results:- nfts.append(CryptoNFT(- wallet=CryptoWallet(address=address),- contract_address=tx["contractAddress"],- token_id=tx["tokenID"],- collection_name=tx["collectionName"],- metadata_url=tx["metadataURL"],- image_url=tx["imageURL"],- name=tx["name"],- ))+ nfts.append(+ CryptoNFT(+ wallet=CryptoWallet(address=address),+ contract_address=tx["contractAddress"],+ token_id=tx["tokenID"],+ collection_name=tx["collectionName"],+ metadata_url=tx["metadataURL"],+ image_url=tx["imageURL"],+ name=tx["name"],+ )+ )return nfts-def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:if not self.neo4j_conn:@@ -139,27 +141,47 @@ class CryptoWalletAddressToNFTs(Scanner):for nfts in results:for nft in nfts:# Create or update wallet node- self.create_node('cryptowallet', 'wallet', nft.wallet.address,- caption=nft.wallet.address, type='cryptowallet')+ self.create_node(+ "cryptowallet",+ "wallet",+ nft.wallet.address,+ caption=nft.wallet.address,+ type="cryptowallet",+ )- # Create or update NFT node+ # Create or update NFT nodenft_key = f"{nft.contract_address}_{nft.token_id}"- self.create_node('nft', 'nft_id', nft_key,- contract_address=nft.contract_address,- token_id=nft.token_id,- collection_name=nft.collection_name,- metadata_url=nft.metadata_url,- image_url=nft.image_url,- name=nft.name,- caption=nft.name, type='nft')+ self.create_node(+ "nft",+ "nft_id",+ nft_key,+ contract_address=nft.contract_address,+ token_id=nft.token_id,+ collection_name=nft.collection_name,+ metadata_url=nft.metadata_url,+ image_url=nft.image_url,+ name=nft.name,+ caption=nft.name,+ type="nft",+ )# Create relationship from wallet to NFT- self.create_relationship('cryptowallet', 'wallet', nft.wallet.address,- 'nft', 'nft_id', nft_key, 'OWNS')- self.log_graph_message(f"Found NFT for {nft.wallet.address}: {nft.contract_address} - {nft.token_id}")+ self.create_relationship(+ "cryptowallet",+ "wallet",+ nft.wallet.address,+ "nft",+ "nft_id",+ nft_key,+ "OWNS",+ )+ self.log_graph_message(+ f"Found NFT for {nft.wallet.address}: {nft.contract_address} - {nft.token_id}"+ )return results+# Make types available at module level for easy accessInputType = CryptoWalletAddressToNFTs.InputType-OutputType = CryptoWalletAddressToNFTs.OutputType\ No newline at end of file+OutputType = CryptoWalletAddressToNFTs.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_transactions.py b/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_transactions.pyindex 7d4b860a..2a03ad8e 100644--- a/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_transactions.py+++ b/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_transactions.py@@ -10,22 +10,24 @@ from flowsint_core.core.logger import LoggerETHERSCAN_API_URL = os.getenv("ETHERSCAN_API_URL")+def wei_to_eth(wei_str):return int(wei_str) / 10**18+class CryptoWalletAddressToTransactions(Scanner):-+# Define types as class attributes - base class handles schema generation automaticallyInputType = List[CryptoWallet]OutputType = List[CryptoWalletTransaction]-+def __init__(self,sketch_id: Optional[str] = None,scan_id: Optional[str] = None,neo4j_conn: Optional[Neo4jConnection] = None,vault=None,- params: Optional[Dict[str, Any]] = None+ params: Optional[Dict[str, Any]] = None,):super().__init__(sketch_id=sketch_id,@@ -33,13 +35,13 @@ class CryptoWalletAddressToTransactions(Scanner):neo4j_conn=neo4j_conn,params_schema=self.get_params_schema(),vault=vault,- params=params+ params=params,)@classmethoddef required_params(cls) -> bool:return True-+@classmethoddef icon(cls) -> str | None:return "cryptowallet"@@ -52,15 +54,15 @@ class CryptoWalletAddressToTransactions(Scanner):"name": "ETHERSCAN_API_KEY","type": "vaultSecret","description": "The Etherscan API key to use for the transaction lookup.",- "required": True+ "required": True,},{"name": "ETHERSCAN_API_URL","type": "url","description": "The Etherscan API URL to use for the transaction lookup.","required": False,- "default": ETHERSCAN_API_URL- }+ "default": ETHERSCAN_API_URL,+ },]@classmethod@@ -70,7 +72,7 @@ class CryptoWalletAddressToTransactions(Scanner):@classmethoddef category(cls) -> str:return "CryptoWallet"-+@classmethoddef key(cls) -> str:return "address"@@ -102,76 +104,90 @@ class CryptoWalletAddressToTransactions(Scanner):transactions = await self._get_transactions(d.address, api_key, api_url)results.append(transactions)except Exception as e:- Logger.error(self.sketch_id, {"message": f"Error resolving transactions for {d.address}: {e}"})+ Logger.error(+ self.sketch_id,+ {"message": f"Error resolving transactions for {d.address}: {e}"},+ )return results-- async def _get_transactions(self, address: str, api_key: str, api_url: str) -> List[CryptoWalletTransaction]:++ async def _get_transactions(+ self, address: str, api_key: str, api_url: str+ ) -> List[CryptoWalletTransaction]:transactions = []"""Get transactions for a wallet address."""params = {- "module": "account",- "action": "txlist",- "address": address,- "startblock": 0,- "endblock": 99999999,- "page": 1,- "offset": 100,- "sort": "asc",- "apikey": api_key- }+ "module": "account",+ "action": "txlist",+ "address": address,+ "startblock": 0,+ "endblock": 99999999,+ "page": 1,+ "offset": 100,+ "sort": "asc",+ "apikey": api_key,+ }try:response = requests.get(api_url, params=params)-+# Raise an exception for HTTP errors (4xx or 5xx status codes)response.raise_for_status()-+except requests.exceptions.ConnectionError as e:- raise ValueError(f"An error occurred connecting to {api_url}: Connection failed - {str(e)}")+ raise ValueError(+ f"An error occurred connecting to {api_url}: Connection failed - {str(e)}"+ )except requests.exceptions.Timeout as e:- raise ValueError(f"An error occurred fetching {api_url}: Request timeout - {str(e)}")+ raise ValueError(+ f"An error occurred fetching {api_url}: Request timeout - {str(e)}"+ )except requests.exceptions.RequestException as e:raise ValueError(f"An error occurred fetching {api_url}: {str(e)}")-+try:data = response.json()except requests.exceptions.JSONDecodeError as e:- raise ValueError(f"An error occurred fetching {api_url}: Invalid JSON response - {str(e)}")-+ raise ValueError(+ f"An error occurred fetching {api_url}: Invalid JSON response - {str(e)}"+ )+# Check if the API returned an errorif data.get("status") != "1":error_message = data.get("message", "Unknown API error")raise ValueError(f"An error occurred fetching {api_url}: {error_message}")-+results = data.get("result", [])for tx in results:# Properly determine source and target based on transaction datasource_address = tx["from"]-+if tx["to"] is not None:target_address = tx["to"]else:# Contract creation transaction- target_address = tx["contractAddress"] if tx["contractAddress"] else address-- transactions.append(CryptoWalletTransaction(- source=CryptoWallet(address=source_address),- target=CryptoWallet(address=target_address),- hash=tx["hash"],- value=wei_to_eth(tx["value"]),- timestamp=tx["timeStamp"],- block_number=tx["blockNumber"],- block_hash=tx["blockHash"],- nonce=tx["nonce"],- transaction_index=tx["transactionIndex"],- gas=tx["gas"],- gas_price=tx["gasPrice"],- gas_used=tx["gasUsed"],- cumulative_gas_used=tx["cumulativeGasUsed"],- input=tx["input"],- contract_address=tx["contractAddress"],- ))+ target_address = (+ tx["contractAddress"] if tx["contractAddress"] else address+ )++ transactions.append(+ CryptoWalletTransaction(+ source=CryptoWallet(address=source_address),+ target=CryptoWallet(address=target_address),+ hash=tx["hash"],+ value=wei_to_eth(tx["value"]),+ timestamp=tx["timeStamp"],+ block_number=tx["blockNumber"],+ block_hash=tx["blockHash"],+ nonce=tx["nonce"],+ transaction_index=tx["transactionIndex"],+ gas=tx["gas"],+ gas_price=tx["gasPrice"],+ gas_used=tx["gasUsed"],+ cumulative_gas_used=tx["cumulativeGasUsed"],+ input=tx["input"],+ contract_address=tx["contractAddress"],+ )+ )return transactions-def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:if not self.neo4j_conn:@@ -180,10 +196,20 @@ class CryptoWalletAddressToTransactions(Scanner):for transactions in results:for tx in transactions:# Create or update both wallet nodes- self.create_node('cryptowallet', 'wallet', tx.source.address,- caption=tx.source.address, type="cryptowallet")- self.create_node('cryptowallet', 'wallet', tx.target.address,- caption=tx.target.address, type="cryptowallet")+ self.create_node(+ "cryptowallet",+ "wallet",+ tx.source.address,+ caption=tx.source.address,+ type="cryptowallet",+ )+ self.create_node(+ "cryptowallet",+ "wallet",+ tx.target.address,+ caption=tx.target.address,+ type="cryptowallet",+ )# Create transaction as an edge between wallets (keeping complex query for transaction properties)tx_query = """@@ -207,30 +233,42 @@ class CryptoWalletAddressToTransactions(Scanner):tx.caption = $hash,tx.type = "transaction""""- self.neo4j_conn.query(tx_query, {- "hash": tx.hash,- "source": tx.source.address,- "target": tx.target.address,- "value": tx.value,- "timestamp": tx.timestamp,- "block_number": tx.block_number,- "block_hash": tx.block_hash,- "nonce": tx.nonce,- "transaction_index": tx.transaction_index,- "gas": tx.gas,- "gas_price": tx.gas_price,- "gas_used": tx.gas_used,- "cumulative_gas_used": tx.cumulative_gas_used,- "input": tx.input,- "contract_address": tx.contract_address,- "sketch_id": self.sketch_id- })-- timestamp_str = datetime.fromtimestamp(int(tx.timestamp)).strftime('%Y-%m-%d %H:%M:%S') if tx.timestamp else 'Unknown time'- self.log_graph_message(f"Transaction on {timestamp_str}: {tx.source.address} -> {tx.target.address}")+ self.neo4j_conn.query(+ tx_query,+ {+ "hash": tx.hash,+ "source": tx.source.address,+ "target": tx.target.address,+ "value": tx.value,+ "timestamp": tx.timestamp,+ "block_number": tx.block_number,+ "block_hash": tx.block_hash,+ "nonce": tx.nonce,+ "transaction_index": tx.transaction_index,+ "gas": tx.gas,+ "gas_price": tx.gas_price,+ "gas_used": tx.gas_used,+ "cumulative_gas_used": tx.cumulative_gas_used,+ "input": tx.input,+ "contract_address": tx.contract_address,+ "sketch_id": self.sketch_id,+ },+ )++ timestamp_str = (+ datetime.fromtimestamp(int(tx.timestamp)).strftime(+ "%Y-%m-%d %H:%M:%S"+ )+ if tx.timestamp+ else "Unknown time"+ )+ self.log_graph_message(+ f"Transaction on {timestamp_str}: {tx.source.address} -> {tx.target.address}"+ )return results+# Make types available at module level for easy accessInputType = CryptoWalletAddressToTransactions.InputType-OutputType = CryptoWalletAddressToTransactions.OutputType\ No newline at end of file+OutputType = CryptoWalletAddressToTransactions.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/domains/domain_to_asn.py b/flowsint-transforms/src/flowsint_transforms/domains/domain_to_asn.pyindex 0a6f920c..21d6b162 100644--- a/flowsint-transforms/src/flowsint_transforms/domains/domain_to_asn.py+++ b/flowsint-transforms/src/flowsint_transforms/domains/domain_to_asn.py@@ -8,6 +8,7 @@ from flowsint_types.asn import ASNfrom flowsint_core.utils import is_valid_domainfrom flowsint_core.core.logger import Logger+class DomainToAsnScanner(Scanner):"""Takes a domain and returns its corresponding ASN."""@@ -22,7 +23,7 @@ class DomainToAsnScanner(Scanner):@classmethoddef category(cls) -> str:return "Domain"-+@classmethoddef key(cls) -> str:return "Domain"@@ -49,37 +50,43 @@ class DomainToAsnScanner(Scanner):try:# First resolve domain to IPip = socket.gethostbyname(domain.domain)-+# Use asnmap to get ASN inforesult = subprocess.run(["asnmap", "-a", ip, "-json"],capture_output=True,text=True,- timeout=30+ timeout=30,)-+if result.returncode == 0:output = result.stdout.strip()if output:asn_data = json.loads(output)- if asn_data and 'as_number' in asn_data:+ if asn_data and "as_number" in asn_data:asn = ASN(- asn=str(asn_data['as_number']),- name=asn_data.get('as_name', ''),- org=asn_data.get('as_org', ''),- country=asn_data.get('as_country', '')+ asn=str(asn_data["as_number"]),+ name=asn_data.get("as_name", ""),+ org=asn_data.get("as_org", ""),+ country=asn_data.get("as_country", ""),)results.append(asn)-+except Exception as e:- Logger.error(self.sketch_id, {"message": f"Error getting ASN for domain {domain.domain}: {e}"})+ Logger.error(+ self.sketch_id,+ {"message": f"Error getting ASN for domain {domain.domain}: {e}"},+ )continue-+return results- def postprocess(self, results: OutputType, input_data: InputType = None) -> OutputType:+ def postprocess(+ self, results: OutputType, input_data: InputType = None+ ) -> OutputType:return results+# Make types available at module level for easy accessInputType = DomainToAsnScanner.InputType-OutputType = DomainToAsnScanner.OutputType\ No newline at end of file+OutputType = DomainToAsnScanner.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/domains/resolve.py b/flowsint-transforms/src/flowsint_transforms/domains/resolve.pyindex 0f38d94b..a9aa53c2 100644--- a/flowsint-transforms/src/flowsint_transforms/domains/resolve.py+++ b/flowsint-transforms/src/flowsint_transforms/domains/resolve.py@@ -5,9 +5,10 @@ from flowsint_types.domain import Domainfrom flowsint_types.ip import Ipfrom flowsint_core.utils import is_valid_domain, is_root_domain+class ResolveScanner(Scanner):"""Resolve domain names to IP addresses."""-+# Define the input and output types as class attributesInputType = List[Domain]OutputType = List[Ip]@@ -19,7 +20,7 @@ class ResolveScanner(Scanner):@classmethoddef category(cls) -> str:return "Domain"-+@classmethoddef key(cls) -> str:return "domain"@@ -263,10 +264,14 @@ Check FlowSint logs for detailed resolution information.if isinstance(item, str):domain_obj = Domain(domain=item, root=is_root_domain(item))elif isinstance(item, dict) and "domain" in item:- domain_obj = Domain(domain=item["domain"], root=is_root_domain(item["domain"]))+ domain_obj = Domain(+ domain=item["domain"], root=is_root_domain(item["domain"])+ )elif isinstance(item, Domain):# If the Domain object already exists, update its root field- domain_obj = Domain(domain=item.domain, root=is_root_domain(item.domain))+ domain_obj = Domain(+ domain=item.domain, root=is_root_domain(item.domain)+ )if domain_obj and is_valid_domain(domain_obj.domain):cleaned.append(domain_obj)return cleaned@@ -283,12 +288,28 @@ Check FlowSint logs for detailed resolution information.def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:for domain_obj, ip_obj in zip(original_input, results):- self.create_node('domain', 'domain', domain_obj.domain, type='domain' if domain_obj.root else 'subdomain')- self.create_node('ip', 'address', ip_obj.address, type='ip')- self.create_relationship('domain', 'domain', domain_obj.domain, 'ip', 'address', ip_obj.address, 'RESOLVES_TO')- self.log_graph_message(f"IP found for domain {domain_obj.domain} -> {ip_obj.address}")+ self.create_node(+ "domain",+ "domain",+ domain_obj.domain,+ type="domain" if domain_obj.root else "subdomain",+ )+ self.create_node("ip", "address", ip_obj.address, type="ip")+ self.create_relationship(+ "domain",+ "domain",+ domain_obj.domain,+ "ip",+ "address",+ ip_obj.address,+ "RESOLVES_TO",+ )+ self.log_graph_message(+ f"IP found for domain {domain_obj.domain} -> {ip_obj.address}"+ )return results+# Make types available at module level for easy accessInputType = ResolveScanner.InputType-OutputType = ResolveScanner.OutputType\ No newline at end of file+OutputType = ResolveScanner.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/domains/subdomains.py b/flowsint-transforms/src/flowsint_transforms/domains/subdomains.pyindex 636bc7b7..83a11a79 100644--- a/flowsint-transforms/src/flowsint_transforms/domains/subdomains.py+++ b/flowsint-transforms/src/flowsint_transforms/domains/subdomains.py@@ -6,10 +6,12 @@ from flowsint_types.domain import Domainfrom flowsint_core.utils import is_valid_domainfrom flowsint_core.core.logger import Loggerfrom tools.network.subfinder import SubfinderTool+from flowsint_core.core.logger import Logger+class SubdomainScanner(Scanner):"""Scanner to find subdomains associated with a domain."""-+# Define types as class attributes - base class handles schema generation automaticallyInputType = List[Domain | str]OutputType = List[Domain]@@ -26,7 +28,6 @@ class SubdomainScanner(Scanner):def key(cls) -> str:return "domain"-def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:cleaned: InputType = []for item in data:@@ -40,7 +41,6 @@ class SubdomainScanner(Scanner):if domain_obj and is_valid_domain(domain_obj.domain):cleaned.append(domain_obj)return cleaned-async def scan(self, data: InputType) -> OutputType:"""Find subdomains using subfinder (if available) or fallback to crt.sh."""@@ -52,7 +52,10 @@ class SubdomainScanner(Scanner):if use_subfinder:subdomains = self.__get_subdomains_from_subfinder(d.domain)if not subdomains:- Logger.warn(self.sketch_id, f"subfinder failed for {d.domain}, falling back to crt.sh")+ Logger.warn(+ self.sketch_id,+ f"subfinder failed for {d.domain}, falling back to crt.sh",+ )subdomains = self.__get_subdomains_from_crtsh(d.domain)else:Logger.info(self.sketch_id, "subfinder not found, using crt.sh only")@@ -69,8 +72,7 @@ class SubdomainScanner(Scanner):subdomains: set[str] = set()try:response = requests.get(- f"https://crt.sh/?q=%25.{domain}&output=json",- timeout=60+ f"https://crt.sh/?q=%25.{domain}&output=json", timeout=60)if response.ok:entries = response.json()@@ -78,7 +80,12 @@ class SubdomainScanner(Scanner):name_value = entry.get("name_value", "")for sub in name_value.split("\n"):sub = sub.strip().lower()- if "*" not in sub and is_valid_domain(sub) and sub.endswith(domain) and sub != domain:+ if (+ "*" not in sub+ and is_valid_domain(sub)+ and sub.endswith(domain)+ and sub != domain+ ):subdomains.add(sub)elif "*" in sub:continue@@ -102,20 +109,31 @@ class SubdomainScanner(Scanner):continuefor subdomain in domain_obj["subdomains"]:output.append(Domain(domain=subdomain))- Logger.info(self.sketch_id, {"message": f"{domain_obj['domain']} -> {subdomain}"})-+ Logger.info(+ self.sketch_id,+ {"message": f"{domain_obj['domain']} -> {subdomain}"},+ )+# Create subdomain node- self.create_node('domain', 'domain', subdomain,- type='domain')-+ self.create_node("domain", "domain", subdomain, type="domain")+# Create relationship from parent domain to subdomain- self.create_relationship('domain', 'domain', domain_obj["domain"],- 'domain', 'domain', subdomain, 'HAS_SUBDOMAIN')-- self.log_graph_message(f"{domain_obj['domain']} -> {len(domain_obj['subdomains'])} subdomain(s) found.")+ self.create_relationship(+ "domain",+ "domain",+ domain_obj["domain"],+ "domain",+ "domain",+ subdomain,+ "HAS_SUBDOMAIN",+ )++ self.log_graph_message(+ f"{domain_obj['domain']} -> {len(domain_obj['subdomains'])} subdomain(s) found."+ )return outputInputType = SubdomainScanner.InputType-OutputType = SubdomainScanner.OutputType\ No newline at end of file+OutputType = SubdomainScanner.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/domains/to_website.py b/flowsint-transforms/src/flowsint_transforms/domains/to_website.pyindex 8bc3f8cb..70116968 100644--- a/flowsint-transforms/src/flowsint_transforms/domains/to_website.py+++ b/flowsint-transforms/src/flowsint_transforms/domains/to_website.py@@ -6,6 +6,7 @@ from flowsint_types.domain import Domainfrom flowsint_types.website import Websitefrom flowsint_core.core.logger import Logger+class DomainToWebsiteScanner(Scanner):"""From domain to website."""@@ -20,7 +21,7 @@ class DomainToWebsiteScanner(Scanner):@classmethoddef category(cls) -> str:return "Domain"-+@classmethoddef key(cls) -> str:return "domain"@@ -48,13 +49,15 @@ class DomainToWebsiteScanner(Scanner):# Try HTTPS firsttry:https_url = f"https://{domain.domain}"- response = requests.head(https_url, timeout=10, allow_redirects=True)+ response = requests.head(+ https_url, timeout=10, allow_redirects=True+ )if response.status_code < 400:results.append(Website(url=https_url, domain=domain))continueexcept requests.RequestException:pass-+# Try HTTP if HTTPS failstry:http_url = f"http://{domain.domain}"@@ -64,15 +67,20 @@ class DomainToWebsiteScanner(Scanner):continueexcept requests.RequestException:pass-+# If both fail, still add HTTPS URL as defaultresults.append(Website(url=f"https://{domain.domain}", domain=domain))-+except Exception as e:- Logger.error(self.sketch_id, {"message": f"Error converting domain {domain.domain} to website: {e}"})+ Logger.error(+ self.sketch_id,+ {+ "message": f"Error converting domain {domain.domain} to website: {e}"+ },+ )# Add HTTPS URL as fallbackresults.append(Website(url=f"https://{domain.domain}", domain=domain))-+return resultsdef postprocess(self, results: OutputType, original_input: InputType) -> OutputType:@@ -80,33 +88,58 @@ class DomainToWebsiteScanner(Scanner):# Log each redirect stepif website.redirects:for i, redirect_url in enumerate(website.redirects):- next_url = website.redirects[i + 1] if i + 1 < len(website.redirects) else str(website.url)+ next_url = (+ website.redirects[i + 1]+ if i + 1 < len(website.redirects)+ else str(website.url)+ )redirect_payload = {"message": f"Redirect: {str(redirect_url)} -> {str(next_url)}"}Logger.info(self.sketch_id, redirect_payload)-+if self.neo4j_conn:# Create domain node- self.create_node('domain', 'domain', website.domain.domain, type="domain")-+ self.create_node(+ "domain", "domain", website.domain.domain, type="domain"+ )+# Create website node- self.create_node('website', 'url', str(website.url),- active=website.active,- redirects=[str(redirect) for redirect in website.redirects] if website.redirects else [],- type="website")-+ self.create_node(+ "website",+ "url",+ str(website.url),+ active=website.active,+ redirects=(+ [str(redirect) for redirect in website.redirects]+ if website.redirects+ else []+ ),+ type="website",+ )+# Create relationship- self.create_relationship('domain', 'domain', website.domain.domain,- 'website', 'url', str(website.url), 'HAS_WEBSITE')-+ self.create_relationship(+ "domain",+ "domain",+ website.domain.domain,+ "website",+ "url",+ str(website.url),+ "HAS_WEBSITE",+ )+is_active_str = "active" if website.active else "inactive"- redirects_str = f" (redirects: {len(website.redirects)})" if website.redirects else ""- self.log_graph_message(f"{website.domain.domain} -> {str(website.url)} ({is_active_str}){redirects_str}")+ redirects_str = (+ f" (redirects: {len(website.redirects)})" if website.redirects else ""+ )+ self.log_graph_message(+ f"{website.domain.domain} -> {str(website.url)} ({is_active_str}){redirects_str}"+ )return results# Make types available at module level for easy accessInputType = DomainToWebsiteScanner.InputType-OutputType = DomainToWebsiteScanner.OutputType\ No newline at end of file+OutputType = DomainToWebsiteScanner.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/domains/whois.py b/flowsint-transforms/src/flowsint_transforms/domains/whois.pyindex 95e5cbd8..162887c4 100644--- a/flowsint-transforms/src/flowsint_transforms/domains/whois.py+++ b/flowsint-transforms/src/flowsint_transforms/domains/whois.py@@ -7,6 +7,7 @@ from flowsint_types.whois import Whoisfrom flowsint_types.email import Emailfrom flowsint_core.core.logger import Logger+class WhoisScanner(Scanner):"""Scan for WHOIS information of a domain."""@@ -21,7 +22,7 @@ class WhoisScanner(Scanner):@classmethoddef category(cls) -> str:return "Domain"-+@classmethoddef key(cls) -> str:return "domain"@@ -52,92 +53,142 @@ class WhoisScanner(Scanner):emails = []if whois_info.emails:if isinstance(whois_info.emails, list):- emails = [Email(email=email) for email in whois_info.emails if email]+ emails = [+ Email(email=email)+ for email in whois_info.emails+ if email+ ]else:emails = [Email(email=whois_info.emails)]-+# Convert datetime objects to ISO format stringscreation_date_str = Noneif whois_info.creation_date:if isinstance(whois_info.creation_date, list):- creation_date_str = whois_info.creation_date[0].isoformat() if whois_info.creation_date else None+ creation_date_str = (+ whois_info.creation_date[0].isoformat()+ if whois_info.creation_date+ else None+ )else:creation_date_str = whois_info.creation_date.isoformat()-+expiration_date_str = Noneif whois_info.expiration_date:if isinstance(whois_info.expiration_date, list):- expiration_date_str = whois_info.expiration_date[0].isoformat() if whois_info.expiration_date else None+ expiration_date_str = (+ whois_info.expiration_date[0].isoformat()+ if whois_info.expiration_date+ else None+ )else:expiration_date_str = whois_info.expiration_date.isoformat()-+whois_obj = Whois(domain=domain.domain,- registrar=str(whois_info.registrar) if whois_info.registrar else None,+ registrar=(+ str(whois_info.registrar) if whois_info.registrar else None+ ),org=str(whois_info.org) if whois_info.org else None,city=str(whois_info.city) if whois_info.city else None,country=str(whois_info.country) if whois_info.country else None,email=emails[0] if emails else None,creation_date=creation_date_str,- expiration_date=expiration_date_str+ expiration_date=expiration_date_str,)results.append(whois_obj)-+except Exception as e:- Logger.error(self.sketch_id, {"message": f"Error getting WHOIS for domain {domain.domain}: {e}"})+ Logger.error(+ self.sketch_id,+ {"message": f"Error getting WHOIS for domain {domain.domain}: {e}"},+ )continue-+return resultsdef postprocess(self, results: OutputType, original_input: InputType) -> OutputType:for whois_obj in results:if not self.neo4j_conn:continue-+# Create domain node- self.create_node('domain', 'domain', whois_obj.domain, type='domain')-- # Create whois node+ self.create_node("domain", "domain", whois_obj.domain, type="domain")++ # Create whois nodewhois_key = f"{whois_obj.domain}_{self.sketch_id}"- self.create_node('whois', 'whois_id', whois_key,- domain=whois_obj.domain,- registrar=whois_obj.registrar,- org=whois_obj.org,- city=whois_obj.city,- country=whois_obj.country,- creation_date=whois_obj.creation_date,- expiration_date=whois_obj.expiration_date,- email=whois_obj.email.email if whois_obj.email else None,- label="Whois", type="whois")-+ self.create_node(+ "whois",+ "whois_id",+ whois_key,+ domain=whois_obj.domain,+ registrar=whois_obj.registrar,+ org=whois_obj.org,+ city=whois_obj.city,+ country=whois_obj.country,+ creation_date=whois_obj.creation_date,+ expiration_date=whois_obj.expiration_date,+ email=whois_obj.email.email if whois_obj.email else None,+ label="Whois",+ type="whois",+ )+# Create relationship between domain and whois- self.create_relationship('domain', 'domain', whois_obj.domain,- 'whois', 'whois_id', whois_key, 'HAS_WHOIS')+ self.create_relationship(+ "domain",+ "domain",+ whois_obj.domain,+ "whois",+ "whois_id",+ whois_key,+ "HAS_WHOIS",+ )# Create organization node if org information is availableif whois_obj.org:- self.create_node('organization', 'name', whois_obj.org,- country=whois_obj.country,- founding_date=whois_obj.creation_date,- description=f"Organization from WHOIS data for {whois_obj.domain}",- caption=whois_obj.org, type="organization")-+ self.create_node(+ "organization",+ "name",+ whois_obj.org,+ country=whois_obj.country,+ founding_date=whois_obj.creation_date,+ description=f"Organization from WHOIS data for {whois_obj.domain}",+ caption=whois_obj.org,+ type="organization",+ )+# Create relationship between organization and domain- self.create_relationship('organization', 'name', whois_obj.org,- 'domain', 'domain', whois_obj.domain, 'HAS_DOMAIN')-- self.log_graph_message(f"{whois_obj.domain} -> {whois_obj.org} (organization)")+ self.create_relationship(+ "organization",+ "name",+ whois_obj.org,+ "domain",+ "domain",+ whois_obj.domain,+ "HAS_DOMAIN",+ )- if whois_obj.email:- self.create_node('email', 'email', whois_obj.email.email, type="email")- self.create_relationship('whois', 'whois_id', whois_key,- 'email', 'email', whois_obj.email.email, 'REGISTERED_BY')-- self.log_graph_message(f"WHOIS for {whois_obj.domain} -> registrar: {whois_obj.registrar} org: {whois_obj.org} city: {whois_obj.city} country: {whois_obj.country} creation_date: {whois_obj.creation_date} expiration_date: {whois_obj.expiration_date}")+ self.log_graph_message(+ f"{whois_obj.domain} -> {whois_obj.org} (organization)"+ )- return results+ if whois_obj.email:+ self.create_node("email", "email", whois_obj.email.email, type="email")+ self.create_relationship(+ "whois",+ "whois_id",+ whois_key,+ "email",+ "email",+ whois_obj.email.email,+ "REGISTERED_BY",+ )+ self.log_graph_message(+ f"WHOIS for {whois_obj.domain} -> registrar: {whois_obj.registrar} org: {whois_obj.org} city: {whois_obj.city} country: {whois_obj.country} creation_date: {whois_obj.creation_date} expiration_date: {whois_obj.expiration_date}"+ )+ return results# Make types available at module level for easy accessdiff --git a/flowsint-transforms/src/flowsint_transforms/emails/to_gravatar.py b/flowsint-transforms/src/flowsint_transforms/emails/to_gravatar.pyindex 84fd2b9e..c8d418bc 100644--- a/flowsint-transforms/src/flowsint_transforms/emails/to_gravatar.py+++ b/flowsint-transforms/src/flowsint_transforms/emails/to_gravatar.py@@ -6,6 +6,7 @@ from flowsint_core.core.logger import Loggerfrom flowsint_types.email import Emailfrom flowsint_types.gravatar import Gravatar+class EmailToGravatarScanner(Scanner):"""From md5 hash of email to gravatar."""@@ -19,7 +20,7 @@ class EmailToGravatarScanner(Scanner):@classmethoddef category(cls) -> str:return "Email"-+@classmethoddef key(cls) -> str:return "email"@@ -40,75 +41,97 @@ class EmailToGravatarScanner(Scanner):async def scan(self, data: InputType) -> OutputType:results: OutputType = []-+for email in data:try:# Generate MD5 hash of emailemail_hash = hashlib.md5(email.email.lower().encode()).hexdigest()-+# Query Gravatar APIgravatar_url = f"https://www.gravatar.com/avatar/{email_hash}?d=404"response = requests.head(gravatar_url, timeout=10)-+if response.status_code == 200:# Gravatar found, get profile infoprofile_url = f"https://www.gravatar.com/{email_hash}.json"profile_response = requests.get(profile_url, timeout=10)-+gravatar_data = {"src": gravatar_url,"hash": email_hash,"profile_url": profile_url,- "exists": True+ "exists": True,}-+if profile_response.status_code == 200:profile_data = profile_response.json()if "entry" in profile_data and profile_data["entry"]:entry = profile_data["entry"][0]- gravatar_data.update({- "display_name": entry.get("displayName"),- "about_me": entry.get("aboutMe"),- "location": entry.get("currentLocation")- })-+ gravatar_data.update(+ {+ "display_name": entry.get("displayName"),+ "about_me": entry.get("aboutMe"),+ "location": entry.get("currentLocation"),+ }+ )+gravatar = Gravatar(**gravatar_data)results.append(gravatar)-+except Exception as e:- Logger.error(self.sketch_id, {"message": f"Error checking Gravatar for email {email.email}: {e}"})+ Logger.error(+ self.sketch_id,+ {+ "message": f"Error checking Gravatar for email {email.email}: {e}"+ },+ )continue-+return resultsdef postprocess(self, results: OutputType, original_input: InputType) -> OutputType:for email_obj, gravatar_obj in zip(original_input, results):if not self.neo4j_conn:continue-+# Create email node- self.create_node('email', 'email', email_obj.email, type='email')-+ self.create_node("email", "email", email_obj.email, type="email")+# Create gravatar nodegravatar_key = f"{email_obj.email}_{self.sketch_id}"- self.create_node('gravatar', 'gravatar_id', gravatar_key,- email=email_obj.email,- hash=gravatar_obj.hash,- src=str(gravatar_obj.src),- display_name=gravatar_obj.display_name,- location=gravatar_obj.location,- about_me=gravatar_obj.about_me,- exists=gravatar_obj.exists,- label="Gravatar", type="gravatar")-+ self.create_node(+ "gravatar",+ "gravatar_id",+ gravatar_key,+ email=email_obj.email,+ hash=gravatar_obj.hash,+ src=str(gravatar_obj.src),+ display_name=gravatar_obj.display_name,+ location=gravatar_obj.location,+ about_me=gravatar_obj.about_me,+ exists=gravatar_obj.exists,+ label="Gravatar",+ type="gravatar",+ )+# Create relationship between email and gravatar- self.create_relationship('email', 'email', email_obj.email,- 'gravatar', 'gravatar_id', gravatar_key, 'HAS_GRAVATAR')-- self.log_graph_message(f"Gravatar found for email {email_obj.email} -> hash: {gravatar_obj.hash}")-+ self.create_relationship(+ "email",+ "email",+ email_obj.email,+ "gravatar",+ "gravatar_id",+ gravatar_key,+ "HAS_GRAVATAR",+ )++ self.log_graph_message(+ f"Gravatar found for email {email_obj.email} -> hash: {gravatar_obj.hash}"+ )+return results+# Make types available at module level for easy accessInputType = EmailToGravatarScanner.InputType-OutputType = EmailToGravatarScanner.OutputType\ No newline at end of file+OutputType = EmailToGravatarScanner.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/emails/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/emails/to_leaks.pyindex c552ff8f..cc04f33d 100644--- a/flowsint-transforms/src/flowsint_transforms/emails/to_leaks.py+++ b/flowsint-transforms/src/flowsint_transforms/emails/to_leaks.py@@ -14,28 +14,29 @@ load_dotenv()HIBP_API_KEY = os.getenv("HIBP_API_KEY")+class EmailToBreachesScanner(Scanner):"""From email to breaches using Have I Been Pwned API."""InputType = List[Email]OutputType = List[tuple] # List of (email, breach) tuples-+def __init__(- self,- sketch_id: Optional[str] = None,- scan_id: Optional[str] = None,- neo4j_conn: Optional[Neo4jConnection] = None,- vault=None,- params: Optional[Dict[str, Any]] = None- ):- super().__init__(- sketch_id=sketch_id,- scan_id=scan_id,- neo4j_conn=neo4j_conn,- params_schema=self.get_params_schema(),- vault=vault,- params=params- )+ self,+ sketch_id: Optional[str] = None,+ scan_id: Optional[str] = None,+ neo4j_conn: Optional[Neo4jConnection] = None,+ vault=None,+ params: Optional[Dict[str, Any]] = None,+ ):+ super().__init__(+ sketch_id=sketch_id,+ scan_id=scan_id,+ neo4j_conn=neo4j_conn,+ params_schema=self.get_params_schema(),+ vault=vault,+ params=params,+ )@classmethoddef name(cls) -> str:@@ -44,15 +45,15 @@ class EmailToBreachesScanner(Scanner):@classmethoddef category(cls) -> str:return "Email"-+@classmethoddef key(cls) -> str:return "email"-+@classmethoddef required_params(cls) -> bool:return True-+@classmethoddef get_params_schema(cls) -> List[Dict[str, Any]]:"""Declare required parameters for this scanner"""@@ -61,15 +62,15 @@ class EmailToBreachesScanner(Scanner):"name": "HIBP_API_KEY","type": "vaultSecret","description": "The HIBP API key to use for breaches lookup.",- "required": True+ "required": True,},{"name": "HIBP_API_URL","type": "url","description": "The HIBP API URL to use for breaches lookup.","required": False,- "default": "https://haveibeenpwned.com/api/v3/breachedaccount/"- }+ "default": "https://haveibeenpwned.com/api/v3/breachedaccount/",+ },]def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:@@ -92,13 +93,15 @@ class EmailToBreachesScanner(Scanner):api_key = self.get_params().get("HIBP_API_KEY", None)api_url = self.get_params().get("HIBP_API_URL", None)if not api_key:- Logger.error(self.sketch_id, {"message": "A valid HIBP_API_KEY is required to scan for breaches."})+ Logger.error(+ self.sketch_id,+ {"message": "A valid HIBP_API_KEY is required to scan for breaches."},+ )if not api_url:- Logger.error(self.sketch_id, {"message": "Could not find HIBP_API_URL in params."})- headers = {- "hibp-api-key": api_key,- "User-Agent": "FlowsInt-Scanner"- }+ Logger.error(+ self.sketch_id, {"message": "Could not find HIBP_API_URL in params."}+ )+ headers = {"hibp-api-key": api_key, "User-Agent": "FlowsInt-Scanner"}Logger.info(self.sketch_id, {"message": f"HIBP API URL: {api_url}"})for email in data:try:@@ -106,10 +109,17 @@ class EmailToBreachesScanner(Scanner):full_url = urljoin(api_url, f"{email.email}?truncateResponse=false")Logger.error(self.sketch_id, {"message": f"full url: {full_url}"})response = requests.get(full_url, headers=headers, timeout=10)- Logger.info(self.sketch_id, {"message": f"HIBP API response: {response.json()}"})+ Logger.info(+ self.sketch_id, {"message": f"HIBP API response: {response.json()}"}+ )if response.status_code == 200:breaches_data = response.json()- Logger.info(self.sketch_id, {"message": f"Found {len(breaches_data)} breaches for {email.email}"})+ Logger.info(+ self.sketch_id,+ {+ "message": f"Found {len(breaches_data)} breaches for {email.email}"+ },+ )for breach_data in breaches_data:breach = Breach(name=breach_data.get("Name", ""),@@ -126,60 +136,103 @@ class EmailToBreachesScanner(Scanner):issensitive=breach_data.get("IsSensitive", False),isretired=breach_data.get("IsRetired", False),isspamlist=breach_data.get("IsSpamList", False),- logopath=breach_data.get("LogoPath", "")+ logopath=breach_data.get("LogoPath", ""),)# Store email and breach as a tupleresults.append((email.email, breach))- Logger.info(self.sketch_id, {"message": f"Added breach: {breach.name} for email: {email.email}"})-+ Logger.info(+ self.sketch_id,+ {+ "message": f"Added breach: {breach.name} for email: {email.email}"+ },+ )+elif response.status_code == 404:# No breaches found for this email- Logger.info(self.sketch_id, {"message": f"No breaches found for email {email.email}"})+ Logger.info(+ self.sketch_id,+ {"message": f"No breaches found for email {email.email}"},+ )continue-+else:- Logger.error(self.sketch_id, {"message": f"HIBP API error for {email.email}: {response.status_code}"})+ Logger.error(+ self.sketch_id,+ {+ "message": f"HIBP API error for {email.email}: {response.status_code}"+ },+ )continue-+except Exception as e:- Logger.error(self.sketch_id, {"message": f"Error checking breaches for email {email.email}: {e}"})+ Logger.error(+ self.sketch_id,+ {+ "message": f"Error checking breaches for email {email.email}: {e}"+ },+ )continue-- Logger.info(self.sketch_id, {"message": f"Scan completed. Total results: {len(results)}"})++ Logger.info(+ self.sketch_id,+ {"message": f"Scan completed. Total results: {len(results)}"},+ )return resultsdef postprocess(self, results: OutputType, original_input: InputType) -> OutputType:- Logger.info(self.sketch_id, {"message": f"Postprocess started. Results count: {len(results)}, Original input count: {len(original_input)}"})-+ Logger.info(+ self.sketch_id,+ {+ "message": f"Postprocess started. Results count: {len(results)}, Original input count: {len(original_input)}"+ },+ )+# Create email nodes firstfor email_obj in original_input:if not self.neo4j_conn:continue# Create email node- self.create_node('email', 'email', email_obj.email, type='email')- Logger.info(self.sketch_id, {"message": f"Created email node: {email_obj.email}"})-+ self.create_node("email", "email", email_obj.email, type="email")+ Logger.info(+ self.sketch_id, {"message": f"Created email node: {email_obj.email}"}+ )+# Process all breachesfor email_address, breach_obj in results:if not self.neo4j_conn:continue-+# Create breach nodebreach_key = f"{breach_obj.name}_{self.sketch_id}"self.create_node(- 'breach', 'breach_id', breach_key,- **breach_obj.dict(),- label=breach_obj.name, type="breach"+ "breach",+ "breach_id",+ breach_key,+ **breach_obj.dict(),+ label=breach_obj.name,+ type="breach",)- Logger.info(self.sketch_id, {"message": f"Created breach node: {breach_key}"})-+ Logger.info(+ self.sketch_id, {"message": f"Created breach node: {breach_key}"}+ )+# Create relationship between the specific email and this breach- self.create_relationship('email', 'email', email_address,- 'breach', 'breach_id', breach_key, 'FOUND_IN_BREACH')- self.log_graph_message(f"Breach found for email {email_address} -> {breach_obj.name} ({breach_obj.title})")-+ self.create_relationship(+ "email",+ "email",+ email_address,+ "breach",+ "breach_id",+ breach_key,+ "FOUND_IN_BREACH",+ )+ self.log_graph_message(+ f"Breach found for email {email_address} -> {breach_obj.name} ({breach_obj.title})"+ )+return results+# Make types available at module level for easy accessInputType = EmailToBreachesScanner.InputType-OutputType = EmailToBreachesScanner.OutputType\ No newline at end of file+OutputType = EmailToBreachesScanner.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/individuals/to_org.py b/flowsint-transforms/src/flowsint_transforms/individuals/to_org.pyindex ae1eb669..b4f3351c 100644--- a/flowsint-transforms/src/flowsint_transforms/individuals/to_org.py+++ b/flowsint-transforms/src/flowsint_transforms/individuals/to_org.py@@ -6,6 +6,7 @@ from flowsint_types.individual import Individualfrom flowsint_core.core.logger import Loggerfrom tools.organizations.sirene import SireneTool+class IndividualToOrgScanner(Scanner):"""Find organization from a person with data from SIRENE (France only)."""@@ -20,7 +21,7 @@ class IndividualToOrgScanner(Scanner):@classmethoddef category(cls) -> str:return "Individual"-+@classmethoddef key(cls) -> str:return "fullname"@@ -35,29 +36,45 @@ class IndividualToOrgScanner(Scanner):if len(parts) >= 2:firstname = parts[0]lastname = " ".join(parts[1:])- cleaned.append(Individual(first_name=firstname, last_name=lastname, full_name=item))- elif isinstance(item, dict) and item.get("first_name") and item.get("last_name"):+ cleaned.append(+ Individual(+ first_name=firstname, last_name=lastname, full_name=item+ )+ )+ elif (+ isinstance(item, dict)+ and item.get("first_name")+ and item.get("last_name")+ ):cleaned.append(Individual(**item))elif isinstance(item, Individual):cleaned.append(item)- if(len(cleaned)==0):- Logger.error(self.sketch_id, {"message": "[INDIVIDUAL_TO_ORG] The input type did not match."})+ if len(cleaned) == 0:+ Logger.error(+ self.sketch_id,+ {"message": "[INDIVIDUAL_TO_ORG] The input type did not match."},+ )return cleanedasync def scan(self, data: InputType) -> OutputType:-+results: OutputType = []for individual in data:try:sirene = SireneTool()raw_orgs = sirene.launch(individual.full_name, limit=25)- if len(raw_orgs)> 0:+ if len(raw_orgs) > 0:for org_dict in raw_orgs:enriched_org = self.enrich_org(org_dict)if enriched_org is not None:results.append(enriched_org)except Exception as e:- Logger.error(self.sketch_id, {"message": f"Error finding organization for {individual.full_name}: {e}"})+ Logger.error(+ self.sketch_id,+ {+ "message": f"Error finding organization for {individual.full_name}: {e}"+ },+ )return resultsdef enrich_org(self, company: Dict) -> Organization:@@ -68,15 +85,16 @@ class IndividualToOrgScanner(Scanner):siege_geo_adresse = Noneif siege.get("latitude") and siege.get("longitude"):from flowsint_types.address import PhysicalAddress+siege_geo_adresse = PhysicalAddress(address=siege.get("adresse", ""),city=siege.get("libelle_commune", ""),country="FR", # SIRENE is French registryzip=siege.get("code_postal", ""),latitude=float(siege.get("latitude", 0)),- longitude=float(siege.get("longitude", 0))+ longitude=float(siege.get("longitude", 0)),)-+# Extract dirigeants and convert to Individual objectsdirigeants = []for dirigeant_data in company.get("dirigeants", []):@@ -85,17 +103,20 @@ class IndividualToOrgScanner(Scanner):last_name=dirigeant_data.get("nom", ""),full_name=f"{dirigeant_data.get('prenoms', '')} {dirigeant_data.get('nom', '')}".strip(),birth_date=dirigeant_data.get("date_de_naissance"),- gender=None # Not available in SIRENE data+ gender=None, # Not available in SIRENE data)dirigeants.append(dirigeant)-+name = company.get("nom_raison_sociale") or company.get("nom_complet")-+# Ensure we have a valid nameif not name:- Logger.error(self.sketch_id, {"message": f"Organization has no valid name: {company}"})+ Logger.error(+ self.sketch_id,+ {"message": f"Organization has no valid name: {company}"},+ )return None-+return Organization(# Basic informationname=name,@@ -103,40 +124,41 @@ class IndividualToOrgScanner(Scanner):nom_complet=company.get("nom_complet"),nom_raison_sociale=company.get("nom_raison_sociale"),sigle=company.get("sigle"),-# Establishment informationnombre_etablissements=company.get("nombre_etablissements"),- nombre_etablissements_ouverts=company.get("nombre_etablissements_ouverts"),-+ nombre_etablissements_ouverts=company.get(+ "nombre_etablissements_ouverts"+ ),# Activity informationactivite_principale=company.get("activite_principale"),section_activite_principale=company.get("section_activite_principale"),-# Company categorycategorie_entreprise=company.get("categorie_entreprise"),annee_categorie_entreprise=company.get("annee_categorie_entreprise"),-# Employment informationcaractere_employeur=company.get("caractere_employeur"),tranche_effectif_salarie=company.get("tranche_effectif_salarie"),- annee_tranche_effectif_salarie=company.get("annee_tranche_effectif_salarie"),-+ annee_tranche_effectif_salarie=company.get(+ "annee_tranche_effectif_salarie"+ ),# Datesdate_creation=company.get("date_creation"),date_fermeture=company.get("date_fermeture"),date_mise_a_jour=company.get("date_mise_a_jour"),date_mise_a_jour_insee=company.get("date_mise_a_jour_insee"),date_mise_a_jour_rne=company.get("date_mise_a_jour_rne"),-# Legal informationnature_juridique=company.get("nature_juridique"),etat_administratif=company.get("etat_administratif"),statut_diffusion=company.get("statut_diffusion"),-# Siege (Headquarters) informationsiege_activite_principale=siege.get("activite_principale"),- siege_activite_principale_registre_metier=siege.get("activite_principale_registre_metier"),- siege_annee_tranche_effectif_salarie=siege.get("annee_tranche_effectif_salarie"),+ siege_activite_principale_registre_metier=siege.get(+ "activite_principale_registre_metier"+ ),+ siege_annee_tranche_effectif_salarie=siege.get(+ "annee_tranche_effectif_salarie"+ ),siege_adresse=siege.get("adresse"),siege_caractere_employeur=siege.get("caractere_employeur"),siege_cedex=siege.get("cedex"),@@ -168,7 +190,9 @@ class IndividualToOrgScanner(Scanner):siege_liste_finess=siege.get("liste_finess"),siege_liste_id_bio=siege.get("liste_id_bio"),siege_liste_idcc=siege.get("liste_idcc"),- siege_liste_id_organisme_formation=siege.get("liste_id_organisme_formation"),+ siege_liste_id_organisme_formation=siege.get(+ "liste_id_organisme_formation"+ ),siege_liste_rge=siege.get("liste_rge"),siege_liste_uai=siege.get("liste_uai"),siege_longitude=siege.get("longitude"),@@ -176,51 +200,85 @@ class IndividualToOrgScanner(Scanner):siege_numero_voie=siege.get("numero_voie"),siege_region=siege.get("region"),siege_siret=siege.get("siret"),- siege_statut_diffusion_etablissement=siege.get("statut_diffusion_etablissement"),+ siege_statut_diffusion_etablissement=siege.get(+ "statut_diffusion_etablissement"+ ),siege_tranche_effectif_salarie=siege.get("tranche_effectif_salarie"),siege_type_voie=siege.get("type_voie"),-# Dirigeants (Leaders)dirigeants=dirigeants if dirigeants else None,-# Matching establishmentsmatching_etablissements=company.get("matching_etablissements"),-# Financial informationfinances=company.get("finances"),-# Complements (Additional information)- complements_collectivite_territoriale=company.get("complements", {}).get("collectivite_territoriale"),- complements_convention_collective_renseignee=company.get("complements", {}).get("convention_collective_renseignee"),+ complements_collectivite_territoriale=company.get(+ "complements", {}+ ).get("collectivite_territoriale"),+ complements_convention_collective_renseignee=company.get(+ "complements", {}+ ).get("convention_collective_renseignee"),complements_liste_idcc=company.get("complements", {}).get("liste_idcc"),- complements_egapro_renseignee=company.get("complements", {}).get("egapro_renseignee"),- complements_est_achats_responsables=company.get("complements", {}).get("est_achats_responsables"),- complements_est_alim_confiance=company.get("complements", {}).get("est_alim_confiance"),- complements_est_association=company.get("complements", {}).get("est_association"),+ complements_egapro_renseignee=company.get("complements", {}).get(+ "egapro_renseignee"+ ),+ complements_est_achats_responsables=company.get("complements", {}).get(+ "est_achats_responsables"+ ),+ complements_est_alim_confiance=company.get("complements", {}).get(+ "est_alim_confiance"+ ),+ complements_est_association=company.get("complements", {}).get(+ "est_association"+ ),complements_est_bio=company.get("complements", {}).get("est_bio"),- complements_est_entrepreneur_individuel=company.get("complements", {}).get("est_entrepreneur_individuel"),- complements_est_entrepreneur_spectacle=company.get("complements", {}).get("est_entrepreneur_spectacle"),+ complements_est_entrepreneur_individuel=company.get(+ "complements", {}+ ).get("est_entrepreneur_individuel"),+ complements_est_entrepreneur_spectacle=company.get(+ "complements", {}+ ).get("est_entrepreneur_spectacle"),complements_est_ess=company.get("complements", {}).get("est_ess"),complements_est_finess=company.get("complements", {}).get("est_finess"),- complements_est_organisme_formation=company.get("complements", {}).get("est_organisme_formation"),- complements_est_qualiopi=company.get("complements", {}).get("est_qualiopi"),- complements_liste_id_organisme_formation=company.get("complements", {}).get("liste_id_organisme_formation"),+ complements_est_organisme_formation=company.get("complements", {}).get(+ "est_organisme_formation"+ ),+ complements_est_qualiopi=company.get("complements", {}).get(+ "est_qualiopi"+ ),+ complements_liste_id_organisme_formation=company.get(+ "complements", {}+ ).get("liste_id_organisme_formation"),complements_est_rge=company.get("complements", {}).get("est_rge"),- complements_est_service_public=company.get("complements", {}).get("est_service_public"),+ complements_est_service_public=company.get("complements", {}).get(+ "est_service_public"+ ),complements_est_l100_3=company.get("complements", {}).get("est_l100_3"),complements_est_siae=company.get("complements", {}).get("est_siae"),- complements_est_societe_mission=company.get("complements", {}).get("est_societe_mission"),+ complements_est_societe_mission=company.get("complements", {}).get(+ "est_societe_mission"+ ),complements_est_uai=company.get("complements", {}).get("est_uai"),- complements_est_patrimoine_vivant=company.get("complements", {}).get("est_patrimoine_vivant"),- complements_bilan_ges_renseigne=company.get("complements", {}).get("bilan_ges_renseigne"),- complements_identifiant_association=company.get("complements", {}).get("identifiant_association"),- complements_statut_entrepreneur_spectacle=company.get("complements", {}).get("statut_entrepreneur_spectacle"),- complements_type_siae=company.get("complements", {}).get("type_siae")+ complements_est_patrimoine_vivant=company.get("complements", {}).get(+ "est_patrimoine_vivant"+ ),+ complements_bilan_ges_renseigne=company.get("complements", {}).get(+ "bilan_ges_renseigne"+ ),+ complements_identifiant_association=company.get("complements", {}).get(+ "identifiant_association"+ ),+ complements_statut_entrepreneur_spectacle=company.get(+ "complements", {}+ ).get("statut_entrepreneur_spectacle"),+ complements_type_siae=company.get("complements", {}).get("type_siae"),)except Exception as e:- Logger.error(self.sketch_id, {"message": f"Error enriching organization: {e}"})+ Logger.error(+ self.sketch_id, {"message": f"Error enriching organization: {e}"}+ )return None-+def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:if not self.neo4j_conn:return results@@ -228,22 +286,26 @@ class IndividualToOrgScanner(Scanner):for org in results:# Create or update the organization node with all SIRENE propertiesorg_key = f"{org.name}_FR"- self.create_node('Organization', 'org_id', org_key,- name=org.name,- country="FR",- siren=org.siren,- siege_siret=org.siege_siret,- nom_complet=org.nom_complet,- nom_raison_sociale=org.nom_raison_sociale,- sigle=org.sigle,- caption=org.name, type='organization')+ self.create_node(+ "Organization",+ "org_id",+ org_key,+ name=org.name,+ country="FR",+ siren=org.siren,+ siege_siret=org.siege_siret,+ nom_complet=org.nom_complet,+ nom_raison_sociale=org.nom_raison_sociale,+ sigle=org.sigle,+ caption=org.name,+ type="organization",+ )self.log_graph_message(f"Found organization: {org.name}")return results+# Make types available at module level for easy accessInputType = IndividualToOrgScanner.InputTypeOutputType = IndividualToOrgScanner.OutputType--diff --git a/flowsint-transforms/src/flowsint_transforms/ips/asn_to_cidrs.py b/flowsint-transforms/src/flowsint_transforms/ips/asn_to_cidrs.pyindex e14ba853..54bfb9e6 100644--- a/flowsint-transforms/src/flowsint_transforms/ips/asn_to_cidrs.py+++ b/flowsint-transforms/src/flowsint_transforms/ips/asn_to_cidrs.py@@ -7,6 +7,7 @@ from flowsint_types.asn import ASNfrom flowsint_core.utils import is_valid_asn, parse_asnfrom flowsint_core.core.logger import Logger+class AsnToCidrsScanner(Scanner):"""Takes an ASN and returns its corresponding CIDRs."""@@ -21,12 +22,14 @@ class AsnToCidrsScanner(Scanner):@classmethoddef category(cls) -> str:return "Asn"-+@classmethoddef key(cls) -> str:- return 'network'+ return "network"- def preprocess(self, data: Union[List[str], List[int], List[dict], InputType]) -> InputType:+ def preprocess(+ self, data: Union[List[str], List[int], List[dict], InputType]+ ) -> InputType:cleaned: InputType = []for item in data:asn_obj = None@@ -60,21 +63,24 @@ class AsnToCidrsScanner(Scanner):cidrs.append(cidr)asn_cidrs.append(cidr)except Exception as e:- Logger.error(self.sketch_id, {"message": f"Failed to parse CIDR {cidr_str}: {str(e)}"})+ Logger.error(+ self.sketch_id,+ {"message": f"Failed to parse CIDR {cidr_str}: {str(e)}"},+ )else:- Logger.warn(self.sketch_id, {"message": f"No CIDRs found for ASN {asn.number}"})-+ Logger.warn(+ self.sketch_id, {"message": f"No CIDRs found for ASN {asn.number}"}+ )+if asn_cidrs: # Only add to mapping if we found valid CIDRsself._asn_to_cidrs_map.append((asn, asn_cidrs))return cidrs-+def __get_cidrs_from_asn(self, asn: int) -> Dict[str, Any]:try:command = f"echo {asn} | asnmap -silent -json | jq -s '.'"result = subprocess.run(- command,- shell=True,- capture_output=True, text=True, timeout=60+ command, shell=True, capture_output=True, text=True, timeout=60)if not result.stdout.strip():Logger.info(self.sketch_id, {"message": f"No CIDRs found for {asn}."})@@ -89,7 +95,7 @@ class AsnToCidrsScanner(Scanner):"as_range": [],"as_name": None,"as_country": None,- "as_number": None+ "as_number": None,}for data in data_array:@@ -105,35 +111,58 @@ class AsnToCidrsScanner(Scanner):return combined_data if combined_data["as_range"] else Noneexcept json.JSONDecodeError:- Logger.error(self.sketch_id, {"message": f"Failed to parse JSON from asnmap output: {result.stdout}"})+ Logger.error(+ self.sketch_id,+ {+ "message": f"Failed to parse JSON from asnmap output: {result.stdout}"+ },+ )return Noneexcept Exception as e:- Logger.error(self.sketch_id, {"message": f"asnmap exception for {asn}: {str(e)}"})+ Logger.error(+ self.sketch_id, {"message": f"asnmap exception for {asn}: {str(e)}"}+ )return Nonedef postprocess(self, results: OutputType, original_input: InputType) -> OutputType:# Create Neo4j relationships between ASNs and their corresponding CIDRs# Use the mapping from scan if available, else fallback to zip- asn_to_cidrs = getattr(self, '_asn_to_cidrs_map', None)+ asn_to_cidrs = getattr(self, "_asn_to_cidrs_map", None)if asn_to_cidrs is not None:for asn, cidr_list in asn_to_cidrs:for cidr in cidr_list:if str(cidr.network) == "0.0.0.0/0":continue # Skip default CIDR for unknown ASNif self.neo4j_conn:- self.create_node('asn', 'number', asn.number,- name=asn.name or "Unknown",- country=asn.country or "Unknown",- label=f"AS{asn.number}",- caption=f"AS{asn.number} - {asn.name or 'Unknown'}",- type='asn')-- self.create_node('cidr', 'network', str(cidr.network),- caption=str(cidr.network), type='cidr')-- self.create_relationship('asn', 'number', asn.number,- 'cidr', 'network', str(cidr.network), 'ANNOUNCES')+ self.create_node(+ "asn",+ "number",+ asn.number,+ name=asn.name or "Unknown",+ country=asn.country or "Unknown",+ label=f"AS{asn.number}",+ caption=f"AS{asn.number} - {asn.name or 'Unknown'}",+ type="asn",+ )++ self.create_node(+ "cidr",+ "network",+ str(cidr.network),+ caption=str(cidr.network),+ type="cidr",+ )++ self.create_relationship(+ "asn",+ "number",+ asn.number,+ "cidr",+ "network",+ str(cidr.network),+ "ANNOUNCES",+ )else:# Fallback: original behavior (one-to-one zip)for asn, cidr in zip(original_input, results):@@ -141,20 +170,37 @@ class AsnToCidrsScanner(Scanner):continue # Skip default CIDR for unknown ASNself.log_graph_message(f"ASN {asn.number} -> {cidr.network}")if self.neo4j_conn:- self.create_node('ASN', 'number', asn.number,- name=asn.name or "Unknown",- country=asn.country or "Unknown",- label=f"AS{asn.number}",- caption=f"AS{asn.number} - {asn.name or 'Unknown'}",- type='asn')-- self.create_node('CIDR', 'network', str(cidr.network),- caption=str(cidr.network), type='cidr')-- self.create_relationship('ASN', 'number', asn.number,- 'CIDR', 'network', str(cidr.network), 'ANNOUNCES')+ self.create_node(+ "ASN",+ "number",+ asn.number,+ name=asn.name or "Unknown",+ country=asn.country or "Unknown",+ label=f"AS{asn.number}",+ caption=f"AS{asn.number} - {asn.name or 'Unknown'}",+ type="asn",+ )++ self.create_node(+ "CIDR",+ "network",+ str(cidr.network),+ caption=str(cidr.network),+ type="cidr",+ )++ self.create_relationship(+ "ASN",+ "number",+ asn.number,+ "CIDR",+ "network",+ str(cidr.network),+ "ANNOUNCES",+ )return results+# Make types available at module level for easy accessInputType = AsnToCidrsScanner.InputType-OutputType = AsnToCidrsScanner.OutputType\ No newline at end of file+OutputType = AsnToCidrsScanner.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/ips/cidr_to_ips.py b/flowsint-transforms/src/flowsint_transforms/ips/cidr_to_ips.pyindex 650d9cd1..130f7a6a 100644--- a/flowsint-transforms/src/flowsint_transforms/ips/cidr_to_ips.py+++ b/flowsint-transforms/src/flowsint_transforms/ips/cidr_to_ips.py@@ -5,6 +5,7 @@ from flowsint_types.cidr import CIDRfrom flowsint_types.ip import Ipfrom flowsint_core.core.logger import Logger+class CidrToIpsScanner(Scanner):"""Takes a CIDR and returns its corresponding IP addresses."""@@ -49,31 +50,36 @@ class CidrToIpsScanner(Scanner):ip = Ip(address=ip_str.strip())ips.append(ip)except Exception as e:- Logger.error(self.sketch_id, {"message": f"Failed to parse IP {ip_str}: {str(e)}"})+ Logger.error(+ self.sketch_id,+ {"message": f"Failed to parse IP {ip_str}: {str(e)}"},+ )else:- Logger.warn(self.sketch_id, {"message": f"No IPs found for CIDR {cidr.network}"})+ Logger.warn(+ self.sketch_id, {"message": f"No IPs found for CIDR {cidr.network}"}+ )return ips-+def __get_ips_from_cidr(self, cidr: str) -> List[str]:try:command = f"echo {cidr} | dnsx -ptr"result = subprocess.run(- command,- shell=True,- capture_output=True,- text=True,- timeout=60+ command, shell=True, capture_output=True, text=True, timeout=60)if not result.stdout.strip():Logger.info(self.sketch_id, {"message": f"No IPs found for {cidr}."})return []-+# Split the output by newlines and filter out empty lines- ip_addresses = [ip.strip() for ip in result.stdout.split('\n') if ip.strip()]+ ip_addresses = [+ ip.strip() for ip in result.stdout.split("\n") if ip.strip()+ ]return ip_addressesexcept Exception as e:- Logger.error(self.sketch_id, {"message": f"dnsx exception for {cidr}: {str(e)}"})+ Logger.error(+ self.sketch_id, {"message": f"dnsx exception for {cidr}: {str(e)}"}+ )return []def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:@@ -81,20 +87,34 @@ class CidrToIpsScanner(Scanner):for cidr, ip in zip(original_input, results):if self.neo4j_conn:# Create CIDR node- self.create_node('cidr', 'network', str(cidr.network),- caption=str(cidr.network), type="cidr")-+ self.create_node(+ "cidr",+ "network",+ str(cidr.network),+ caption=str(cidr.network),+ type="cidr",+ )+# Create IP node- self.create_node('ip', 'address', ip.address,- caption=ip.address, type="ip")-+ self.create_node(+ "ip", "address", ip.address, caption=ip.address, type="ip"+ )+# Create relationship- self.create_relationship('cidr', 'network', str(cidr.network),- 'ip', 'address', ip.address, 'CONTAINS')-+ self.create_relationship(+ "cidr",+ "network",+ str(cidr.network),+ "ip",+ "address",+ ip.address,+ "CONTAINS",+ )+self.log_graph_message(f"Found {len(results)} IPs for CIDR {cidr.network}")return results+# Make types available at module level for easy accessInputType = CidrToIpsScanner.InputType-OutputType = CidrToIpsScanner.OutputType\ No newline at end of file+OutputType = CidrToIpsScanner.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/ips/geolocation.py b/flowsint-transforms/src/flowsint_transforms/ips/geolocation.pyindex 368adf89..332b2060 100644--- a/flowsint-transforms/src/flowsint_transforms/ips/geolocation.py+++ b/flowsint-transforms/src/flowsint_transforms/ips/geolocation.py@@ -4,13 +4,14 @@ from pydantic import TypeAdapterfrom flowsint_core.core.scanner_base import Scannerfrom flowsint_types.ip import Ip, Ipfrom flowsint_core.utils import resolve_type, is_valid_ip+InputType: TypeAlias = List[Ip]OutputType: TypeAlias = List[Ip]+class GeolocationScanner(Scanner):"""Get geolocation data for IP addresses."""-# Define types as class attributes - base class handles schema generation automaticallyInputType = List[Ip]OutputType = List[Ip]@@ -22,7 +23,7 @@ class GeolocationScanner(Scanner):@classmethoddef category(cls) -> str:return "Ip"-+@classmethoddef key(cls) -> str:return "address"@@ -37,7 +38,7 @@ class GeolocationScanner(Scanner):"properties": [{"name": prop, "type": resolve_type(info, schema)}for prop, info in details["properties"].items()- ]+ ],}@classmethod@@ -50,7 +51,7 @@ class GeolocationScanner(Scanner):"properties": [{"name": prop, "type": resolve_type(info, schema)}for prop, info in details["properties"].items()- ]+ ],}def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:@@ -89,10 +90,20 @@ class GeolocationScanner(Scanner):"""Update IP nodes in Neo4j with geolocation information."""if self.neo4j_conn:for ip in results:- self.create_node('ip', 'address', ip.address,- latitude=ip.latitude, longitude=ip.longitude,- country=ip.country, city=ip.city, isp=ip.isp, type='ip')- self.log_graph_message(f"Geolocated {ip.address} to {ip.city}, {ip.country} (lat: {ip.latitude}, lon: {ip.longitude})")+ self.create_node(+ "ip",+ "address",+ ip.address,+ latitude=ip.latitude,+ longitude=ip.longitude,+ country=ip.country,+ city=ip.city,+ isp=ip.isp,+ type="ip",+ )+ self.log_graph_message(+ f"Geolocated {ip.address} to {ip.city}, {ip.country} (lat: {ip.latitude}, lon: {ip.longitude})"+ )return resultsdef get_location_data(self, address: str) -> Dict[str, Any]:@@ -112,10 +123,13 @@ class GeolocationScanner(Scanner):"isp": data.get("isp"),}else:- raise ValueError(f"Geolocation failed for {address}: {data.get('message')}")+ raise ValueError(+ f"Geolocation failed for {address}: {data.get('message')}"+ )except Exception as e:print(f"Failed to geolocate {address}: {e}")return {}+InputType = GeolocationScanner.InputType-OutputType = GeolocationScanner.OutputType\ No newline at end of file+OutputType = GeolocationScanner.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/ips/ip_to_asn.py b/flowsint-transforms/src/flowsint_transforms/ips/ip_to_asn.pyindex 97ffd52c..85c59210 100644--- a/flowsint-transforms/src/flowsint_transforms/ips/ip_to_asn.py+++ b/flowsint-transforms/src/flowsint_transforms/ips/ip_to_asn.py@@ -8,6 +8,7 @@ from flowsint_types.asn import ASNfrom flowsint_core.utils import is_valid_ipfrom flowsint_core.core.logger import Logger+class IpToAsnScanner(Scanner):"""Takes an IP address and returns its corresponding ASN."""@@ -22,7 +23,7 @@ class IpToAsnScanner(Scanner):@classmethoddef category(cls) -> str:return "Ip"-+@classmethoddef key(cls) -> str:return "address"@@ -45,7 +46,7 @@ class IpToAsnScanner(Scanner):async def scan(self, data: InputType) -> OutputType:results: OutputType = []-+for ip in data:try:# Use asnmap to get ASN info@@ -53,48 +54,59 @@ class IpToAsnScanner(Scanner):["asnmap", "-a", ip.address, "-json"],capture_output=True,text=True,- timeout=30+ timeout=30,)-+if result.returncode == 0:output = result.stdout.strip()if output:asn_data = json.loads(output)- if asn_data and 'as_number' in asn_data:+ if asn_data and "as_number" in asn_data:asn = ASN(- asn=str(asn_data['as_number']),- name=asn_data.get('as_name', ''),- org=asn_data.get('as_org', ''),- country=asn_data.get('as_country', '')+ asn=str(asn_data["as_number"]),+ name=asn_data.get("as_name", ""),+ org=asn_data.get("as_org", ""),+ country=asn_data.get("as_country", ""),)results.append(asn)-+except Exception as e:- Logger.error(self.sketch_id, {"message": f"Error getting ASN for IP {ip.address}: {e}"})+ Logger.error(+ self.sketch_id,+ {"message": f"Error getting ASN for IP {ip.address}: {e}"},+ )continue-+return results- def postprocess(self, results: OutputType, input_data: InputType = None) -> OutputType:+ def postprocess(+ self, results: OutputType, input_data: InputType = None+ ) -> OutputType:# Create Neo4j relationships between IPs and their corresponding ASNsif input_data and self.neo4j_conn:for ip, asn in zip(input_data, results):# Create IP node- self.create_node('ip', 'address', ip.address,- caption=ip.address, type="ip")-+ self.create_node(+ "ip", "address", ip.address, caption=ip.address, type="ip"+ )+# Create ASN node- self.create_node('asn', 'network', asn.asn,- caption=f"AS{asn.asn}", type="asn")-+ self.create_node(+ "asn", "network", asn.asn, caption=f"AS{asn.asn}", type="asn"+ )+# Create relationship- self.create_relationship('ip', 'address', ip.address,- 'asn', 'network', asn.asn, 'BELONGS_TO')-- self.log_graph_message(f"IP {ip.address} belongs to AS{asn.asn} ({asn.name})")-+ self.create_relationship(+ "ip", "address", ip.address, "asn", "network", asn.asn, "BELONGS_TO"+ )++ self.log_graph_message(+ f"IP {ip.address} belongs to AS{asn.asn} ({asn.name})"+ )+return results+# Make types available at module level for easy accessInputType = IpToAsnScanner.InputType-OutputType = IpToAsnScanner.OutputType\ No newline at end of file+OutputType = IpToAsnScanner.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/ips/reverse_resolve.py b/flowsint-transforms/src/flowsint_transforms/ips/reverse_resolve.pyindex 239ad228..daa88388 100644--- a/flowsint-transforms/src/flowsint_transforms/ips/reverse_resolve.py+++ b/flowsint-transforms/src/flowsint_transforms/ips/reverse_resolve.py@@ -10,6 +10,7 @@ from flowsint_core.utils import is_valid_ipPTR_BLACKLIST = re.compile(r"^ip\d+\.ip-\d+-\d+-\d+-\d+\.")+class ReverseResolveScanner(Scanner):"""Resolve IP addresses to domain names using PTR, Certificate Transparency and optional API calls."""@@ -47,7 +48,7 @@ class ReverseResolveScanner(Scanner):async def scan(self, data: InputType) -> OutputType:results: OutputType = []-+for ip in data:try:# Try PTR lookup@@ -59,7 +60,7 @@ class ReverseResolveScanner(Scanner):continueexcept socket.herror:pass-+# Try Certificate Transparency logstry:ct_url = f"https://crt.sh/?q={ip.address}&output=json"@@ -74,16 +75,22 @@ class ReverseResolveScanner(Scanner):breakexcept Exception:pass-+except Exception as e:- Logger.error(self.sketch_id, {"message": f"Error reverse resolving IP {ip.address}: {e}"})+ Logger.error(+ self.sketch_id,+ {"message": f"Error reverse resolving IP {ip.address}: {e}"},+ )continue-+return results- def postprocess(self, results: OutputType, input_data: InputType = None) -> OutputType:+ def postprocess(+ self, results: OutputType, input_data: InputType = None+ ) -> OutputType:return results+# Make types available at module level for easy accessInputType = ReverseResolveScanner.InputType-OutputType = ReverseResolveScanner.OutputType\ No newline at end of file+OutputType = ReverseResolveScanner.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/leaks/hibp.py b/flowsint-transforms/src/flowsint_transforms/leaks/hibp.pyindex b5a89a89..2ac37d72 100644--- a/flowsint-transforms/src/flowsint_transforms/leaks/hibp.py+++ b/flowsint-transforms/src/flowsint_transforms/leaks/hibp.py@@ -10,6 +10,7 @@ load_dotenv()HIBP_API_KEY = os.getenv("HIBP_API_KEY")+class HibpScanner(Scanner):"""Queries HaveIBeenPwned for potential leaks."""@@ -20,11 +21,11 @@ class HibpScanner(Scanner):@classmethoddef name(cls) -> str:return "hibp_scanner"-+@classmethoddef category(cls) -> str:return "leaks"-+@classmethoddef key(cls) -> str:return "email"@@ -62,12 +63,17 @@ class HibpScanner(Scanner):}results.append(email_result)except Exception as e:- results.append({- "email": email,- "error": f"Error during scan: {str(e)}",- })- Logger.error(self.sketch_id, {"message": f"Error scanning email {email}: {str(e)}"})-+ results.append(+ {+ "email": email,+ "error": f"Error during scan: {str(e)}",+ }+ )+ Logger.error(+ self.sketch_id,+ {"message": f"Error scanning email {email}: {str(e)}"},+ )+return resultsdef postprocess(self, results: OutputType, original_input: InputType) -> OutputType:@@ -78,23 +84,37 @@ class HibpScanner(Scanner):for result in results:if "error" not in result:email = result["email"]-+# Create email node- self.create_node('email', 'address', email,- caption=email, type='email')-+ self.create_node("email", "address", email, caption=email, type="email")+# Create breach relationshipsfor breach in result.get("breaches", []):if breach and isinstance(breach, dict):breach_name = breach.get("Name", "Unknown")- self.create_node('breach', 'name', breach_name,- caption=breach_name, type='breach')- self.create_relationship('email', 'address', email,- 'breach', 'name', breach_name, 'FOUND_IN_BREACH')- self.log_graph_message(f"Email {email} found in breach: {breach_name}")+ self.create_node(+ "breach",+ "name",+ breach_name,+ caption=breach_name,+ type="breach",+ )+ self.create_relationship(+ "email",+ "address",+ email,+ "breach",+ "name",+ breach_name,+ "FOUND_IN_BREACH",+ )+ self.log_graph_message(+ f"Email {email} found in breach: {breach_name}"+ )return results+# Make types available at module level for easy accessInputType = HibpScanner.InputTypeOutputType = HibpScanner.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/n8n/connector.py b/flowsint-transforms/src/flowsint_transforms/n8n/connector.pyindex 2840caa4..36089513 100644--- a/flowsint-transforms/src/flowsint_transforms/n8n/connector.py+++ b/flowsint-transforms/src/flowsint_transforms/n8n/connector.py@@ -5,31 +5,32 @@ from flowsint_core.core.scanner_base import Scannerfrom flowsint_core.core.logger import Loggerfrom flowsint_core.core.graph_db import Neo4jConnection+class N8nConnector(Scanner):"""Connect to your custom n8n workflows to process data through webhooks."""-+# Define types as class attributes - base class handles schema generation automaticallyInputType = List[Any]OutputType = List[Any]-+def __init__(- self,- sketch_id: Optional[str] = None,- scan_id: Optional[str] = None,- neo4j_conn: Optional[Neo4jConnection] = None,- vault=None,- params: Optional[Dict[str, Any]] = None- ):- super().__init__(- sketch_id=sketch_id,- scan_id=scan_id,- neo4j_conn=neo4j_conn,- params_schema=self.get_params_schema(),- vault=vault,- params=params- )+ self,+ sketch_id: Optional[str] = None,+ scan_id: Optional[str] = None,+ neo4j_conn: Optional[Neo4jConnection] = None,+ vault=None,+ params: Optional[Dict[str, Any]] = None,+ ):+ super().__init__(+ sketch_id=sketch_id,+ scan_id=scan_id,+ neo4j_conn=neo4j_conn,+ params_schema=self.get_params_schema(),+ vault=vault,+ params=params,+ )@classmethoddef documentation(cls) -> str:@@ -223,7 +224,7 @@ Check FlowSint logs for detailed debugging information.@classmethoddef category(cls) -> str:return "external"-+@classmethoddef required_params(cls) -> bool:return True@@ -245,14 +246,14 @@ Check FlowSint logs for detailed debugging information."name": "auth_token","type": "vaultSecret","required": False,- "description": "Optional authentication token for the webhook"+ "description": "Optional authentication token for the webhook",},{"name": "extra_payload","type": "string","required": False,- "description": "Optional JSON string with additional data to include in the payload"- }+ "description": "Optional JSON string with additional data to include in the payload",+ },]async def scan(self, values: InputType) -> OutputType:@@ -266,7 +267,7 @@ Check FlowSint logs for detailed debugging information.payload = {"sketch_id": self.sketch_id,"type": values[0] if values else None,- "inputs": values+ "inputs": values,}if "extra_payload" in params and params["extra_payload"] is not None:@@ -274,41 +275,83 @@ Check FlowSint logs for detailed debugging information.extra = json.loads(params["extra_payload"])payload.update(extra)except json.JSONDecodeError:- Logger.warn(self.sketch_id, {"message": "extra_payload is not valid JSON"})+ Logger.warn(+ self.sketch_id, {"message": "extra_payload is not valid JSON"}+ )- Logger.info(self.sketch_id, {"message": f"Sending request to n8n webhook with payload: {json.dumps(payload)}"})+ Logger.info(+ self.sketch_id,+ {+ "message": f"Sending request to n8n webhook with payload: {json.dumps(payload)}"+ },+ )try:async with aiohttp.ClientSession() as session:async with session.post(url, headers=headers, json=payload) as response:- Logger.info(self.sketch_id, {"message": f"n8n webhook responded with status: {response.status}"})-+ Logger.info(+ self.sketch_id,+ {+ "message": f"n8n webhook responded with status: {response.status}"+ },+ )+# Log the raw response text for debuggingresponse_text = await response.text()- Logger.info(self.sketch_id, {"message": f"n8n webhook raw response: {response_text}"})-+ Logger.info(+ self.sketch_id,+ {"message": f"n8n webhook raw response: {response_text}"},+ )+if response.status != 200:- Logger.warn(self.sketch_id, {"message": f"n8n responded with non-200 status: {response.status} - Response: {response_text}"})- raise Exception(f"n8n responded with {response.status}: {response_text}")-+ Logger.warn(+ self.sketch_id,+ {+ "message": f"n8n responded with non-200 status: {response.status} - Response: {response_text}"+ },+ )+ raise Exception(+ f"n8n responded with {response.status}: {response_text}"+ )+try:data = json.loads(response_text)- Logger.info(self.sketch_id, {"message": f"n8n connector received response: {json.dumps(data)}"})+ Logger.info(+ self.sketch_id,+ {+ "message": f"n8n connector received response: {json.dumps(data)}"+ },+ )return dataexcept json.JSONDecodeError as e:- Logger.warn(self.sketch_id, {"message": f"Failed to parse n8n response as JSON: {str(e)} - Raw response: {response_text}"})+ Logger.warn(+ self.sketch_id,+ {+ "message": f"Failed to parse n8n response as JSON: {str(e)} - Raw response: {response_text}"+ },+ )# Return the raw text wrapped in a list of dicts as expected- return [{"raw_response": response_text, "error": "Response was not valid JSON"}]-+ return [+ {+ "raw_response": response_text,+ "error": "Response was not valid JSON",+ }+ ]+except Exception as e:- Logger.warn(self.sketch_id, {"message": f"Error calling n8n webhook: {str(e)}"})+ Logger.warn(+ self.sketch_id, {"message": f"Error calling n8n webhook: {str(e)}"}+ )# Re-raise the exception so the caller knows something went wrongraise-+def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:- Logger.success(self.sketch_id, {"message": f"n8n connector results: {json.dumps(results)}"})+ Logger.success(+ self.sketch_id, {"message": f"n8n connector results: {json.dumps(results)}"}+ )return results+# Make types available at module level for easy accessInputType = N8nConnector.InputTypeOutputType = N8nConnector.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/organizations/__init__.py b/flowsint-transforms/src/flowsint_transforms/organizations/__init__.pyindex e69de29b..8b137891 100644--- a/flowsint-transforms/src/flowsint_transforms/organizations/__init__.py+++ b/flowsint-transforms/src/flowsint_transforms/organizations/__init__.py@@ -0,0 +1 @@+diff --git a/flowsint-transforms/src/flowsint_transforms/organizations/org_to_asn.py b/flowsint-transforms/src/flowsint_transforms/organizations/org_to_asn.pyindex e0f31b25..ff93f57b 100644--- a/flowsint-transforms/src/flowsint_transforms/organizations/org_to_asn.py+++ b/flowsint-transforms/src/flowsint_transforms/organizations/org_to_asn.py@@ -6,6 +6,7 @@ from flowsint_types.organization import Organizationfrom flowsint_types.asn import ASNfrom flowsint_core.core.logger import Logger+class OrgToAsnScanner(Scanner):"""Takes an organization and returns its corresponding ASN."""@@ -20,7 +21,7 @@ class OrgToAsnScanner(Scanner):@classmethoddef category(cls) -> str:return "Organization"-+@classmethoddef key(cls) -> str:return "name"@@ -46,24 +47,26 @@ class OrgToAsnScanner(Scanner):for org in data:asn_data = self.__get_asn_from_asnmap(org.name)if asn_data:- asns.append(ASN(- number=int(asn_data["as_number"].lstrip("AS")),- name=asn_data["as_name"],- country=asn_data["as_country"],- cidrs=[]- ))+ asns.append(+ ASN(+ number=int(asn_data["as_number"].lstrip("AS")),+ name=asn_data["as_name"],+ country=asn_data["as_country"],+ cidrs=[],+ )+ )else:- Logger.info(self.sketch_id, {"message": f"No ASN found for org {org.name}."})+ Logger.info(+ self.sketch_id, {"message": f"No ASN found for org {org.name}."}+ )return asns-+def __get_asn_from_asnmap(self, name: str) -> Dict[str, Any]:try:# Properly run the shell pipeline using shell=Truecommand = f"echo {name} | asnmap -silent -json | jq -s"result = subprocess.run(- command,- shell=True,- capture_output=True, text=True, timeout=60+ command, shell=True, capture_output=True, text=True, timeout=60)if not result.stdout.strip():return None@@ -77,7 +80,7 @@ class OrgToAsnScanner(Scanner):"as_range": [],"as_name": None,"as_country": None,- "as_number": None+ "as_number": None,}for data in data_array:@@ -93,11 +96,19 @@ class OrgToAsnScanner(Scanner):return combined_data if combined_data["as_number"] else Noneexcept json.JSONDecodeError as e:- Logger.error(self.sketch_id, {"message": f"An error occurred while parsing the JSON output from asnmap: {str(e)}"})+ Logger.error(+ self.sketch_id,+ {+ "message": f"An error occurred while parsing the JSON output from asnmap: {str(e)}"+ },+ )return Noneexcept Exception as e:- Logger.error(self.sketch_id, {"message": f"An error occurred while running asnmap: {str(e)}"})+ Logger.error(+ self.sketch_id,+ {"message": f"An error occurred while running asnmap: {str(e)}"},+ )return Nonedef postprocess(self, results: OutputType, original_input: InputType) -> OutputType:@@ -106,28 +117,47 @@ class OrgToAsnScanner(Scanner):# Skip if no valid ASN was foundif result_asn.number == 0:continue-+if self.neo4j_conn:# Create organization node- self.create_node('organization', 'name', input_org.name,- caption=input_org.name, type="organization")-+ self.create_node(+ "organization",+ "name",+ input_org.name,+ caption=input_org.name,+ type="organization",+ )+# Create ASN node- self.create_node('asn', 'number', result_asn.number,- name=result_asn.name,- country=result_asn.country,- label=f"AS{result_asn.number}",- caption=f"AS{result_asn.number} - {result_asn.name}",- type="asn")-+ self.create_node(+ "asn",+ "number",+ result_asn.number,+ name=result_asn.name,+ country=result_asn.country,+ label=f"AS{result_asn.number}",+ caption=f"AS{result_asn.number} - {result_asn.name}",+ type="asn",+ )+# Create relationship- self.create_relationship('organization', 'name', input_org.name,- 'asn', 'number', result_asn.number, 'BELONGS_TO')-- self.log_graph_message(f"Found for {input_org.name} -> ASN {result_asn.number}")+ self.create_relationship(+ "organization",+ "name",+ input_org.name,+ "asn",+ "number",+ result_asn.number,+ "BELONGS_TO",+ )++ self.log_graph_message(+ f"Found for {input_org.name} -> ASN {result_asn.number}"+ )return results+# Make types available at module level for easy accessInputType = OrgToAsnScanner.InputType-OutputType = OrgToAsnScanner.OutputType\ No newline at end of file+OutputType = OrgToAsnScanner.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/organizations/to_infos.py b/flowsint-transforms/src/flowsint_transforms/organizations/to_infos.pyindex 9af1afd2..a9be4cf6 100644--- a/flowsint-transforms/src/flowsint_transforms/organizations/to_infos.py+++ b/flowsint-transforms/src/flowsint_transforms/organizations/to_infos.py@@ -19,7 +19,7 @@ class OrgToInfosScanner(Scanner):@classmethoddef category(cls) -> str:return "Organization"-+@classmethoddef key(cls) -> str:return "name"@@ -38,19 +38,22 @@ class OrgToInfosScanner(Scanner):return cleanedasync def scan(self, data: InputType) -> OutputType:-+results: OutputType = []for org in data:try:sirene = SireneTool()raw_orgs = sirene.launch(org.name, limit=25)- if len(raw_orgs)> 0:+ if len(raw_orgs) > 0:for org_dict in raw_orgs:enriched_org = self.enrich_org(org_dict)if enriched_org is not None:results.append(enriched_org)except Exception as e:- Logger.error(self.sketch_id, {"message": f"Error enriching organization {org.name}: {e}"})+ Logger.error(+ self.sketch_id,+ {"message": f"Error enriching organization {org.name}: {e}"},+ )return resultsdef enrich_org(self, company: Dict) -> Organization:@@ -61,35 +64,40 @@ class OrgToInfosScanner(Scanner):siege_geo_adresse = Noneif siege.get("latitude") and siege.get("longitude"):from flowsint_types.address import PhysicalAddress+siege_geo_adresse = PhysicalAddress(address=siege.get("adresse", ""),city=siege.get("libelle_commune", ""),country="FR", # SIRENE is French registryzip=siege.get("code_postal", ""),latitude=float(siege.get("latitude", 0)),- longitude=float(siege.get("longitude", 0))+ longitude=float(siege.get("longitude", 0)),)-+# Extract dirigeants and convert to Individual objectsdirigeants = []for dirigeant_data in company.get("dirigeants", []):from flowsint_types.individual import Individual+dirigeant = Individual(first_name=dirigeant_data.get("prenoms", ""),last_name=dirigeant_data.get("nom", ""),full_name=f"{dirigeant_data.get('prenoms', '')} {dirigeant_data.get('nom', '')}".strip(),birth_date=dirigeant_data.get("date_de_naissance"),- gender=None # Not available in SIRENE data+ gender=None, # Not available in SIRENE data)dirigeants.append(dirigeant)-+name = company.get("nom_raison_sociale") or company.get("nom_complet")-+# Ensure we have a valid nameif not name:- Logger.error(self.sketch_id, {"message": f"Organization has no valid name: {company}"})+ Logger.error(+ self.sketch_id,+ {"message": f"Organization has no valid name: {company}"},+ )return None-+return Organization(# Basic informationname=name,@@ -97,40 +105,41 @@ class OrgToInfosScanner(Scanner):nom_complet=company.get("nom_complet"),nom_raison_sociale=company.get("nom_raison_sociale"),sigle=company.get("sigle"),-# Establishment informationnombre_etablissements=company.get("nombre_etablissements"),- nombre_etablissements_ouverts=company.get("nombre_etablissements_ouverts"),-+ nombre_etablissements_ouverts=company.get(+ "nombre_etablissements_ouverts"+ ),# Activity informationactivite_principale=company.get("activite_principale"),section_activite_principale=company.get("section_activite_principale"),-# Company categorycategorie_entreprise=company.get("categorie_entreprise"),annee_categorie_entreprise=company.get("annee_categorie_entreprise"),-# Employment informationcaractere_employeur=company.get("caractere_employeur"),tranche_effectif_salarie=company.get("tranche_effectif_salarie"),- annee_tranche_effectif_salarie=company.get("annee_tranche_effectif_salarie"),-+ annee_tranche_effectif_salarie=company.get(+ "annee_tranche_effectif_salarie"+ ),# Datesdate_creation=company.get("date_creation"),date_fermeture=company.get("date_fermeture"),date_mise_a_jour=company.get("date_mise_a_jour"),date_mise_a_jour_insee=company.get("date_mise_a_jour_insee"),date_mise_a_jour_rne=company.get("date_mise_a_jour_rne"),-# Legal informationnature_juridique=company.get("nature_juridique"),etat_administratif=company.get("etat_administratif"),statut_diffusion=company.get("statut_diffusion"),-# Siege (Headquarters) informationsiege_activite_principale=siege.get("activite_principale"),- siege_activite_principale_registre_metier=siege.get("activite_principale_registre_metier"),- siege_annee_tranche_effectif_salarie=siege.get("annee_tranche_effectif_salarie"),+ siege_activite_principale_registre_metier=siege.get(+ "activite_principale_registre_metier"+ ),+ siege_annee_tranche_effectif_salarie=siege.get(+ "annee_tranche_effectif_salarie"+ ),siege_adresse=siege.get("adresse"),siege_caractere_employeur=siege.get("caractere_employeur"),siege_cedex=siege.get("cedex"),@@ -162,7 +171,9 @@ class OrgToInfosScanner(Scanner):siege_liste_finess=siege.get("liste_finess"),siege_liste_id_bio=siege.get("liste_id_bio"),siege_liste_idcc=siege.get("liste_idcc"),- siege_liste_id_organisme_formation=siege.get("liste_id_organisme_formation"),+ siege_liste_id_organisme_formation=siege.get(+ "liste_id_organisme_formation"+ ),siege_liste_rge=siege.get("liste_rge"),siege_liste_uai=siege.get("liste_uai"),siege_longitude=siege.get("longitude"),@@ -170,51 +181,85 @@ class OrgToInfosScanner(Scanner):siege_numero_voie=siege.get("numero_voie"),siege_region=siege.get("region"),siege_siret=siege.get("siret"),- siege_statut_diffusion_etablissement=siege.get("statut_diffusion_etablissement"),+ siege_statut_diffusion_etablissement=siege.get(+ "statut_diffusion_etablissement"+ ),siege_tranche_effectif_salarie=siege.get("tranche_effectif_salarie"),siege_type_voie=siege.get("type_voie"),-# Dirigeants (Leaders)dirigeants=dirigeants if dirigeants else None,-# Matching establishmentsmatching_etablissements=company.get("matching_etablissements"),-# Financial informationfinances=company.get("finances"),-# Complements (Additional information)- complements_collectivite_territoriale=company.get("complements", {}).get("collectivite_territoriale"),- complements_convention_collective_renseignee=company.get("complements", {}).get("convention_collective_renseignee"),+ complements_collectivite_territoriale=company.get(+ "complements", {}+ ).get("collectivite_territoriale"),+ complements_convention_collective_renseignee=company.get(+ "complements", {}+ ).get("convention_collective_renseignee"),complements_liste_idcc=company.get("complements", {}).get("liste_idcc"),- complements_egapro_renseignee=company.get("complements", {}).get("egapro_renseignee"),- complements_est_achats_responsables=company.get("complements", {}).get("est_achats_responsables"),- complements_est_alim_confiance=company.get("complements", {}).get("est_alim_confiance"),- complements_est_association=company.get("complements", {}).get("est_association"),+ complements_egapro_renseignee=company.get("complements", {}).get(+ "egapro_renseignee"+ ),+ complements_est_achats_responsables=company.get("complements", {}).get(+ "est_achats_responsables"+ ),+ complements_est_alim_confiance=company.get("complements", {}).get(+ "est_alim_confiance"+ ),+ complements_est_association=company.get("complements", {}).get(+ "est_association"+ ),complements_est_bio=company.get("complements", {}).get("est_bio"),- complements_est_entrepreneur_individuel=company.get("complements", {}).get("est_entrepreneur_individuel"),- complements_est_entrepreneur_spectacle=company.get("complements", {}).get("est_entrepreneur_spectacle"),+ complements_est_entrepreneur_individuel=company.get(+ "complements", {}+ ).get("est_entrepreneur_individuel"),+ complements_est_entrepreneur_spectacle=company.get(+ "complements", {}+ ).get("est_entrepreneur_spectacle"),complements_est_ess=company.get("complements", {}).get("est_ess"),complements_est_finess=company.get("complements", {}).get("est_finess"),- complements_est_organisme_formation=company.get("complements", {}).get("est_organisme_formation"),- complements_est_qualiopi=company.get("complements", {}).get("est_qualiopi"),- complements_liste_id_organisme_formation=company.get("complements", {}).get("liste_id_organisme_formation"),+ complements_est_organisme_formation=company.get("complements", {}).get(+ "est_organisme_formation"+ ),+ complements_est_qualiopi=company.get("complements", {}).get(+ "est_qualiopi"+ ),+ complements_liste_id_organisme_formation=company.get(+ "complements", {}+ ).get("liste_id_organisme_formation"),complements_est_rge=company.get("complements", {}).get("est_rge"),- complements_est_service_public=company.get("complements", {}).get("est_service_public"),+ complements_est_service_public=company.get("complements", {}).get(+ "est_service_public"+ ),complements_est_l100_3=company.get("complements", {}).get("est_l100_3"),complements_est_siae=company.get("complements", {}).get("est_siae"),- complements_est_societe_mission=company.get("complements", {}).get("est_societe_mission"),+ complements_est_societe_mission=company.get("complements", {}).get(+ "est_societe_mission"+ ),complements_est_uai=company.get("complements", {}).get("est_uai"),- complements_est_patrimoine_vivant=company.get("complements", {}).get("est_patrimoine_vivant"),- complements_bilan_ges_renseigne=company.get("complements", {}).get("bilan_ges_renseigne"),- complements_identifiant_association=company.get("complements", {}).get("identifiant_association"),- complements_statut_entrepreneur_spectacle=company.get("complements", {}).get("statut_entrepreneur_spectacle"),- complements_type_siae=company.get("complements", {}).get("type_siae")+ complements_est_patrimoine_vivant=company.get("complements", {}).get(+ "est_patrimoine_vivant"+ ),+ complements_bilan_ges_renseigne=company.get("complements", {}).get(+ "bilan_ges_renseigne"+ ),+ complements_identifiant_association=company.get("complements", {}).get(+ "identifiant_association"+ ),+ complements_statut_entrepreneur_spectacle=company.get(+ "complements", {}+ ).get("statut_entrepreneur_spectacle"),+ complements_type_siae=company.get("complements", {}).get("type_siae"),)except Exception as e:- Logger.error(self.sketch_id, {"message": f"Error enriching organization: {e}"})+ Logger.error(+ self.sketch_id, {"message": f"Error enriching organization: {e}"}+ )return None-+def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:if not self.neo4j_conn:return results@@ -222,103 +267,153 @@ class OrgToInfosScanner(Scanner):for org in results:# Create or update the organization node with all SIRENE propertiesorg_key = f"{org.name}_FR"- self.create_node('Organization', 'org_id', org_key,- name=org.name,- country="FR",- siren=org.siren,- siege_siret=org.siege_siret,- nom_complet=org.nom_complet,- nom_raison_sociale=org.nom_raison_sociale,- sigle=org.sigle,- nombre_etablissements=org.nombre_etablissements,- nombre_etablissements_ouverts=org.nombre_etablissements_ouverts,- activite_principale=org.activite_principale,- section_activite_principale=org.section_activite_principale,- categorie_entreprise=org.categorie_entreprise,- annee_categorie_entreprise=org.annee_categorie_entreprise,- caractere_employeur=org.caractere_employeur,- tranche_effectif_salarie=org.tranche_effectif_salarie,- annee_tranche_effectif_salarie=org.annee_tranche_effectif_salarie,- date_creation=org.date_creation,- date_fermeture=org.date_fermeture,- date_mise_a_jour=org.date_mise_a_jour,- date_mise_a_jour_insee=org.date_mise_a_jour_insee,- date_mise_a_jour_rne=org.date_mise_a_jour_rne,- nature_juridique=org.nature_juridique,- etat_administratif=org.etat_administratif,- statut_diffusion=org.statut_diffusion,- caption=org.name, type='organization')+ self.create_node(+ "Organization",+ "org_id",+ org_key,+ name=org.name,+ country="FR",+ siren=org.siren,+ siege_siret=org.siege_siret,+ nom_complet=org.nom_complet,+ nom_raison_sociale=org.nom_raison_sociale,+ sigle=org.sigle,+ nombre_etablissements=org.nombre_etablissements,+ nombre_etablissements_ouverts=org.nombre_etablissements_ouverts,+ activite_principale=org.activite_principale,+ section_activite_principale=org.section_activite_principale,+ categorie_entreprise=org.categorie_entreprise,+ annee_categorie_entreprise=org.annee_categorie_entreprise,+ caractere_employeur=org.caractere_employeur,+ tranche_effectif_salarie=org.tranche_effectif_salarie,+ annee_tranche_effectif_salarie=org.annee_tranche_effectif_salarie,+ date_creation=org.date_creation,+ date_fermeture=org.date_fermeture,+ date_mise_a_jour=org.date_mise_a_jour,+ date_mise_a_jour_insee=org.date_mise_a_jour_insee,+ date_mise_a_jour_rne=org.date_mise_a_jour_rne,+ nature_juridique=org.nature_juridique,+ etat_administratif=org.etat_administratif,+ statut_diffusion=org.statut_diffusion,+ caption=org.name,+ type="organization",+ )if org.siren:self.log_graph_message(f"{org.name}: SIREN {org.siren} -> {org.name}")# Add SIRET as identifier if availableif org.siege_siret:- self.log_graph_message(f"{org.name}: SIRET {org.siege_siret} -> {org.name}")+ self.log_graph_message(+ f"{org.name}: SIRET {org.siege_siret} -> {org.name}"+ )# Add dirigeants (leaders) as Individual nodes with relationshipsif org.dirigeants:for dirigeant in org.dirigeants:- self.create_node('Individual', 'full_name', dirigeant.full_name,- first_name=dirigeant.first_name,- last_name=dirigeant.last_name,- birth_date=dirigeant.birth_date,- gender=dirigeant.gender,- caption=dirigeant.full_name, type='individual')-- self.create_relationship('Organization', 'org_id', org_key,- 'Individual', 'full_name', dirigeant.full_name, 'HAS_LEADER')- self.log_graph_message(f"{org.name}: HAS_LEADER -> {dirigeant.full_name}")+ self.create_node(+ "Individual",+ "full_name",+ dirigeant.full_name,+ first_name=dirigeant.first_name,+ last_name=dirigeant.last_name,+ birth_date=dirigeant.birth_date,+ gender=dirigeant.gender,+ caption=dirigeant.full_name,+ type="individual",+ )++ self.create_relationship(+ "Organization",+ "org_id",+ org_key,+ "Individual",+ "full_name",+ dirigeant.full_name,+ "HAS_LEADER",+ )+ self.log_graph_message(+ f"{org.name}: HAS_LEADER -> {dirigeant.full_name}"+ )# Add siege address as PhysicalAddress node if availableif org.siege_geo_adresse:address = org.siege_geo_adresseaddress_key = f"{address.address}_{address.city}_{address.country}"- self.create_node('PhysicalAddress', 'address_id', address_key,- address=address.address,- city=address.city,- country=address.country,- zip=address.zip,- latitude=address.latitude,- longitude=address.longitude,- label=f"{address.address}, {address.city}",- caption=f"{address.address}, {address.city}",- type='location')-- self.create_relationship('Organization', 'org_id', org_key,- 'PhysicalAddress', 'address_id', address_key, 'HAS_ADDRESS')- self.log_graph_message(f"{org.name}: HAS_ADDRESS -> {address.address}, {address.city}")+ self.create_node(+ "PhysicalAddress",+ "address_id",+ address_key,+ address=address.address,+ city=address.city,+ country=address.country,+ zip=address.zip,+ latitude=address.latitude,+ longitude=address.longitude,+ label=f"{address.address}, {address.city}",+ caption=f"{address.address}, {address.city}",+ type="location",+ )++ self.create_relationship(+ "Organization",+ "org_id",+ org_key,+ "PhysicalAddress",+ "address_id",+ address_key,+ "HAS_ADDRESS",+ )+ self.log_graph_message(+ f"{org.name}: HAS_ADDRESS -> {address.address}, {address.city}"+ )# Add siege location as Location node if coordinates are available but no PhysicalAddresselif org.siege_latitude and org.siege_longitude:location_key = f"{org.siege_latitude}_{org.siege_longitude}"- self.create_node('Location', 'location_id', location_key,- latitude=float(org.siege_latitude),- longitude=float(org.siege_longitude),- address=org.siege_adresse,- city=org.siege_libelle_commune,- country="FR",- zip=org.siege_code_postal,- label=f"{org.siege_adresse or 'Unknown'}, {org.siege_libelle_commune or 'Unknown'}",- caption=f"{org.siege_adresse or 'Unknown'}, {org.siege_libelle_commune or 'Unknown'}",- type='location')-- self.create_relationship('Organization', 'org_id', org_key,- 'Location', 'location_id', location_key, 'LOCATED_AT')- self.log_graph_message(f"{org.name}: LOCATED_AT -> {org.siege_libelle_commune or 'Unknown'}")+ self.create_node(+ "Location",+ "location_id",+ location_key,+ latitude=float(org.siege_latitude),+ longitude=float(org.siege_longitude),+ address=org.siege_adresse,+ city=org.siege_libelle_commune,+ country="FR",+ zip=org.siege_code_postal,+ label=f"{org.siege_adresse or 'Unknown'}, {org.siege_libelle_commune or 'Unknown'}",+ caption=f"{org.siege_adresse or 'Unknown'}, {org.siege_libelle_commune or 'Unknown'}",+ type="location",+ )++ self.create_relationship(+ "Organization",+ "org_id",+ org_key,+ "Location",+ "location_id",+ location_key,+ "LOCATED_AT",+ )+ self.log_graph_message(+ f"{org.name}: LOCATED_AT -> {org.siege_libelle_commune or 'Unknown'}"+ )# Add activity codes as Activity nodesif org.activite_principale:- self.log_graph_message(f"{org.name}: HAS_ACTIVITY -> {org.activite_principale}")+ self.log_graph_message(+ f"{org.name}: HAS_ACTIVITY -> {org.activite_principale}"+ )# Add legal nature as LegalNature nodeif org.nature_juridique:- self.log_graph_message(f"{org.name}: HAS_LEGAL_NATURE -> {org.nature_juridique}")+ self.log_graph_message(+ f"{org.name}: HAS_LEGAL_NATURE -> {org.nature_juridique}"+ )return results+# Make types available at module level for easy accessInputType = OrgToInfosScanner.InputTypeOutputType = OrgToInfosScanner.OutputType--diff --git a/flowsint-transforms/src/flowsint_transforms/phones/ignorant.py b/flowsint-transforms/src/flowsint_transforms/phones/ignorant.pyindex 4d447f19..9c5cb65c 100644--- a/flowsint-transforms/src/flowsint_transforms/phones/ignorant.py+++ b/flowsint-transforms/src/flowsint_transforms/phones/ignorant.py@@ -5,20 +5,21 @@ from flowsint_core.utils import is_valid_numberfrom flowsint_core.core.logger import Loggerimport httpx+class IgnorantScanner(Scanner):-+# Define types as class attributes - base class handles schema generation automaticallyInputType = List[str] # Phone numbers as stringsOutputType = List[Dict[str, Any]] # Results as dictionaries-+@classmethoddef name(cls) -> str:return "ignorant_scanner"-+@classmethoddef category(cls) -> str:return "phones"-+@classmethoddef key(cls) -> str:return "number"@@ -44,50 +45,46 @@ class IgnorantScanner(Scanner):result = await self._perform_ignorant_research(cleaned_phone)results.append(result)else:- results.append({- "number": phone,- "error": "Invalid phone number"- })+ results.append({"number": phone, "error": "Invalid phone number"})except Exception as e:- results.append({- "number": phone,- "error": f"Unexpected error in Ignorant scan: {str(e)}"- })- Logger.error(self.sketch_id, {"message": f"Error scanning phone {phone}: {str(e)}"})+ results.append(+ {+ "number": phone,+ "error": f"Unexpected error in Ignorant scan: {str(e)}",+ }+ )+ Logger.error(+ self.sketch_id,+ {"message": f"Error scanning phone {phone}: {str(e)}"},+ )return results-+async def _perform_ignorant_research(self, phone: str) -> Dict[str, Any]:try:# Import necessary modules for each platformfrom ignorant.modules.shopping.amazon import amazonfrom ignorant.modules.social_media.instagram import instagramfrom ignorant.modules.social_media.snapchat import snapchat-+# Create an HTTP client for asynchronous requestsasync with httpx.AsyncClient() as client:results = []modules = [amazon, snapchat, instagram]-+# Execute the modules in paralleltasks = [module(phone, "+33", client) for module in modules]responses = await asyncio.gather(*tasks)-+# Add results from each modulefor response in responses:if response:results.append(response)-- return {- "number": phone,- "platforms": results- }-++ return {"number": phone, "platforms": results}+except Exception as e:- return {- "number": phone,- "error": f"Error in Ignorant research: {str(e)}"- }-+ return {"number": phone, "error": f"Error in Ignorant research: {str(e)}"}+def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:"""Create Neo4j relationships for found phone accounts.@@ -97,17 +94,25 @@ class IgnorantScanner(Scanner):for result in results:if "error" not in result and "platforms" in result:- self.create_node('phone', 'number', result["number"],- caption=result["number"], type='phone')-+ self.create_node(+ "phone",+ "number",+ result["number"],+ caption=result["number"],+ type="phone",+ )+# Create platform relationshipsfor platform_result in result["platforms"]:if platform_result and isinstance(platform_result, dict):platform_name = platform_result.get("platform", "unknown")- self.log_graph_message(f"Phone {result['number']} found on {platform_name}")+ self.log_graph_message(+ f"Phone {result['number']} found on {platform_name}"+ )return results+# Make types available at module level for easy accessInputType = IgnorantScanner.InputTypeOutputType = IgnorantScanner.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/socials/maigret.py b/flowsint-transforms/src/flowsint_transforms/socials/maigret.pyindex a215ca01..ba6dbf07 100644--- a/flowsint-transforms/src/flowsint_transforms/socials/maigret.py+++ b/flowsint-transforms/src/flowsint_transforms/socials/maigret.py@@ -9,6 +9,7 @@ from flowsint_core.core.logger import Loggerfalse_positives = ["LeagueOfLegends"]+class MaigretScanner(Scanner):"""Scans usernames for associated social accounts using Maigret."""@@ -42,7 +43,7 @@ class MaigretScanner(Scanner):if obj and obj.username and is_valid_username(obj.username):cleaned.append(obj)return cleaned-+def run_maigret(self, username: str) -> Path:output_file = Path(f"/tmp/report_{username}_simple.json")try:@@ -50,13 +51,18 @@ class MaigretScanner(Scanner):["maigret", username, "-J", "simple", "-fo", "/tmp"],capture_output=True,text=True,- timeout=100+ timeout=100,)except Exception as e:- Logger.error(self.sketch_id, {"message": f"Maigret execution failed for {username}: {e}"})+ Logger.error(+ self.sketch_id,+ {"message": f"Maigret execution failed for {username}: {e}"},+ )return output_file-- def parse_maigret_output(self, username: str, output_file: Path) -> List[SocialProfile]:++ def parse_maigret_output(+ self, username: str, output_file: Path+ ) -> List[SocialProfile]:results: List[SocialProfile] = []if not output_file.exists():return results@@ -65,13 +71,16 @@ class MaigretScanner(Scanner):with open(output_file, "r") as f:raw_data = json.load(f)except Exception as e:- Logger.error(self.sketch_id, {"message": f"Failed to load output file for {username}: {e}"})+ Logger.error(+ self.sketch_id,+ {"message": f"Failed to load output file for {username}: {e}"},+ )return resultsfor platform, profile in raw_data.items():if profile.get("status", {}).get("status") != "Claimed":continue-+if any(fp in platform for fp in false_positives):continue@@ -82,26 +91,37 @@ class MaigretScanner(Scanner):continuetry:- followers = int(ids.get("follower_count", 0)) if ids.get("follower_count") else None- following = int(ids.get("following_count", 0)) if ids.get("following_count") else None+ followers = (+ int(ids.get("follower_count", 0))+ if ids.get("follower_count")+ else None+ )+ following = (+ int(ids.get("following_count", 0))+ if ids.get("following_count")+ else None+ )posts = (- int(ids.get("public_repos_count", 0)) +- int(ids.get("public_gists_count", 0))- if "public_repos_count" in ids or "public_gists_count" in ids else None+ int(ids.get("public_repos_count", 0))+ + int(ids.get("public_gists_count", 0))+ if "public_repos_count" in ids or "public_gists_count" in ids+ else None)except ValueError:followers = following = posts = None- results.append(SocialProfile(- username=username,- profile_url=profile_url,- platform=platform,- profile_picture_url=ids.get("image"),- bio=None,- followers_count=followers,- following_count=following,- posts_count=posts- ))+ results.append(+ SocialProfile(+ username=username,+ profile_url=profile_url,+ platform=platform,+ profile_picture_url=ids.get("image"),+ bio=None,+ followers_count=followers,+ following_count=following,+ posts_count=posts,+ )+ )return results@@ -121,30 +141,44 @@ class MaigretScanner(Scanner):for profile in results:# Create social profile node- self.create_node('social_profile', 'profile_url', profile.profile_url,- username=profile.username,- platform=profile.platform,- profile_picture_url=profile.profile_picture_url,- bio=profile.bio,- followers_count=profile.followers_count,- following_count=profile.following_count,- posts_count=profile.posts_count,- label=f"{profile.platform}:{profile.username}",- caption=f"{profile.platform}:{profile.username}",- color="#1DA1F2",- type="social_profile")-+ self.create_node(+ "social_profile",+ "profile_url",+ profile.profile_url,+ username=profile.username,+ platform=profile.platform,+ profile_picture_url=profile.profile_picture_url,+ bio=profile.bio,+ followers_count=profile.followers_count,+ following_count=profile.following_count,+ posts_count=profile.posts_count,+ label=f"{profile.platform}:{profile.username}",+ caption=f"{profile.platform}:{profile.username}",+ color="#1DA1F2",+ type="social_profile",+ )+# Create username node- self.create_node('username', 'username', profile.username, type='username')-+ self.create_node("username", "username", profile.username, type="username")+# Create relationship- self.create_relationship('username', 'username', profile.username,- 'social_profile', 'profile_url', profile.profile_url, 'HAS_SOCIAL_ACCOUNT')-- self.log_graph_message(f"{profile.username} -> account found on {profile.platform}")+ self.create_relationship(+ "username",+ "username",+ profile.username,+ "social_profile",+ "profile_url",+ profile.profile_url,+ "HAS_SOCIAL_ACCOUNT",+ )++ self.log_graph_message(+ f"{profile.username} -> account found on {profile.platform}"+ )return results+# Make types available at module level for easy accessInputType = MaigretScanner.InputTypeOutputType = MaigretScanner.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/socials/sherlock.py b/flowsint-transforms/src/flowsint_transforms/socials/sherlock.pyindex 06f9d693..73609be7 100644--- a/flowsint-transforms/src/flowsint_transforms/socials/sherlock.py+++ b/flowsint-transforms/src/flowsint_transforms/socials/sherlock.py@@ -2,7 +2,7 @@ import subprocessfrom pathlib import Pathfrom typing import List, Unionfrom flowsint_core.utils import is_valid_username-from flowsint_types.social import Social+from flowsint_types.social import SocialProfilefrom flowsint_core.core.scanner_base import Scannerfrom flowsint_core.core.logger import Logger@@ -11,30 +11,30 @@ class SherlockScanner(Scanner):"""Scans the usernames for associated social accounts using Sherlock."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[Social]- OutputType = List[Social]+ InputType = List[SocialProfile]+ OutputType = List[SocialProfile]@classmethoddef name(cls) -> str:return "sherlock_scanner"-+@classmethoddef category(cls) -> str:return "social_account"-+@classmethoddef key(cls) -> str:return "username"-+def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:cleaned: InputType = []for item in data:obj = Noneif isinstance(item, str):- obj = Social(username=item)+ obj = SocialProfile(username=item)elif isinstance(item, dict) and "username" in item:- obj = Social(username=item["username"])- elif isinstance(item, Social):+ obj = SocialProfile(username=item["username"])+ elif isinstance(item, SocialProfile):obj = itemif obj and obj.username and is_valid_username(obj.username):@@ -44,7 +44,7 @@ class SherlockScanner(Scanner):async def scan(self, data: InputType) -> OutputType:"""Performs the scan using Sherlock on the list of usernames."""results: OutputType = []-+for social in data:username = social.usernameoutput_file = Path(f"/tmp/sherlock_{username}.txt")@@ -54,15 +54,25 @@ class SherlockScanner(Scanner):["sherlock", username, "-o", str(output_file)],capture_output=True,text=True,- timeout=100+ timeout=100,)if result.returncode != 0:- Logger.error(self.sketch_id, {"message": f"Sherlock failed for {username}: {result.stderr.strip()}"})+ Logger.error(+ self.sketch_id,+ {+ "message": f"Sherlock failed for {username}: {result.stderr.strip()}"+ },+ )continueif not output_file.exists():- Logger.error(self.sketch_id, {"message": f"Sherlock did not produce any output file for {username}."})+ Logger.error(+ self.sketch_id,+ {+ "message": f"Sherlock did not produce any output file for {username}."+ },+ )continuefound_accounts = {}@@ -75,16 +85,22 @@ class SherlockScanner(Scanner):# Create Social objects for each found accountfor platform, url in found_accounts.items():- results.append(Social(- username=username,- platform=platform,- url=url- ))+ results.append(+ SocialProfile(username=username, platform=platform, url=url)+ )except subprocess.TimeoutExpired:- Logger.error(self.sketch_id, {"message": f"Sherlock scan for {username} timed out."})+ Logger.error(+ self.sketch_id,+ {"message": f"Sherlock scan for {username} timed out."},+ )except Exception as e:- Logger.error(self.sketch_id, {"message": f"Unexpected error in Sherlock scan for {username}: {str(e)}"})+ Logger.error(+ self.sketch_id,+ {+ "message": f"Unexpected error in Sherlock scan for {username}: {str(e)}"+ },+ )return results@@ -94,13 +110,22 @@ class SherlockScanner(Scanner):return resultsfor social in results:- self.create_node('social', 'username', social.username,- platform=social.platform, url=social.url,- caption=social.platform, type='social')- self.log_graph_message(f"Found social account: {social.username} on {social.platform}")+ self.create_node(+ "social",+ "username",+ social.username,+ platform=social.platform,+ url=social.url,+ caption=social.platform,+ type="social",+ )+ self.log_graph_message(+ f"Found social account: {social.username} on {social.platform}"+ )return results+# Make types available at module level for easy accessInputType = SherlockScanner.InputTypeOutputType = SherlockScanner.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/websites/to_crawler.py b/flowsint-transforms/src/flowsint_transforms/websites/to_crawler.pyindex a2e7ba19..72d35300 100644--- a/flowsint-transforms/src/flowsint_transforms/websites/to_crawler.py+++ b/flowsint-transforms/src/flowsint_transforms/websites/to_crawler.py@@ -8,6 +8,7 @@ from flowsint_core.core.logger import Loggerfrom tools.network.reconcrawl import ReconCrawlToolfrom pydantic import BaseModel+class ReturnType(BaseModel):website: Websiteemails: Optional[Email]@@ -28,7 +29,7 @@ class WebsiteToCrawler(Scanner):@classmethoddef category(cls) -> str:return "Website"-+@classmethoddef key(cls) -> str:return "url"@@ -55,53 +56,94 @@ class WebsiteToCrawler(Scanner):if website_obj:cleaned.append(website_obj)return cleaned-+async def scan(self, data: InputType) -> OutputType:"""Crawl websites to extract emails and phone numbers."""results = []-+for website in data:try:- Logger.info(self.sketch_id, {"message": f"Starting comprehensive crawl of {str(website.url)}"})+ Logger.info(+ self.sketch_id,+ {"message": f"Starting comprehensive crawl of {str(website.url)}"},+ )crawler = ReconCrawlTool()- crawl_result = crawler.launch(website.url, {"recursive": True, "verify_ssl": True, "max_pages": 500, "timeout": 10, "delay": 1.0, "verbose": False})+ crawl_result = crawler.launch(+ website.url,+ {+ "recursive": True,+ "verify_ssl": True,+ "max_pages": 500,+ "timeout": 10,+ "delay": 1.0,+ "verbose": False,+ },+ )website_result = {- "website": str(website.url), # Store as string instead of Website object+ "website": str(+ website.url+ ), # Store as string instead of Website object"emails": [],"phones": [],}for item in crawl_result:- Logger.info(self.sketch_id, {"message": f"{item.type}: {item.value}"})+ Logger.info(+ self.sketch_id, {"message": f"{item.type}: {item.value}"}+ )if item.source_url:- Logger.info(self.sketch_id, {"message": f" Found on: {item.source_url}"})+ Logger.info(+ self.sketch_id,+ {"message": f" Found on: {item.source_url}"},+ )if item.type == "email":website_result["emails"].append(Email(email=item.value))if item.type == "phone":website_result["phones"].append(Phone(number=item.value))# Log results- Logger.info(self.sketch_id, {- "message": f"Crawl completed for {str(website.url)}: {len(website_result['emails'])} emails, {len(website_result['phones'])} phones found."- })-+ Logger.info(+ self.sketch_id,+ {+ "message": f"Crawl completed for {str(website.url)}: {len(website_result['emails'])} emails, {len(website_result['phones'])} phones found."+ },+ )+if not website_result["emails"] and not website_result["phones"]:- Logger.info(self.sketch_id, {"message": f"No emails or phones found for website {str(website.url)}."})+ Logger.info(+ self.sketch_id,+ {+ "message": f"No emails or phones found for website {str(website.url)}."+ },+ )elif not website_result["emails"]:- Logger.info(self.sketch_id, {"message": f"No emails found for website {str(website.url)}"})+ Logger.info(+ self.sketch_id,+ {"message": f"No emails found for website {str(website.url)}"},+ )elif not website_result["phones"]:- Logger.info(self.sketch_id, {"message": f"No phones found for website {str(website.url)}"})-+ Logger.info(+ self.sketch_id,+ {"message": f"No phones found for website {str(website.url)}"},+ )+results.append(website_result)-+except Exception as e:# Log error but continue with other websites- Logger.error(self.sketch_id, {"message": f"Error crawling {str(website.url)}: {str(e)}"})+ Logger.error(+ self.sketch_id,+ {"message": f"Error crawling {str(website.url)}: {str(e)}"},+ )# Add empty result for failed website- results.append({- "website": str(website.url), # Store as string instead of Website object- "emails": [],- "phones": [],- })+ results.append(+ {+ "website": str(+ website.url+ ), # Store as string instead of Website object+ "emails": [],+ "phones": [],+ }+ )continuereturn results@@ -110,29 +152,55 @@ class WebsiteToCrawler(Scanner):# Create Neo4j relationships between websites and their corresponding emails and phonesfor input_website, result in zip(original_input, results):website_url = str(input_website.url)-+# Create website nodeif self.neo4j_conn:- self.create_node('website', 'url', website_url,- caption=website_url, type='website')-+ self.create_node(+ "website", "url", website_url, caption=website_url, type="website"+ )+# Create email nodes and relationshipsfor email in result["emails"]:- self.create_node('email', 'email', email.email,- caption=email.email, type='email')- self.create_relationship('website', 'url', website_url,- 'email', 'email', email.email, 'HAS_EMAIL')- self.log_graph_message(f"Found email {email.email} for website {website_url}")-+ self.create_node(+ "email", "email", email.email, caption=email.email, type="email"+ )+ self.create_relationship(+ "website",+ "url",+ website_url,+ "email",+ "email",+ email.email,+ "HAS_EMAIL",+ )+ self.log_graph_message(+ f"Found email {email.email} for website {website_url}"+ )+# Create phone nodes and relationshipsfor phone in result["phones"]:- self.create_node('phone', 'number', phone.number,- caption=phone.number, type='phone')- self.create_relationship('website', 'url', website_url,- 'phone', 'number', phone.number, 'HAS_PHONE')- self.log_graph_message(f"Found phone {phone.number} for website {website_url}")+ self.create_node(+ "phone",+ "number",+ phone.number,+ caption=phone.number,+ type="phone",+ )+ self.create_relationship(+ "website",+ "url",+ website_url,+ "phone",+ "number",+ phone.number,+ "HAS_PHONE",+ )+ self.log_graph_message(+ f"Found phone {phone.number} for website {website_url}"+ )return results-++InputType = WebsiteToCrawler.InputType-OutputType = WebsiteToCrawler.OutputType\ No newline at end of file+OutputType = WebsiteToCrawler.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/websites/to_domain.py b/flowsint-transforms/src/flowsint_transforms/websites/to_domain.pyindex a847c8ff..0d1403e5 100644--- a/flowsint-transforms/src/flowsint_transforms/websites/to_domain.py+++ b/flowsint-transforms/src/flowsint_transforms/websites/to_domain.py@@ -5,6 +5,7 @@ from flowsint_types.website import Websitefrom flowsint_types.domain import Domainfrom flowsint_core.core.logger import Logger+class WebsiteToDomainScanner(Scanner):"""From website to domain."""@@ -19,7 +20,7 @@ class WebsiteToDomainScanner(Scanner):@classmethoddef category(cls) -> str:return "Website"-+@classmethoddef key(cls) -> str:return "website"@@ -44,28 +45,36 @@ class WebsiteToDomainScanner(Scanner):try:parsed_url = urlparse(website.url)domain_name = parsed_url.netloc-+# Remove port if present- if ':' in domain_name:- domain_name = domain_name.split(':')[0]-+ if ":" in domain_name:+ domain_name = domain_name.split(":")[0]+# Remove www. prefix if present- if domain_name.startswith('www.'):+ if domain_name.startswith("www."):domain_name = domain_name[4:]-+if domain_name:domain_obj = Domain(domain=domain_name)results.append(domain_obj)-+except Exception as e:- Logger.error(self.sketch_id, {"message": f"Error extracting domain from website {website.url}: {e}"})+ Logger.error(+ self.sketch_id,+ {+ "message": f"Error extracting domain from website {website.url}: {e}"+ },+ )continue-+return results- def postprocess(self, results: OutputType, input_data: InputType = None) -> OutputType:+ def postprocess(+ self, results: OutputType, input_data: InputType = None+ ) -> OutputType:return results+# Make types available at module level for easy accessInputType = WebsiteToDomainScanner.InputType-OutputType = WebsiteToDomainScanner.OutputType\ No newline at end of file+OutputType = WebsiteToDomainScanner.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/websites/to_links.py b/flowsint-transforms/src/flowsint_transforms/websites/to_links.pyindex 324712b2..fa465eaf 100644--- a/flowsint-transforms/src/flowsint_transforms/websites/to_links.py+++ b/flowsint-transforms/src/flowsint_transforms/websites/to_links.py@@ -5,6 +5,7 @@ from flowsint_types.website import Websitefrom flowsint_core.core.logger import Loggerfrom reconspread import Crawler+class WebsiteToLinks(Scanner):"""From website to spread crawler that extracts domains and internal/external links."""@@ -19,7 +20,7 @@ class WebsiteToLinks(Scanner):@classmethoddef category(cls) -> str:return "Website"-+@classmethoddef key(cls) -> str:return "url"@@ -45,34 +46,56 @@ class WebsiteToLinks(Scanner):if website_obj:cleaned.append(website_obj)return cleaned-+async def scan(self, data: InputType) -> OutputType:"""Crawl websites using reconspread to extract internal and external links."""results = []-+for website in data:try:- Logger.info(self.sketch_id, {"message": f"Starting reconspread crawl of {str(website.url)}"})-+ Logger.info(+ self.sketch_id,+ {"message": f"Starting reconspread crawl of {str(website.url)}"},+ )+# Extract main domain from input website (needed in callback)main_domain = self.extract_domain(str(website.url))-+# Create main website and domain nodes upfrontif self.neo4j_conn:- self.create_node('website', 'url', str(website.url),- caption=str(website.url), type='website')+ self.create_node(+ "website",+ "url",+ str(website.url),+ caption=str(website.url),+ type="website",+ )if main_domain:- self.create_node('domain', 'name', main_domain,- caption=main_domain, type='domain')- self.create_relationship('website', 'url', str(website.url),- 'domain', 'name', main_domain, 'BELONGS_TO_DOMAIN')- self.log_graph_message(f"Website {str(website.url)} belongs to domain {main_domain}")-+ self.create_node(+ "domain",+ "name",+ main_domain,+ caption=main_domain,+ type="domain",+ )+ self.create_relationship(+ "website",+ "url",+ str(website.url),+ "domain",+ "name",+ main_domain,+ "BELONGS_TO_DOMAIN",+ )+ self.log_graph_message(+ f"Website {str(website.url)} belongs to domain {main_domain}"+ )+# Store discovered URLsinternal_urls = []external_urls = []external_domains = set()-+def url_handler(url, is_external=False):"""Custom callback to handle URLs as they're discovered."""if is_external:@@ -82,38 +105,95 @@ class WebsiteToLinks(Scanner):external_domains.add(domain)# Create external website node immediatelyif self.neo4j_conn:- self.create_node('website', 'url', url,- caption=url, type='website')- self.create_relationship('website', 'url', str(website.url),- 'website', 'url', url, 'LINKS_TO')- self.log_graph_message(f"Website {str(website.url)} links to external website {url}")-+ self.create_node(+ "website", "url", url, caption=url, type="website"+ )+ self.create_relationship(+ "website",+ "url",+ str(website.url),+ "website",+ "url",+ url,+ "LINKS_TO",+ )+ self.log_graph_message(+ f"Website {str(website.url)} links to external website {url}"+ )+# Create external domain node and link external website to its domainif domain != main_domain:- self.create_node('domain', 'name', domain,- caption=domain, type='domain')- self.create_relationship('website', 'url', url,- 'domain', 'name', domain, 'BELONGS_TO_DOMAIN')- self.create_relationship('website', 'url', str(website.url),- 'domain', 'name', domain, 'LINKS_TO_DOMAIN')- self.log_graph_message(f"External website {url} belongs to domain {domain}")- self.log_graph_message(f"Website {str(website.url)} links to external domain {domain}")- Logger.info(self.sketch_id, {"message": f"[EXTERNAL] Found: {url} -> Domain: {domain}"})+ self.create_node(+ "domain",+ "name",+ domain,+ caption=domain,+ type="domain",+ )+ self.create_relationship(+ "website",+ "url",+ url,+ "domain",+ "name",+ domain,+ "BELONGS_TO_DOMAIN",+ )+ self.create_relationship(+ "website",+ "url",+ str(website.url),+ "domain",+ "name",+ domain,+ "LINKS_TO_DOMAIN",+ )+ self.log_graph_message(+ f"External website {url} belongs to domain {domain}"+ )+ self.log_graph_message(+ f"Website {str(website.url)} links to external domain {domain}"+ )+ Logger.info(+ self.sketch_id,+ {"message": f"[EXTERNAL] Found: {url} -> Domain: {domain}"},+ )else:internal_urls.append(url)# Create internal website node immediately- if self.neo4j_conn and url != str(website.url): # Don't create duplicate of main website- self.create_node('website', 'url', url,- caption=url, type='website')- self.create_relationship('website', 'url', str(website.url),- 'website', 'url', url, 'LINKS_TO')- self.log_graph_message(f"Website {str(website.url)} links to internal website {url}")-+ if self.neo4j_conn and url != str(+ website.url+ ): # Don't create duplicate of main website+ self.create_node(+ "website", "url", url, caption=url, type="website"+ )+ self.create_relationship(+ "website",+ "url",+ str(website.url),+ "website",+ "url",+ url,+ "LINKS_TO",+ )+ self.log_graph_message(+ f"Website {str(website.url)} links to internal website {url}"+ )+# Also link internal websites to main domainif main_domain:- self.create_relationship('website', 'url', url,- 'domain', 'name', main_domain, 'BELONGS_TO_DOMAIN')- Logger.info(self.sketch_id, {"message": f"[INTERNAL] Found: {url}"})+ self.create_relationship(+ "website",+ "url",+ url,+ "domain",+ "name",+ main_domain,+ "BELONGS_TO_DOMAIN",+ )+ Logger.info(+ self.sketch_id, {"message": f"[INTERNAL] Found: {url}"}+ )# Create crawler with custom callbackcrawler = Crawler(@@ -121,7 +201,7 @@ class WebsiteToLinks(Scanner):recursive=True,same_domain_only=False,verbose=False, # Disable default verbose output- _on_result_callback=url_handler # Use our custom handler+ _on_result_callback=url_handler, # Use our custom handler)# Perform the crawl@@ -130,12 +210,12 @@ class WebsiteToLinks(Scanner):# Get final results (backup in case callback missed anything)crawl_results = crawler.get_results()-+# Ensure we have all internal URLsfor url in crawl_results.internal:if url not in internal_urls:internal_urls.append(url)-+# Ensure we have all external URLs and domainsfor url in crawl_results.external:if url not in external_urls:@@ -149,44 +229,71 @@ class WebsiteToLinks(Scanner):"main_domain": main_domain,"internal_urls": internal_urls,"external_urls": external_urls,- "external_domains": list(external_domains)+ "external_domains": list(external_domains),}# Log results- Logger.info(self.sketch_id, {- "message": f"Spread crawl completed for {str(website.url)}: "- f"Main domain: {main_domain}, "- f"{len(internal_urls)} internal URLs, "- f"{len(external_urls)} external URLs, "- f"{len(external_domains)} external domains found."- })-+ Logger.info(+ self.sketch_id,+ {+ "message": f"Spread crawl completed for {str(website.url)}: "+ f"Main domain: {main_domain}, "+ f"{len(internal_urls)} internal URLs, "+ f"{len(external_urls)} external URLs, "+ f"{len(external_domains)} external domains found."+ },+ )+results.append(website_result)-+except Exception as e:# Log error but continue with other websites- Logger.error(self.sketch_id, {"message": f"Error crawling {str(website.url)}: {str(e)}"})-+ Logger.error(+ self.sketch_id,+ {"message": f"Error crawling {str(website.url)}: {str(e)}"},+ )+# Still create main website and domain nodes even on errormain_domain = self.extract_domain(str(website.url))if self.neo4j_conn:- self.create_node('website', 'url', str(website.url),- caption=str(website.url), type='website')+ self.create_node(+ "website",+ "url",+ str(website.url),+ caption=str(website.url),+ type="website",+ )if main_domain:- self.create_node('domain', 'name', main_domain,- caption=main_domain, type='domain')- self.create_relationship('website', 'url', str(website.url),- 'domain', 'name', main_domain, 'BELONGS_TO_DOMAIN')- self.log_graph_message(f"Website {str(website.url)} belongs to domain {main_domain}")-+ self.create_node(+ "domain",+ "name",+ main_domain,+ caption=main_domain,+ type="domain",+ )+ self.create_relationship(+ "website",+ "url",+ str(website.url),+ "domain",+ "name",+ main_domain,+ "BELONGS_TO_DOMAIN",+ )+ self.log_graph_message(+ f"Website {str(website.url)} belongs to domain {main_domain}"+ )+# Add empty result for failed website- results.append({- "website": str(website.url),- "main_domain": main_domain,- "internal_urls": [],- "external_urls": [],- "external_domains": []- })+ results.append(+ {+ "website": str(website.url),+ "main_domain": main_domain,+ "internal_urls": [],+ "external_urls": [],+ "external_domains": [],+ }+ )continuereturn results@@ -195,6 +302,7 @@ class WebsiteToLinks(Scanner):# Neo4j nodes and relationships are created in real-time during the callback# No additional processing needed herereturn results-++InputType = WebsiteToLinks.InputType-OutputType = WebsiteToLinks.OutputType\ No newline at end of file+OutputType = WebsiteToLinks.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/websites/to_webtrackers.py b/flowsint-transforms/src/flowsint_transforms/websites/to_webtrackers.pyindex 0675f3de..ba4e8dc7 100644--- a/flowsint-transforms/src/flowsint_transforms/websites/to_webtrackers.py+++ b/flowsint-transforms/src/flowsint_transforms/websites/to_webtrackers.py@@ -7,6 +7,7 @@ from flowsint_core.core.graph_db import Neo4jConnectionfrom flowsint_core.core.vault import VaultProtocolfrom recontrack import TrackingCodeExtractor+class WebsiteToWebtrackersScanner(Scanner):"""From website to webtrackers."""@@ -21,7 +22,7 @@ class WebsiteToWebtrackersScanner(Scanner):neo4j_conn: Optional[Neo4jConnection] = None,params_schema: Optional[List[Dict[str, Any]]] = None,vault: Optional[VaultProtocol] = None,- params: Optional[Dict[str, Any]] = None+ params: Optional[Dict[str, Any]] = None,):super().__init__(sketch_id, scan_id, neo4j_conn, params_schema, vault, params)self.tracker_website_mapping: List[tuple[WebTracker, Website]] = []@@ -33,7 +34,7 @@ class WebsiteToWebtrackersScanner(Scanner):@classmethoddef category(cls) -> str:return "Website"-+@classmethoddef key(cls) -> str:return "website"@@ -55,31 +56,39 @@ class WebsiteToWebtrackersScanner(Scanner):async def scan(self, data: InputType) -> OutputType:results: OutputType = []extractor = TrackingCodeExtractor()-+for website in data:try:# Extract tracking codes from the websitetracking_data = extractor.extract(str(website.url))-+for tracker_info in tracking_data:tracker = WebTracker(name=tracker_info.get("name", ""),tracker_id=tracker_info.get("id", ""),category=tracker_info.get("category", ""),- website_url=str(website.url)+ website_url=str(website.url),)results.append(tracker)self.tracker_website_mapping.append((tracker, website))-+except Exception as e:- Logger.error(self.sketch_id, {"message": f"Error extracting web trackers from {website.url}: {e}"})+ Logger.error(+ self.sketch_id,+ {+ "message": f"Error extracting web trackers from {website.url}: {e}"+ },+ )continue-+return results- def postprocess(self, results: OutputType, input_data: InputType = None) -> OutputType:+ def postprocess(+ self, results: OutputType, input_data: InputType = None+ ) -> OutputType:return results+# Make types available at module level for easy accessInputType = WebsiteToWebtrackersScanner.InputType-OutputType = WebsiteToWebtrackersScanner.OutputType\ No newline at end of file+OutputType = WebsiteToWebtrackersScanner.OutputTypediff --git a/flowsint-transforms/src/tools/base.py b/flowsint-transforms/src/tools/base.pyindex 6ee7f8fb..8f49e643 100644--- a/flowsint-transforms/src/tools/base.py+++ b/flowsint-transforms/src/tools/base.py@@ -1,6 +1,7 @@from abc import ABC, abstractmethodfrom typing import Any+class Tool(ABC):@classmethod@abstractmethod@@ -11,12 +12,12 @@ class Tool(ABC):@abstractmethoddef category(cls) -> str:pass-+@classmethod@abstractmethoddef description(cls) -> str:pass-+@classmethod@abstractmethoddef version(cls) -> str:diff --git a/flowsint-transforms/src/tools/dockertool.py b/flowsint-transforms/src/tools/dockertool.pyindex 8ad24aac..e1a652d7 100644--- a/flowsint-transforms/src/tools/dockertool.py+++ b/flowsint-transforms/src/tools/dockertool.py@@ -2,11 +2,12 @@ from docker import from_envfrom docker.errors import ImageNotFound, APIError, DockerExceptionfrom .base import Tool+class DockerTool(Tool):def __init__(self, image: str, default_tag: str = "latest"):self.image = f"{image}:{default_tag}"self.client = from_env()-+@classmethoddef get_image(cls) -> str:return cls.image@@ -17,7 +18,7 @@ class DockerTool(Tool):self.client.images.pull(self.image)except APIError as e:raise RuntimeError(f"Failed to pull image {self.image}: {e.explanation}")-+def version(self) -> str:try:output = self.client.containers.run(@@ -27,19 +28,18 @@ class DockerTool(Tool):stdout=True,stderr=True,detach=False,- tty=False+ tty=False,)return output.decode().strip()except Exception as e:return f"unknown (error: {str(e)})"-+def is_installed(self) -> bool:try:self.client.images.get(self.image)return Trueexcept ImageNotFound:return False-def launch(self, command: str, volumes: dict = None, timeout: int = 30):self.install()@@ -63,13 +63,13 @@ class DockerTool(Tool):except DockerException as e:# Try to get more detailed error informationerror_detail = str(e)- if hasattr(e, 'response') and hasattr(e.response, 'json'):+ if hasattr(e, "response") and hasattr(e.response, "json"):try:error_json = e.response.json()error_detail = f"{str(e)} - Details: {error_json}"except:pass-+# Check if it's a container exit errorif "returned non-zero exit status" in str(e):# Try to run the command with stderr capture to see what went wrong@@ -90,5 +90,7 @@ class DockerTool(Tool):return test_result.decode()except DockerException as test_e:error_detail = f"{str(e)} - Test run also failed: {str(test_e)}"-- raise RuntimeError(f"Docker error while running {self.image}: {error_detail}")++ raise RuntimeError(+ f"Docker error while running {self.image}: {error_detail}"+ )diff --git a/flowsint-transforms/src/tools/network/asnmap.py b/flowsint-transforms/src/tools/network/asnmap.pyindex c92b11a8..757445c2 100644--- a/flowsint-transforms/src/tools/network/asnmap.py+++ b/flowsint-transforms/src/tools/network/asnmap.py@@ -2,17 +2,18 @@ import jsonfrom typing import Any, Literalfrom ..dockertool import DockerTool+class AsnmapTool(DockerTool):image = "projectdiscovery/asnmap"default_tag = "latest"-+def __init__(self):super().__init__(self.image, self.default_tag)-+@classmethoddef name(cls) -> str:return "asnmap"-+@classmethoddef description(cls) -> str:return "ASN mapping and network reconnaissance tool."@@ -31,16 +32,17 @@ class AsnmapTool(DockerTool):command="--version",remove=True,stderr=True,- stdout=True+ stdout=True,)output_str = output.decode()import re- match = re.search(r'(v[\d\.]+)', output_str)++ match = re.search(r"(v[\d\.]+)", output_str)version = match.group(1) if match else "unknown"return versionexcept Exception as e:return f"unknown (error: {str(e)})"-+def update(self) -> None:# Pull the latest imageself.install()@@ -48,15 +50,14 @@ class AsnmapTool(DockerTool):def is_installed(self) -> bool:return super().is_installed()- def launch(self, item: str, type: Literal["domain", "organization", "ip", "asn"] = "domain") -> Any:- flags = {- "domain" : '-d',- 'org': '-org',- 'ip' : '-i',- 'asn': '-a'- }+ def launch(+ self, item: str, type: Literal["domain", "organization", "ip", "asn"] = "domain"+ ) -> Any:+ flags = {"domain": "-d", "org": "-org", "ip": "-i", "asn": "-a"}if type not in flags:- raise ValueError(f"Invalid type: '{type}'. Valid types are: {list(flags.keys())}")+ raise ValueError(+ f"Invalid type: '{type}'. Valid types are: {list(flags.keys())}"+ )flag = flags[type]try:# Use the -target argument as asnmap expects@@ -69,5 +70,6 @@ class AsnmapTool(DockerTool):raise RuntimeError(f"Failed to parse JSON output from asnmap: {str(e)}")except Exception as e:# Try to get more info from the container logs- raise RuntimeError(f"Error running asnmap: {str(e)}. Output: {getattr(e, 'output', 'No output')}")-+ raise RuntimeError(+ f"Error running asnmap: {str(e)}. Output: {getattr(e, 'output', 'No output')}"+ )diff --git a/flowsint-transforms/src/tools/network/httpx.py b/flowsint-transforms/src/tools/network/httpx.pyindex 2d1e89eb..ed13c88a 100644--- a/flowsint-transforms/src/tools/network/httpx.py+++ b/flowsint-transforms/src/tools/network/httpx.py@@ -2,17 +2,18 @@ import jsonfrom typing import Any, Listfrom ..dockertool import DockerTool+class HttpxTool(DockerTool):image = "projectdiscovery/httpx"default_tag = "latest"-+def __init__(self):super().__init__(self.image, self.default_tag)-+@classmethoddef name(cls) -> str:return "httpx"-+@classmethoddef description(cls) -> str:return "An HTTP toolkit that probes services, web servers, and other valuable metadata."@@ -31,17 +32,17 @@ class HttpxTool(DockerTool):command="--version",remove=True,stderr=True,- stdout=True+ stdout=True,)output_str = output.decode()import re- match = re.search(r'(v[\d\.]+)', output_str)++ match = re.search(r"(v[\d\.]+)", output_str)version = match.group(1) if match else "unknown"return versionexcept Exception as e:return f"unknown (error: {str(e)})"--+def update(self) -> None:# Pull the latest imageself.install()@@ -49,21 +50,21 @@ class HttpxTool(DockerTool):def is_installed(self) -> bool:return super().is_installed()- def launch(self, target: str, args: List[str]| None = None) -> Any:+ def launch(self, target: str, args: List[str] | None = None) -> Any:if args is None:args = []- args_str = ' '.join(args) if args else ''+ args_str = " ".join(args) if args else ""command = f"-u {target} {args_str} -json -silent"result = super().launch(command)-+# Handle empty resultif not result or result.strip() == "":return []-+# Handle multiple JSON lines (one per result)- lines = result.strip().split('\n')+ lines = result.strip().split("\n")results = []-+for line in lines:line = line.strip()if line:@@ -72,4 +73,3 @@ class HttpxTool(DockerTool):except json.JSONDecodeError as e:raise ereturn results-diff --git a/flowsint-transforms/src/tools/network/reconcrawl.py b/flowsint-transforms/src/tools/network/reconcrawl.pyindex abd35129..8183b4a5 100644--- a/flowsint-transforms/src/tools/network/reconcrawl.py+++ b/flowsint-transforms/src/tools/network/reconcrawl.py@@ -2,15 +2,16 @@ from typing import Any, Dictfrom ..base import Tool+class ReconCrawlTool(Tool):-+def __init__(self):super().__init__()-+@classmethoddef name(cls) -> str:return "reconcrawl"-+@classmethoddef description(cls) -> str:return "Emails and phone numbers crawler from websites by analyzing their HTML and embedded scripts."@@ -21,31 +22,34 @@ class ReconCrawlTool(Tool):def install(self) -> None:pass+def version(self) -> str:pass-def is_installed(self) -> bool:try:import reconcrawl+return Trueexcept ImportError:return Falsedef launch(self, url: str, args: Dict[str, Any] = None) -> Any:from reconcrawl import Crawler+crawler = Crawler(- url=str(url),- max_pages=args.get("max_pages", 500),- timeout=args.get("timeout", 30),- delay=args.get("delay", 1.0),- verbose=args.get("verbose", False),- recursive=args.get("recursive", True),- verify_ssl=args.get("verify_ssl", False) # Default to False for compatibility- )+ url=str(url),+ max_pages=args.get("max_pages", 500),+ timeout=args.get("timeout", 30),+ delay=args.get("delay", 1.0),+ verbose=args.get("verbose", False),+ recursive=args.get("recursive", True),+ verify_ssl=args.get(+ "verify_ssl", False+ ), # Default to False for compatibility+ )crawler.fetch()crawler.extract_emails()crawler.extract_phones()crawl_result = crawler.get_results()return crawl_result-diff --git a/flowsint-transforms/src/tools/network/subfinder.py b/flowsint-transforms/src/tools/network/subfinder.pyindex fce9b9e7..2c8c5e81 100644--- a/flowsint-transforms/src/tools/network/subfinder.py+++ b/flowsint-transforms/src/tools/network/subfinder.py@@ -2,17 +2,18 @@ from typing import Any, Listfrom ..dockertool import DockerToolfrom flowsint_core.utils import is_valid_domain+class SubfinderTool(DockerTool):image = "projectdiscovery/subfinder"default_tag = "latest"-+def __init__(self):super().__init__(self.image, self.default_tag)-+@classmethoddef name(cls) -> str:return "subfinder"-+@classmethoddef description(cls) -> str:return "Fast passive subdomain enumeration tool."@@ -32,17 +33,17 @@ class SubfinderTool(DockerTool):command="--version",remove=True,stderr=True,- stdout=True+ stdout=True,)output_str = output.decode()import re- match = re.search(r'(v[\d\.]+)', output_str)++ match = re.search(r"(v[\d\.]+)", output_str)version = match.group(1) if match else "unknown"return versionexcept Exception as e:return f"unknown (error: {str(e)})"--+def update(self) -> None:# Pull the latest imageself.install()@@ -55,9 +56,13 @@ class SubfinderTool(DockerTool):if args is None:args = []command = f"-d {domain} {' '.join(args)}"- result = super().launch(command)+ result = super().launch(command)for sub in result.split("\n"):- if is_valid_domain(sub) and sub.endswith(domain) and sub != domain and not sub.startswith("."):+ if (+ is_valid_domain(sub)+ and sub.endswith(domain)+ and sub != domain+ and not sub.startswith(".")+ ):subdomains.add(sub)return list(subdomains)-diff --git a/flowsint-transforms/src/tools/organizations/sirene.py b/flowsint-transforms/src/tools/organizations/sirene.pyindex fc9bded9..a19ad8bb 100644--- a/flowsint-transforms/src/tools/organizations/sirene.py+++ b/flowsint-transforms/src/tools/organizations/sirene.py@@ -3,16 +3,17 @@ from typing import Dictimport requestsfrom ..base import Tool+class SireneTool(Tool):-+@classmethoddef name(cls) -> str:return "sirene"-+@classmethoddef version(cls) -> str:return "1.0.0"-+@classmethoddef description(cls) -> str:return "The Sirene API allows you to query the Sirene directory of businesses and establishments, managed by Insee."@@ -21,18 +22,22 @@ class SireneTool(Tool):def category(cls) -> str:return "Business intelligence"- def launch(self, query: str, limit: int=25) -> list[Dict]:- # Query can be multiple types of entries: full names ("<first> + <last>"), name, geolocation, etc.+ def launch(self, query: str, limit: int = 25) -> list[Dict]:+ # Query can be multiple types of entries: full names ("<first> + <last>"), name, geolocation, etc.try:query = query.replace(" ", "+")- params = {"q": query, "per_page": limit }- resp = requests.get("https://recherche-entreprises.api.gouv.fr/search", params=params, timeout=10)+ params = {"q": query, "per_page": limit}+ resp = requests.get(+ "https://recherche-entreprises.api.gouv.fr/search",+ params=params,+ timeout=10,+ )resp.raise_for_status()data = resp.json()if len(data.get("results")) == 0:raise ValueError(f"No match found for {query}.")return data["results"]except Exception as e:- raise RuntimeError(f"Error querying Sirene API: {str(e)}. Output: {getattr(e, 'output', 'No output')}")--+ raise RuntimeError(+ f"Error querying Sirene API: {str(e)}. Output: {getattr(e, 'output', 'No output')}"+ )diff --git a/flowsint-transforms/tests/conftest.py b/flowsint-transforms/tests/conftest.pyindex c713514f..be95bac0 100644--- a/flowsint-transforms/tests/conftest.py+++ b/flowsint-transforms/tests/conftest.py@@ -1,9 +1,12 @@import pytestfrom tests.logger import TestLogger+@pytest.fixture(autouse=True)def mock_logger(monkeypatch):"""Automatically replace the production Logger with TestLogger for all tests."""monkeypatch.setattr("flowsint_core.core.logger.Logger", TestLogger)# Mock the emit_event_task to do nothing- monkeypatch.setattr("flowsint_core.core.logger.emit_event_task.delay", lambda *args, **kwargs: None)\ No newline at end of file+ monkeypatch.setattr(+ "flowsint_core.core.logger.emit_event_task.delay", lambda *args, **kwargs: None+ )diff --git a/flowsint-transforms/tests/insert_command.py b/flowsint-transforms/tests/insert_command.pyindex 103db359..8e6cc2de 100644--- a/flowsint-transforms/tests/insert_command.py+++ b/flowsint-transforms/tests/insert_command.py@@ -1,11 +1,10 @@-import sysimport osimport asyncioif __name__ == "__main__":sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))-+from flowsint_types.domain import Domainfrom flowsint_types.ip import Ipfrom flowsint_transforms.domains.resolve import ResolveScanner@@ -14,15 +13,16 @@ from flowsint_transforms.domains.resolve import ResolveScannerasync def main():# Create test datadomains = [Domain(domain="adaltas.com")]- ips = [Ip(address='12.23.34.45'), Ip(address='56.67.78.89')]-+ ips = [Ip(address="12.23.34.45"), Ip(address="56.67.78.89")]+# Test the scannerscanner = ResolveScanner("sketch_123", "scan_123")-+# Test the new KISS postprocess methodscanner.postprocess(ips[:1], domains) # Only use first IP to match domains length-+print("Postprocess test completed successfully!")+if __name__ == "__main__":- asyncio.run(main())\ No newline at end of file+ asyncio.run(main())diff --git a/flowsint-transforms/tests/logger.py b/flowsint-transforms/tests/logger.pyindex 87242d03..b00a2f02 100644--- a/flowsint-transforms/tests/logger.py+++ b/flowsint-transforms/tests/logger.py@@ -5,7 +5,7 @@ EventLevel = Literal["info", "warn", "error", "success", "debug"]LEVEL_MAP = {"info": "INFO",- "warn": "WARN",+ "warn": "WARN","error": "FAILED","success": "SUCCESS","debug": "DEBUG",@@ -17,13 +17,15 @@ class TestLogger:def _format_message(type: str, message: str) -> str:"""Format the log message with type prefix"""return f"[{type.upper()}] {message}"-+@staticmethoddef _create_log(sketch_id: Union[str, UUID], log_type: str, content: str) -> Any:"""Create a dummy log object for testing"""+class DummyLog:def __init__(self):- self.id = 'dummy_id'+ self.id = "dummy_id"+return DummyLog()@staticmethoddiff --git a/flowsint-transforms/tests/scanners/crypto/wallet_to_nfts.py b/flowsint-transforms/tests/scanners/crypto/wallet_to_nfts.pyindex 4c5864b6..9e7778e6 100644--- a/flowsint-transforms/tests/scanners/crypto/wallet_to_nfts.py+++ b/flowsint-transforms/tests/scanners/crypto/wallet_to_nfts.py@@ -1,17 +1,22 @@from flowsint_transforms.crypto.wallet_to_nfts import CryptoWalletAddressToNFTsfrom flowsint_types.wallet import CryptoWallet, CryptoNFTfrom pydantic import HttpUrl+scanner = CryptoWalletAddressToNFTs("sketch_123", "scan_123")+def test_wallet_address_to_transactions_name():assert scanner.name() == "wallet_to_nfts"+def test_wallet_address_to_transactions_category():assert scanner.category() == "crypto"+def test_wallet_address_to_transactions_key():assert scanner.key() == "address"+def test_preprocess_with_string():input_data = ["0x742d35Cc6634C0532925a3b844Bc454e4438f44e"]result = scanner.preprocess(input_data)@@ -19,6 +24,7 @@ def test_preprocess_with_string():assert isinstance(result[0], CryptoWallet)assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"+def test_preprocess_with_dict():input_data = [{"address": "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"}]result = scanner.preprocess(input_data)@@ -26,6 +32,7 @@ def test_preprocess_with_dict():assert isinstance(result[0], CryptoWallet)assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"+def test_preprocess_with_wallet_object():wallet = CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")input_data = [wallet]@@ -34,26 +41,29 @@ def test_preprocess_with_wallet_object():assert isinstance(result[0], CryptoWallet)assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"+def test_scan_mocked_transactions(monkeypatch):# Mock the _get_transactions methoddef mock_get_nfts(address):return [CryptoNFT(- wallet=CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e"),+ wallet=CryptoWallet(+ address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e"+ ),contract_address="0x123",token_id="1",collection_name="Test Collection",metadata_url="https://example.com/metadata.json",image_url="https://example.com/image.png",- name="Test NFT"+ name="Test NFT",)]-+monkeypatch.setattr(scanner, "_get_nfts", mock_get_nfts)-+input_data = [CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")]result = scanner.scan(input_data)-+assert len(result) == 1assert len(result[0]) == 1assert result[0][0].contract_address == "0x123"@@ -61,4 +71,3 @@ def test_scan_mocked_transactions(monkeypatch):assert result[0][0].metadata_url == HttpUrl("https://example.com/metadata.json")assert result[0][0].image_url == HttpUrl("https://example.com/image.png")assert result[0][0].name == "Test NFT"-\ No newline at end of filediff --git a/flowsint-transforms/tests/scanners/crypto/wallet_to_transactions.py b/flowsint-transforms/tests/scanners/crypto/wallet_to_transactions.pyindex 50475221..af0deb74 100644--- a/flowsint-transforms/tests/scanners/crypto/wallet_to_transactions.py+++ b/flowsint-transforms/tests/scanners/crypto/wallet_to_transactions.py@@ -1,18 +1,28 @@import pytest-from flowsint_transforms.crypto.wallet_to_transactions import CryptoWalletAddressToTransactions+from flowsint_transforms.crypto.wallet_to_transactions import (+ CryptoWalletAddressToTransactions,+)from flowsint_types.wallet import CryptoWallet, CryptoWalletTransaction-scanner = CryptoWalletAddressToTransactions("sketch_123", "scan_123", params={"ETHERSCAN_API_KEY": "ta-clef-api"},)+scanner = CryptoWalletAddressToTransactions(+ "sketch_123",+ "scan_123",+ params={"ETHERSCAN_API_KEY": "ta-clef-api"},+)+def test_wallet_address_to_transactions_name():assert scanner.name() == "wallet_to_transactions"+def test_wallet_address_to_transactions_category():assert scanner.category() == "CryptoCryptoWallet"+def test_wallet_address_to_transactions_key():assert scanner.key() == "address"+def test_preprocess_with_string():input_data = ["0x742d35Cc6634C0532925a3b844Bc454e4438f44e"]result = scanner.preprocess(input_data)@@ -20,6 +30,7 @@ def test_preprocess_with_string():assert isinstance(result[0], CryptoWallet)assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"+def test_preprocess_with_dict():input_data = [{"address": "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"}]result = scanner.preprocess(input_data)@@ -27,6 +38,7 @@ def test_preprocess_with_dict():assert isinstance(result[0], CryptoWallet)assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"+def test_preprocess_with_wallet_object():wallet = CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")input_data = [wallet]@@ -35,6 +47,7 @@ def test_preprocess_with_wallet_object():assert isinstance(result[0], CryptoWallet)assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"+@pytest.mark.asyncioasync def test_scan_mocked_transactions(monkeypatch):# Mock the _get_transactions method - note it takes address and api_key parameters@@ -42,7 +55,9 @@ async def test_scan_mocked_transactions(monkeypatch):return [CryptoWalletTransaction(hash="0x123",- source=CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e"),+ source=CryptoWallet(+ address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e"+ ),target=CryptoWallet(address="0x456"),value=1.0, # 1 ETHtimestamp="1234567890",@@ -55,15 +70,15 @@ async def test_scan_mocked_transactions(monkeypatch):gas_used="21000",cumulative_gas_used="21000",input="0x",- contract_address=None+ contract_address=None,)]-+monkeypatch.setattr(scanner, "_get_transactions", mock_get_transactions)-+input_data = [CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")]result = await scanner.scan(input_data)-+assert len(result) == 1assert len(result[0]) == 1assert result[0][0].hash == "0x123"@@ -72,13 +87,20 @@ async def test_scan_mocked_transactions(monkeypatch):assert result[0][0].value == 1.0assert result[0][0].timestamp == "1234567890"+def test_scanner_requires_api_key():"""Test that the scanner validates required ETHERSCAN_API_KEY parameter at construction"""- with pytest.raises(ValueError, match="Scanner wallet_to_transactions received invalid params"):+ with pytest.raises(+ ValueError, match="Scanner wallet_to_transactions received invalid params"+ ):CryptoWalletAddressToTransactions("sketch_123", "scan_123", params={})+def test_scanner_with_invalid_api_key_type():"""Test that the scanner validates parameter types"""- with pytest.raises(ValueError, match="Scanner wallet_to_transactions received invalid params"):- CryptoWalletAddressToTransactions("sketch_123", "scan_123", params={"ETHERSCAN_API_KEY": 123})-\ No newline at end of file+ with pytest.raises(+ ValueError, match="Scanner wallet_to_transactions received invalid params"+ ):+ CryptoWalletAddressToTransactions(+ "sketch_123", "scan_123", params={"ETHERSCAN_API_KEY": 123}+ )diff --git a/flowsint-transforms/tests/scanners/domains/resolve.py b/flowsint-transforms/tests/scanners/domains/resolve.pyindex bccf4c69..de72d91a 100644--- a/flowsint-transforms/tests/scanners/domains/resolve.py+++ b/flowsint-transforms/tests/scanners/domains/resolve.py@@ -6,18 +6,20 @@ import pytestscanner = ResolveScanner("sketch_123", "scan_123")+def test_preprocess_valid_domains():domains = [Domain(domain="example.com"),Domain(domain="example2.com"),]result = scanner.preprocess(domains)-+result_domains = [d.domain for d in result]expected_domains = [d.domain for d in domains]assert result_domains == expected_domains-++def test_unprocessed_valid_domains():domains = ["example.com",@@ -26,8 +28,9 @@ def test_unprocessed_valid_domains():result = scanner.preprocess(domains)result_domains = [d for d in result]expected_domains = [Domain(domain=d) for d in domains]- assert result_domains == expected_domains-+ assert result_domains == expected_domains++def test_preprocess_invalid_domains():domains = [Domain(domain="example.com"),@@ -41,6 +44,7 @@ def test_preprocess_invalid_domains():assert "example.org" in result_domainsassert "invalid_domain" not in result_domains+def test_preprocess_multiple_formats():domains = [{"domain": "example.com"},@@ -56,6 +60,7 @@ def test_preprocess_multiple_formats():assert "invalid_domain" not in result_domainsassert "example.io" not in result_domains+@pytest.mark.asyncioasync def test_scan_returns_ip(monkeypatch):# on crée une fonction mock qui retourne une IP@@ -70,78 +75,79 @@ async def test_scan_returns_ip(monkeypatch):assert isinstance(output, list)assert output[0].address == "12.23.34.45"+def test_schemas():input_schema = scanner.input_schema()output_schema = scanner.output_schema()-+# Test the structure and key properties rather than exact match- assert input_schema['type'] == 'Domain'- assert isinstance(input_schema['properties'], list)- input_property_names = [prop['name'] for prop in input_schema['properties']]- assert 'domain' in input_property_names-- assert output_schema['type'] == 'Ip'- assert isinstance(output_schema['properties'], list)- output_property_names = [prop['name'] for prop in output_schema['properties']]- assert 'address' in output_property_names+ assert input_schema["type"] == "Domain"+ assert isinstance(input_schema["properties"], list)+ input_property_names = [prop["name"] for prop in input_schema["properties"]]+ assert "domain" in input_property_names++ assert output_schema["type"] == "Ip"+ assert isinstance(output_schema["properties"], list)+ output_property_names = [prop["name"] for prop in output_schema["properties"]]+ assert "address" in output_property_namesclass TestResolveInputOutputTypes:"""Test the InputType/OutputType functionality for ResolveScanner"""-+def test_input_output_types_are_defined(self):"""Test that InputType and OutputType are properly defined"""- assert hasattr(ResolveScanner, 'InputType')- assert hasattr(ResolveScanner, 'OutputType')+ assert hasattr(ResolveScanner, "InputType")+ assert hasattr(ResolveScanner, "OutputType")assert ResolveScanner.InputType == List[Domain]assert ResolveScanner.OutputType == List[Ip]-+def test_schemas_use_generate_methods(self):"""Test that schema methods use the new generate methods"""# These should work without errorinput_schema = ResolveScanner.generate_input_schema()output_schema = ResolveScanner.generate_output_schema()-+assert isinstance(input_schema, dict)assert isinstance(output_schema, dict)assert input_schema["type"] == "Domain"assert output_schema["type"] == "Ip"-+def test_schema_methods_return_same_as_generate_methods(self):"""Test that input_schema() and output_schema() return the same as generate methods"""assert ResolveScanner.input_schema() == ResolveScanner.generate_input_schema()assert ResolveScanner.output_schema() == ResolveScanner.generate_output_schema()-+def test_input_schema_properties(self):"""Test input schema has expected properties"""schema = ResolveScanner.input_schema()-+properties = schema["properties"]property_names = [p["name"] for p in properties]-+# Domain should have these propertiesassert "domain" in property_names-+def test_output_schema_properties(self):"""Test output schema has expected properties"""schema = ResolveScanner.output_schema()-+properties = schema["properties"]property_names = [p["name"] for p in properties]-+# Ip should have these propertiesassert "address" in property_names-+def test_type_accessibility_from_instance(self):"""Test that types are accessible from scanner instance"""scanner_instance = ResolveScanner("test", "test")-+assert scanner_instance.InputType == List[Domain]assert scanner_instance.OutputType == List[Ip]-+# Should be able to generate schemas from instanceinput_schema = scanner_instance.generate_input_schema()output_schema = scanner_instance.generate_output_schema()-+assert input_schema["type"] == "Domain"assert output_schema["type"] == "Ip"diff --git a/flowsint-transforms/tests/scanners/domains/subdomains.py b/flowsint-transforms/tests/scanners/domains/subdomains.pyindex 65712f43..d323721c 100644--- a/flowsint-transforms/tests/scanners/domains/subdomains.py+++ b/flowsint-transforms/tests/scanners/domains/subdomains.py@@ -3,18 +3,20 @@ from flowsint_types.domain import Domain, Domainscanner = SubdomainScanner("sketch_123", "scan_123")+def test_preprocess_valid_domains():domains = [Domain(domain="example.com"),Domain(domain="example2.com"),]result = scanner.preprocess(domains)-+result_domains = [d.domain for d in result]expected_domains = [d.domain for d in domains]assert result_domains == expected_domains-++def test_unprocessed_valid_domains():domains = ["example.com",@@ -23,8 +25,9 @@ def test_unprocessed_valid_domains():result = scanner.preprocess(domains)result_domains = [d for d in result]expected_domains = [Domain(domain=d) for d in domains]- assert result_domains == expected_domains-+ assert result_domains == expected_domains++def test_preprocess_invalid_domains():domains = [Domain(domain="example.com"),@@ -38,6 +41,7 @@ def test_preprocess_invalid_domains():assert "example.org" in result_domainsassert "invalid_domain" not in result_domains+def test_preprocess_multiple_formats():domains = [{"domain": "example.com"},@@ -83,13 +87,9 @@ def test_scan_extracts_subdomains(monkeypatch):input_data = [Domain(domain="example.com")]domains = scanner.execute(input_data)assert isinstance(domains, list)- for sub in domains:+ for sub in domains:print(sub)assert isinstance(sub, Domain)- expected = sorted([- "mail.example.com",- "www.example.com",- "api.example.com"- ])+ expected = sorted(["mail.example.com", "www.example.com", "api.example.com"])print(domains)# assert domains[0].subdomains == expecteddiff --git a/flowsint-transforms/tests/scanners/domains/whois.py b/flowsint-transforms/tests/scanners/domains/whois.pyindex 8146dfc4..c03eac81 100644--- a/flowsint-transforms/tests/scanners/domains/whois.py+++ b/flowsint-transforms/tests/scanners/domains/whois.py@@ -3,18 +3,20 @@ from flowsint_types.domain import Domainscanner = WhoisScanner("sketch_123", "scan_123")+def test_preprocess_valid_domains():domains = [Domain(domain="example.com"),Domain(domain="example2.com"),]result = scanner.preprocess(domains)-+result_domains = [d.domain for d in result]expected_domains = [d.domain for d in domains]assert result_domains == expected_domains-++def test_unprocessed_valid_domains():domains = ["example.com",@@ -23,8 +25,9 @@ def test_unprocessed_valid_domains():result = scanner.preprocess(domains)result_domains = [d for d in result]expected_domains = [Domain(domain=d) for d in domains]- assert result_domains == expected_domains-+ assert result_domains == expected_domains++def test_preprocess_invalid_domains():domains = [Domain(domain="example.com"),@@ -38,6 +41,7 @@ def test_preprocess_invalid_domains():assert "example.org" in result_domainsassert "invalid_domain" not in result_domains+def test_preprocess_multiple_formats():domains = [{"domain": "example.com"},@@ -53,6 +57,7 @@ def test_preprocess_multiple_formats():assert "invalid_domain" not in result_domainsassert "example.io" not in result_domains+def test_scan_returns_whois_objects(monkeypatch):# Patch `whois.whois` to avoid real network callmock_whois = lambda domain: {@@ -62,7 +67,7 @@ def test_scan_returns_whois_objects(monkeypatch):"country": "MockCountry","emails": ["[email protected]"],"creation_date": "2020-01-01",- "expiration_date": "2030-01-01"+ "expiration_date": "2030-01-01",}monkeypatch.setattr("whois.whois", mock_whois)@@ -74,8 +79,25 @@ def test_scan_returns_whois_objects(monkeypatch):assert output[0].whois.org == "MockOrg"assert output[0].whois.email.email == "[email protected]"+def test_schemas():input_schema = scanner.input_schema()output_schema = scanner.output_schema()- assert input_schema == {'type': 'Domain', 'properties': [{'name': 'domain', 'type': 'string'}, {'name': 'subdomains', 'type': 'array | null'}, {'name': 'ips', 'type': 'array | null'}, {'name': 'whois', 'type': 'Whois | null'}]}- assert output_schema == {'type': 'Domain', 'properties': [{'name': 'domain', 'type': 'string'}, {'name': 'subdomains', 'type': 'array | null'}, {'name': 'ips', 'type': 'array | null'}, {'name': 'whois', 'type': 'Whois | null'}]}+ assert input_schema == {+ "type": "Domain",+ "properties": [+ {"name": "domain", "type": "string"},+ {"name": "subdomains", "type": "array | null"},+ {"name": "ips", "type": "array | null"},+ {"name": "whois", "type": "Whois | null"},+ ],+ }+ assert output_schema == {+ "type": "Domain",+ "properties": [+ {"name": "domain", "type": "string"},+ {"name": "subdomains", "type": "array | null"},+ {"name": "ips", "type": "array | null"},+ {"name": "whois", "type": "Whois | null"},+ ],+ }diff --git a/flowsint-transforms/tests/scanners/emails/to_gravatar.py b/flowsint-transforms/tests/scanners/emails/to_gravatar.pyindex 13824bcc..871b5b4f 100644--- a/flowsint-transforms/tests/scanners/emails/to_gravatar.py+++ b/flowsint-transforms/tests/scanners/emails/to_gravatar.py@@ -28,7 +28,9 @@ class TestEmailToGravatarScanner:assert schema["type"] == "Email"assert "properties" in schema# Check that email property is present- email_prop = next((prop for prop in schema["properties"] if prop["name"] == "email"), None)+ email_prop = next(+ (prop for prop in schema["properties"] if prop["name"] == "email"), None+ )assert email_prop is not Noneassert email_prop["type"] == "string"@@ -38,8 +40,12 @@ class TestEmailToGravatarScanner:assert schema["type"] == "Gravatar"assert "properties" in schema# Check that required properties are present- src_prop = next((prop for prop in schema["properties"] if prop["name"] == "src"), None)- hash_prop = next((prop for prop in schema["properties"] if prop["name"] == "hash"), None)+ src_prop = next(+ (prop for prop in schema["properties"] if prop["name"] == "src"), None+ )+ hash_prop = next(+ (prop for prop in schema["properties"] if prop["name"] == "hash"), None+ )assert src_prop is not Noneassert hash_prop is not None@@ -114,7 +120,7 @@ class TestEmailToGravatarScanner:result = scanner.preprocess([])assert result == []- @patch('requests.get')+ @patch("requests.get")def test_scan_successful_gravatar(self, mock_get):"""Test successful gravatar retrieval"""# Mock successful response@@ -130,7 +136,7 @@ class TestEmailToGravatarScanner:assert result[0].hash == hashlib.md5("[email protected]".encode()).hexdigest()assert "gravatar.com/avatar/" in str(result[0].src)- @patch('requests.get')+ @patch("requests.get")def test_scan_failed_request(self, mock_get):"""Test handling of failed HTTP requests"""# Mock failed response@@ -143,7 +149,7 @@ class TestEmailToGravatarScanner:assert len(result) == 0- @patch('requests.get')+ @patch("requests.get")def test_scan_request_exception(self, mock_get):"""Test handling of request exceptions"""# Mock exception@@ -154,7 +160,7 @@ class TestEmailToGravatarScanner:assert len(result) == 0- @patch('requests.get')+ @patch("requests.get")def test_scan_multiple_emails(self, mock_get):"""Test scanning multiple emails"""# Mock successful responses@@ -173,9 +179,10 @@ class TestEmailToGravatarScanner:assert all(isinstance(gravatar, Gravatar) for gravatar in result)assert mock_get.call_count == 3- @patch('requests.get')+ @patch("requests.get")def test_scan_mixed_success_failure(self, mock_get):"""Test scanning with mixed success and failure"""+# Mock mixed responses - check the actual URL being calleddef side_effect(url, *args, **kwargs):mock_response = Mock()@@ -203,7 +210,9 @@ class TestEmailToGravatarScanner:"""Test postprocessing with Neo4j connection"""# Mock Neo4j connectionmock_neo4j = Mock()- scanner_with_neo4j = EmailToGravatarScanner("sketch_123", "scan_123", neo4j_conn=mock_neo4j)+ scanner_with_neo4j = EmailToGravatarScanner(+ "sketch_123", "scan_123", neo4j_conn=mock_neo4j+ )gravatars = [Gravatar(src="https://www.gravatar.com/avatar/hash1", hash="hash1"),@@ -218,7 +227,7 @@ class TestEmailToGravatarScanner:# Verify Neo4j queries were executedassert mock_neo4j.query.call_count == 2-+# Check that results are returned unchangedassert result == gravatars@@ -261,7 +270,7 @@ class TestEmailToGravatarScanner:def test_execute_full_workflow(self):"""Test the complete execute workflow"""- with patch('requests.get') as mock_get:+ with patch("requests.get") as mock_get:# Mock successful responsemock_response = Mock()mock_response.status_code = 200@@ -272,18 +281,20 @@ class TestEmailToGravatarScanner:assert len(result) == 1assert isinstance(result[0], Gravatar)- assert result[0].hash == hashlib.md5("[email protected]".encode()).hexdigest()+ assert (+ result[0].hash == hashlib.md5("[email protected]".encode()).hexdigest()+ )def test_execute_with_invalid_input(self):"""Test execute with invalid input"""emails = ["not-an-email", "also-invalid"]-- with patch('requests.get') as mock_get:++ with patch("requests.get") as mock_get:# Mock successful response for any requestmock_response = Mock()mock_response.status_code = 200mock_get.return_value = mock_response-+result = scanner.execute(emails)# The scanner processes any string as an email, so it will create Email objects@@ -295,8 +306,8 @@ class TestEmailToGravatarScanner:"""Test that gravatar hash is calculated correctly"""email = "[email protected]"expected_hash = hashlib.md5(email.encode()).hexdigest()-- with patch('requests.get') as mock_get:++ with patch("requests.get") as mock_get:mock_response = Mock()mock_response.status_code = 200mock_get.return_value = mock_response@@ -312,8 +323,8 @@ class TestEmailToGravatarScanner:email = "[email protected]"expected_hash = hashlib.md5(email.encode()).hexdigest()expected_url = f"https://www.gravatar.com/avatar/{expected_hash}"-- with patch('requests.get') as mock_get:++ with patch("requests.get") as mock_get:mock_response = Mock()mock_response.status_code = 200mock_get.return_value = mock_response@@ -322,4 +333,4 @@ class TestEmailToGravatarScanner:result = scanner.scan(emails)assert len(result) == 1- assert str(result[0].src) == expected_url\ No newline at end of file+ assert str(result[0].src) == expected_urldiff --git a/flowsint-transforms/tests/scanners/emails/to_leaks.py b/flowsint-transforms/tests/scanners/emails/to_leaks.pyindex ce099e18..0880682e 100644--- a/flowsint-transforms/tests/scanners/emails/to_leaks.py+++ b/flowsint-transforms/tests/scanners/emails/to_leaks.py@@ -6,15 +6,19 @@ from flowsint_types.breach import Breachscanner = EmailToBreachesScanner("sketch_123", "scan_123")+def test_scanner_name():assert EmailToBreachesScanner.name() == "to_leaks"+def test_scanner_category():assert EmailToBreachesScanner.category() == "Email"+def test_scanner_key():assert EmailToBreachesScanner.key() == "email"+def test_preprocess_string_emails():emails = [@@ -24,6 +28,7 @@ def test_preprocess_string_emails():expected_emails = [Email(email=email) for email in emails]assert result == expected_emails+def test_preprocess_dict_emails():emails = [{"email": "[email protected]"},@@ -33,6 +38,7 @@ def test_preprocess_dict_emails():expected_emails = [Email(email=email["email"]) for email in emails]assert result == expected_emails+def test_preprocess_email_objects():emails = [Email(email="[email protected]"),@@ -41,6 +47,7 @@ def test_preprocess_email_objects():result = scanner.preprocess(emails)assert result == emails+def test_preprocess_mixed_formats():emails = [@@ -49,27 +56,28 @@ def test_preprocess_mixed_formats():{"invalid_key": "[email protected]"},]result = scanner.preprocess(emails)-+result_emails = [email.email for email in result]assert "[email protected]" in result_emailsassert "[email protected]" in result_emailsassert "[email protected]" in result_emailsassert "[email protected]" not in result_emails-@patch('src.transforms.emails.to_leaks.requests.get')++@patch("src.transforms.emails.to_leaks.requests.get")def test_scan_successful_response(mock_get):# Mock successful API responsemock_response = MagicMock()mock_response.status_code = 200mock_response.json.return_value = [{"Name": "TestBreach", "Title": "Test Breach", "Domain": "test.com"},- {"Name": "AnotherBreach", "Title": "Another Breach", "Domain": "another.com"}+ {"Name": "AnotherBreach", "Title": "Another Breach", "Domain": "another.com"},]mock_get.return_value = mock_response-+emails = [Email(email="[email protected]")]result = scanner.scan(emails)-+assert len(result) == 2assert isinstance(result[0], Breach)assert isinstance(result[1], Breach)@@ -78,85 +86,90 @@ def test_scan_successful_response(mock_get):assert result[0].breach["name"] == "testbreach"assert result[1].breach["name"] == "anotherbreach"-@patch('src.transforms.emails.to_leaks.requests.get')++@patch("src.transforms.emails.to_leaks.requests.get")def test_scan_no_breaches_found(mock_get):# Mock 404 response (no breaches found)mock_response = MagicMock()mock_response.status_code = 404mock_get.return_value = mock_response-+emails = [Email(email="[email protected]")]result = scanner.scan(emails)-+assert len(result) == 0-@patch('src.transforms.emails.to_leaks.requests.get')++@patch("src.transforms.emails.to_leaks.requests.get")def test_scan_api_error(mock_get):# Mock API errormock_get.side_effect = Exception("API Error")-+emails = [Email(email="[email protected]")]result = scanner.scan(emails)-+assert len(result) == 0-@patch('src.transforms.emails.to_leaks.requests.get')++@patch("src.transforms.emails.to_leaks.requests.get")def test_scan_missing_name_field(mock_get):# Mock API response with missing "Name" fieldmock_response = MagicMock()mock_response.status_code = 200mock_response.json.return_value = [{"Title": "Test Breach", "Domain": "test.com"}, # Missing "Name" field- {"Name": "ValidBreach", "Title": "Valid Breach", "Domain": "valid.com"}+ {"Name": "ValidBreach", "Title": "Valid Breach", "Domain": "valid.com"},]mock_get.return_value = mock_response-+emails = [Email(email="[email protected]")]result = scanner.scan(emails)-+assert len(result) == 2assert result[0].name == "unknown" # Should default to "unknown"assert result[1].name == "validbreach" # Should use the provided nameassert result[0].breach["title"] == "Test Breach"assert result[1].breach["name"] == "validbreach"-@patch('src.transforms.emails.to_leaks.HIBP_API_KEY', None)++@patch("src.transforms.emails.to_leaks.HIBP_API_KEY", None)def test_scan_no_api_key():"""Test that scanner raises ValueError when HIBP_API_KEY is not set."""emails = [Email(email="[email protected]")]-+with pytest.raises(ValueError, match="HIBP_API_KEY not set"):scanner.scan(emails)+def test_postprocess():# Test postprocess method with mocked neo4j connectionscanner.neo4j_conn = MagicMock()-+# Create breach objects with the new structurebreach1 = Breach(name="testbreach",title="Test Breach",domain="test.com",pwncount=1000,- breach={"name": "testbreach", "title": "Test Breach"}+ breach={"name": "testbreach", "title": "Test Breach"},)breach2 = Breach(- name="anotherbreach",+ name="anotherbreach",title="Another Breach",domain="another.com",pwncount=2000,- breach={"name": "anotherbreach", "title": "Another Breach"}+ breach={"name": "anotherbreach", "title": "Another Breach"},)-+breaches = [breach1, breach2]original_input = [Email(email="[email protected]")]-+result = scanner.postprocess(breaches, original_input)-+assert result == breaches# Verify that neo4j queries were called:# - 2 breach node creation queries- # - 1 email node creation query+ # - 1 email node creation query# - 2 relationship creation queries# Total: 5 queries- assert scanner.neo4j_conn.query.call_count == 5\ No newline at end of file+ assert scanner.neo4j_conn.query.call_count == 5diff --git a/flowsint-transforms/tests/scanners/ips/asn_to_cidrs.py b/flowsint-transforms/tests/scanners/ips/asn_to_cidrs.pyindex 234ae660..fc40c860 100644--- a/flowsint-transforms/tests/scanners/ips/asn_to_cidrs.py+++ b/flowsint-transforms/tests/scanners/ips/asn_to_cidrs.py@@ -3,18 +3,20 @@ from flowsint_types.asn import ASNscanner = AsnToCidrsScanner("sketch_123", "scan_123")+def test_preprocess_valid_asns():asns = [ASN(number=15169),ASN(number=13335),]result = scanner.preprocess(asns)-+result_numbers = [asn.number for asn in result]expected_numbers = [asn.number for asn in asns]assert result_numbers == expected_numbers-++def test_unprocessed_valid_asns():asns = ["15169",@@ -23,8 +25,9 @@ def test_unprocessed_valid_asns():result = scanner.preprocess(asns)result_asns = [asn for asn in result]expected_asns = [ASN(number=int(asn)) for asn in asns]- assert result_asns == expected_asns-+ assert result_asns == expected_asns++def test_preprocess_invalid_asns():asns = [ASN(number=15169),@@ -38,6 +41,7 @@ def test_preprocess_invalid_asns():assert 13335 in result_numbersassert 999999999999 not in result_numbers+def test_preprocess_multiple_formats():asns = [{"number": 15169},@@ -50,19 +54,23 @@ def test_preprocess_multiple_formats():result_numbers = [asn.number for asn in result]assert 15169 in result_numbersassert 13335 in result_numbers- assert "invalid_key" not in result_numbers # Should be filtered out due to invalid key+ assert (+ "invalid_key" not in result_numbers+ ) # Should be filtered out due to invalid keydef test_schemas():input_schema = scanner.input_schema()output_schema = scanner.output_schema()-+# Input schema should have number fieldassert "properties" in input_schema- number_prop = next((prop for prop in input_schema["properties"] if prop["name"] == "number"), None)+ number_prop = next(+ (prop for prop in input_schema["properties"] if prop["name"] == "number"), None+ )assert number_prop is not Noneassert number_prop["type"] == "integer"-+# Output schema should have network fieldassert "properties" in output_schemaprop_names = [prop["name"] for prop in output_schema["properties"]]diff --git a/flowsint-transforms/tests/scanners/ips/cidr_to_ips.py b/flowsint-transforms/tests/scanners/ips/cidr_to_ips.pyindex 76a0f036..0803c9da 100644--- a/flowsint-transforms/tests/scanners/ips/cidr_to_ips.py+++ b/flowsint-transforms/tests/scanners/ips/cidr_to_ips.py@@ -6,18 +6,20 @@ from tests.logger import TestLoggerlogger = TestLogger()scanner = CidrToIpsScanner("sketch_123", "scan_123", logger)+def test_preprocess_valid_cidrs():cidrs = [CIDR(network="8.8.8.0/24"),CIDR(network="1.1.1.0/24"),]result = scanner.preprocess(cidrs)-+result_networks = [cidr.network for cidr in result]expected_networks = [cidr.network for cidr in cidrs]assert result_networks == expected_networks+def test_preprocess_unprocessed_valid_cidrs():cidrs = ["8.8.8.0/24",@@ -28,6 +30,7 @@ def test_preprocess_unprocessed_valid_cidrs():expected_cidrs = [CIDR(network=c) for c in cidrs]assert result_cidrs == expected_cidrs+def test_preprocess_invalid_cidrs():cidrs = [CIDR(network="8.8.8.0/24"),@@ -40,6 +43,7 @@ def test_preprocess_invalid_cidrs():assert "invalid-cidr" not in result_networksassert "not-a-cidr" not in result_networks+def test_preprocess_multiple_formats():cidrs = [{"network": "8.8.8.0/24"},@@ -54,6 +58,7 @@ def test_preprocess_multiple_formats():assert "1.1.1.0/24" not in result_networksassert "InvalidCIDR" not in result_networks+def test_scan_extracts_ips(monkeypatch):mock_dnsx_output = """8.35.200.128.35.200.112@@ -75,21 +80,17 @@ def test_scan_extracts_ips(monkeypatch):input_data = [CIDR(network="8.35.200.0/24")]ips = scanner.scan(input_data)-+assert isinstance(ips, list)assert len(ips) == 4-- expected_ips = [- "8.35.200.12",- "8.35.200.112",- "8.35.200.16",- "8.35.200.170"- ]-++ expected_ips = ["8.35.200.12", "8.35.200.112", "8.35.200.16", "8.35.200.170"]+for ip in ips:assert isinstance(ip, Ip)assert ip.address in expected_ips+def test_scan_handles_empty_output(monkeypatch):class MockSubprocessResult:def __init__(self):@@ -103,10 +104,11 @@ def test_scan_handles_empty_output(monkeypatch):input_data = [CIDR(network="8.8.8.0/24")]ips = scanner.scan(input_data)-+assert isinstance(ips, list)assert len(ips) == 0+def test_scan_handles_subprocess_exception(monkeypatch):def mock_subprocess_run(cmd, shell, capture_output, text, timeout):raise Exception("Subprocess failed")@@ -115,6 +117,6 @@ def test_scan_handles_subprocess_exception(monkeypatch):input_data = [CIDR(network="8.8.8.0/24")]ips = scanner.scan(input_data)-+assert isinstance(ips, list)- assert len(ips) == 0\ No newline at end of file+ assert len(ips) == 0diff --git a/flowsint-transforms/tests/scanners/ips/geolocation.py b/flowsint-transforms/tests/scanners/ips/geolocation.pyindex 378beba8..607ef8b6 100644--- a/flowsint-transforms/tests/scanners/ips/geolocation.py+++ b/flowsint-transforms/tests/scanners/ips/geolocation.py@@ -3,6 +3,7 @@ from flowsint_types.ip import Ip, Ipscanner = GeolocationScanner("sketch_123", "scan_123")+def test_preprocess_valid_ips():ips = [Ip(address="8.8.8.8"),@@ -13,6 +14,7 @@ def test_preprocess_valid_ips():expected_ips = [d.address for d in ips]assert result_ips == expected_ips+def test_preprocess_string_ips():ips = ["8.8.8.8",@@ -23,6 +25,7 @@ def test_preprocess_string_ips():expected_ips = [d for d in ips]assert [ip.address for ip in result] == expected_ips+def test_preprocess_invalid_ips():ips = [Ip(address="8.8.8.8"),@@ -35,6 +38,7 @@ def test_preprocess_invalid_ips():assert "1.1.1.1" in result_ipsassert "invalid_ip" not in result_ips+def test_preprocess_multiple_formats():ips = [{"address": "8.8.8.8"},@@ -48,6 +52,7 @@ def test_preprocess_multiple_formats():assert "1.1.1.1" in result_ipsassert "1.2.3.4" not in result_ips+def test_scan_returns_ip(monkeypatch):# Mock of get_location_datadef mock_get_location_data(address):@@ -56,7 +61,7 @@ def test_scan_returns_ip(monkeypatch):"longitude": -122.0838,"country": "US","city": "Mountain View",- "isp": "Google LLC"+ "isp": "Google LLC",}monkeypatch.setattr(scanner, "get_location_data", mock_get_location_data)@@ -70,9 +75,29 @@ def test_scan_returns_ip(monkeypatch):assert output[0].country == "US"assert output[0].isp == "Google LLC"+def test_schemas():input_schema = scanner.input_schema()output_schema = scanner.output_schema()- assert input_schema == {'type': 'Ip', 'properties': [{'name': 'address', 'type': 'string'}, {'name': 'latitude', 'type': 'number | null'}, {'name': 'longitude', 'type': 'number | null'}, {'name': 'country', 'type': 'string | null'}, {'name': 'city', 'type': 'string | null'}, {'name': 'isp', 'type': 'string | null'}]}- assert output_schema == {'type': 'Ip', 'properties': [{'name': 'address', 'type': 'string'}, {'name': 'latitude', 'type': 'number | null'}, {'name': 'longitude', 'type': 'number | null'}, {'name': 'country', 'type': 'string | null'}, {'name': 'city', 'type': 'string | null'}, {'name': 'isp', 'type': 'string | null'}]}-+ assert input_schema == {+ "type": "Ip",+ "properties": [+ {"name": "address", "type": "string"},+ {"name": "latitude", "type": "number | null"},+ {"name": "longitude", "type": "number | null"},+ {"name": "country", "type": "string | null"},+ {"name": "city", "type": "string | null"},+ {"name": "isp", "type": "string | null"},+ ],+ }+ assert output_schema == {+ "type": "Ip",+ "properties": [+ {"name": "address", "type": "string"},+ {"name": "latitude", "type": "number | null"},+ {"name": "longitude", "type": "number | null"},+ {"name": "country", "type": "string | null"},+ {"name": "city", "type": "string | null"},+ {"name": "isp", "type": "string | null"},+ ],+ }diff --git a/flowsint-transforms/tests/scanners/ips/ip_to_asn.py b/flowsint-transforms/tests/scanners/ips/ip_to_asn.pyindex 769f81d7..091017e7 100644--- a/flowsint-transforms/tests/scanners/ips/ip_to_asn.py+++ b/flowsint-transforms/tests/scanners/ips/ip_to_asn.py@@ -10,18 +10,20 @@ logger = TestLogger()# The scanner will get a mock logger from conftest.py automaticallyscanner = IpToAsnScanner("sketch_123", "scan_123", logger)+def test_preprocess_valid_ips():ips = [Ip(address="8.8.8.8"),Ip(address="1.1.1.1"),]result = scanner.preprocess(ips)-+result_addresses = [ip.address for ip in result]expected_addresses = [ip.address for ip in ips]assert result_addresses == expected_addresses-++def test_unprocessed_valid_ips():ips = ["8.8.8.8",@@ -30,8 +32,9 @@ def test_unprocessed_valid_ips():result = scanner.preprocess(ips)result_ips = [ip for ip in result]expected_ips = [Ip(address=ip) for ip in ips]- assert result_ips == expected_ips-+ assert result_ips == expected_ips++def test_preprocess_invalid_ips():ips = [Ip(address="8.8.8.8"),@@ -45,6 +48,7 @@ def test_preprocess_invalid_ips():assert "192.168.1.1" in result_addressesassert "invalid_ip" not in result_addresses+def test_preprocess_multiple_formats():ips = [{"address": "8.8.8.8"},@@ -58,7 +62,9 @@ def test_preprocess_multiple_formats():assert "8.8.8.8" in result_addressesassert "192.168.1.1" in result_addressesassert "10.0.0.1" in result_addresses- assert "1.1.1.1" not in result_addresses # Should be filtered out due to invalid key+ assert (+ "1.1.1.1" not in result_addresses+ ) # Should be filtered out due to invalid keydef test_scan_extracts_asn_info(monkeypatch):@@ -67,7 +73,7 @@ def test_scan_extracts_asn_info(monkeypatch):"as_number": "AS15169","as_name": "GOOGLE","as_country": "US",- "as_range": ["8.8.8.0/24", "8.8.4.0/24"]+ "as_range": ["8.8.8.0/24", "8.8.4.0/24"],}class MockSubprocessResult:@@ -85,10 +91,10 @@ def test_scan_extracts_asn_info(monkeypatch):input_data = [Ip(address="8.8.8.8")]asns = scanner.scan(input_data)-+assert isinstance(asns, list)assert len(asns) == 1-+asn = asns[0]assert isinstance(asn, ASN)assert asn.number == 15169 # AS15169 -> 15169@@ -113,10 +119,10 @@ def test_scan_handles_no_asn_found(monkeypatch):input_data = [Ip(address="192.168.1.1")]asns = scanner.scan(input_data)-+assert isinstance(asns, list)assert len(asns) == 1-+asn = asns[0]assert isinstance(asn, ASN)assert asn.number == 0@@ -133,10 +139,10 @@ def test_scan_handles_subprocess_exception(monkeypatch):input_data = [Ip(address="8.8.8.8")]asns = scanner.scan(input_data)-+assert isinstance(asns, list)assert len(asns) == 1-+asn = asns[0]assert isinstance(asn, ASN)assert asn.number == 0@@ -151,15 +157,15 @@ def test_scan_multiple_ips(monkeypatch):"as_number": "AS15169","as_name": "GOOGLE","as_country": "US",- "as_range": ["8.8.8.0/24"]+ "as_range": ["8.8.8.0/24"],},"1.1.1.1": {"input": "1.1.1.1","as_number": "AS13335","as_name": "CLOUDFLARE","as_country": "US",- "as_range": ["1.1.1.0/24"]- }+ "as_range": ["1.1.1.0/24"],+ },}class MockSubprocessResult:@@ -176,13 +182,13 @@ def test_scan_multiple_ips(monkeypatch):input_data = [Ip(address="8.8.8.8"), Ip(address="1.1.1.1")]asns = scanner.scan(input_data)-+assert len(asns) == 2-+# Check first ASNassert asns[0].number == 15169assert asns[0].name == "GOOGLE"-+# Check second ASNassert asns[1].number == 13335assert asns[1].name == "CLOUDFLARE"@@ -191,13 +197,15 @@ def test_scan_multiple_ips(monkeypatch):def test_schemas():input_schema = scanner.input_schema()output_schema = scanner.output_schema()-+# Input schema should have address fieldassert "properties" in input_schema- address_prop = next((prop for prop in input_schema["properties"] if prop["name"] == "address"), None)+ address_prop = next(+ (prop for prop in input_schema["properties"] if prop["name"] == "address"), None+ )assert address_prop is not Noneassert address_prop["type"] == "string"-+# Output schema should have ASN fieldsassert "properties" in output_schemaprop_names = [prop["name"] for prop in output_schema["properties"]]@@ -211,20 +219,22 @@ def test_postprocess_creates_neo4j_relationships(monkeypatch):# Mock Neo4j connectionmock_neo4j = Mock()scanner.neo4j_conn = mock_neo4j-+input_data = [Ip(address="8.8.8.8")]- asn_results = [ASN(- number=15169,- name="GOOGLE",- country="US",- cidrs=[CIDR(network="8.8.8.0/24")]- )]-+ asn_results = [+ ASN(+ number=15169,+ name="GOOGLE",+ country="US",+ cidrs=[CIDR(network="8.8.8.0/24")],+ )+ ]+result = scanner.postprocess(asn_results, input_data)-+# Verify Neo4j query was calledmock_neo4j.query.assert_called_once()-+# Check the query parameterscall_args = mock_neo4j.query.call_argsparams = call_args[0][1]@@ -233,7 +243,7 @@ def test_postprocess_creates_neo4j_relationships(monkeypatch):assert params["asn_name"] == "GOOGLE"assert params["asn_country"] == "US"assert params["sketch_id"] == "sketch_123"-+# Should return the same resultsassert result == asn_results@@ -242,19 +252,16 @@ def test_postprocess_skips_unknown_asns(monkeypatch):# Mock Neo4j connectionmock_neo4j = Mock()scanner.neo4j_conn = mock_neo4j-+input_data = [Ip(address="192.168.1.1")]- asn_results = [ASN(- number=0, # Unknown ASN- name="Unknown",- country="Unknown",- cidrs=[]- )]-+ asn_results = [+ ASN(number=0, name="Unknown", country="Unknown", cidrs=[]) # Unknown ASN+ ]+result = scanner.postprocess(asn_results, input_data)-+# Verify Neo4j query was NOT called for unknown ASNmock_neo4j.query.assert_not_called()-+# Should return the same results- assert result == asn_results\ No newline at end of file+ assert result == asn_resultsdiff --git a/flowsint-transforms/tests/scanners/organizations/to_infos.py b/flowsint-transforms/tests/scanners/organizations/to_infos.pyindex c61720cb..26eb180f 100644--- a/flowsint-transforms/tests/scanners/organizations/to_infos.py+++ b/flowsint-transforms/tests/scanners/organizations/to_infos.py@@ -3,17 +3,15 @@ from flowsint_types.organization import Organizationscanner = OrgToInfosScanner("sketch_123", "scan_123")+def test_preprocess_valid_names():- data = [- Organization(name="OpenAI"),- {"name": "Inria"},- "OVH"- ]+ data = [Organization(name="OpenAI"), {"name": "Inria"}, "OVH"]result = scanner.preprocess(data)result_names = [org.name for org in result]assert result_names == ["OpenAI", "Inria", "OVH"]+# def test_preprocess_invalid_entries():# data = [# {"wrong_key": "value"},@@ -25,6 +23,7 @@ def test_preprocess_valid_names():# result = scanner.preprocess(data)# assert result == []+def test_execute():scanner.execute(["Karim Terrache"])- assert True\ No newline at end of file+ assert Truediff --git a/flowsint-transforms/tests/scanners/socials/maigret.py b/flowsint-transforms/tests/scanners/socials/maigret.pyindex 289b739d..e0c368a7 100644--- a/flowsint-transforms/tests/scanners/socials/maigret.py+++ b/flowsint-transforms/tests/scanners/socials/maigret.py@@ -1,9 +1,10 @@from pathlib import Pathfrom flowsint_transforms.socials.maigret import MaigretScanner-from flowsint_types.social import Social+from flowsint_types.social import SocialProfilescanner = MaigretScanner("sketch_123", "scan_123")+def test_unprocessed_valid_usernames():usernames = ["toto123",@@ -11,14 +12,15 @@ def test_unprocessed_valid_usernames():]result = scanner.preprocess(usernames)result_usernames = [d for d in result]- expected_usernames = [Social(username=d) for d in usernames]- assert result_usernames == expected_usernames-+ expected_usernames = [SocialProfile(username=d) for d in usernames]+ assert result_usernames == expected_usernames++def test_preprocess_invalid_usernames():usernames = [- Social(username="toto123"),- Social(username="DorianXd78_Official"),- Social(username="This is not a username"),+ SocialProfile(username="toto123"),+ SocialProfile(username="DorianXd78_Official"),+ SocialProfile(username="This is not a username"),]result = scanner.preprocess(usernames)@@ -27,11 +29,12 @@ def test_preprocess_invalid_usernames():assert "DorianXd78_Official" in result_usernamesassert "This is not a username" not in result_usernames+def test_preprocess_multiple_formats():usernames = [{"username": "toto123"},{"invalid_key": "ValId_UseRnAme"},- Social(username="DorianXd78_Official"),+ SocialProfile(username="DorianXd78_Official"),"MySimpleUsername",]result = scanner.preprocess(usernames)@@ -41,13 +44,14 @@ def test_preprocess_multiple_formats():assert "DorianXd78_Official" in result_usernamesassert "ValId_UseRnAme" not in result_usernamesassert "MySimpleUsername" in result_usernames-++def test_parsing_invalid_output_file():results = scanner.parse_maigret_output("toto123", Path("/this/path/does/not/exist"))assert results == []-++def test_parsing():results = scanner.parse_maigret_output("toto123", Path("/tmp/maigret_test.json"))print(results)assert len(results) == 2-diff --git a/flowsint-transforms/tests/scanners/websites/to_links.py b/flowsint-transforms/tests/scanners/websites/to_links.pyindex 2a8dff58..02c668ab 100644--- a/flowsint-transforms/tests/scanners/websites/to_links.py+++ b/flowsint-transforms/tests/scanners/websites/to_links.py@@ -1,5 +1,5 @@import pytest-from unittest.mock import Mock, patch, call+from unittest.mock import Mock, patchfrom flowsint_transforms.websites.to_links import WebsiteToLinksfrom flowsint_types.website import Website@@ -14,10 +14,10 @@ class MockCrawler:def __init__(self, url, recursive=True, verbose=False, _on_result_callback=None):self.url = urlself.callback = _on_result_callback-+def fetch(self):pass-+def extract_urls(self):# Simulate callback callsif self.callback:@@ -25,11 +25,14 @@ class MockCrawler:self.callback("https://example.com/page2", is_external=False)self.callback("https://external.com/page", is_external=True)self.callback("https://another-external.org/resource", is_external=True)-+def get_results(self):return MockCrawlResults(internal=["https://example.com/page1", "https://example.com/page2"],- external=["https://external.com/page", "https://another-external.org/resource"]+ external=[+ "https://external.com/page",+ "https://another-external.org/resource",+ ],)@@ -37,102 +40,210 @@ class MockCrawler:async def test_website_to_links_real_time_neo4j_creation():"""Test that Neo4j nodes are created in real-time during the callback."""scanner = WebsiteToLinks(sketch_id="test", scan_id="test")-+# Mock neo4j connection and methodsscanner.neo4j_conn = Mock()scanner.create_node = Mock()scanner.create_relationship = Mock()scanner.log_graph_message = Mock()-+# Test inputwebsites = [Website(url="https://example.com")]-- with patch('src.transforms.websites.to_links.Crawler', MockCrawler):++ with patch("src.transforms.websites.to_links.Crawler", MockCrawler):results = await scanner.scan(websites)-+# Verify main website and domain nodes were created upfront- scanner.create_node.assert_any_call('website', 'url', 'https://example.com',- caption='https://example.com', type='website')- scanner.create_node.assert_any_call('domain', 'name', 'example.com',- caption='example.com', type='domain')-+ scanner.create_node.assert_any_call(+ "website",+ "url",+ "https://example.com",+ caption="https://example.com",+ type="website",+ )+ scanner.create_node.assert_any_call(+ "domain", "name", "example.com", caption="example.com", type="domain"+ )+# Verify main website to domain relationship- scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com',- 'domain', 'name', 'example.com', 'BELONGS_TO_DOMAIN')-+ scanner.create_relationship.assert_any_call(+ "website",+ "url",+ "https://example.com",+ "domain",+ "name",+ "example.com",+ "BELONGS_TO_DOMAIN",+ )+# Verify internal website nodes were created in callback- scanner.create_node.assert_any_call('website', 'url', 'https://example.com/page1',- caption='https://example.com/page1', type='website')- scanner.create_node.assert_any_call('website', 'url', 'https://example.com/page2',- caption='https://example.com/page2', type='website')-+ scanner.create_node.assert_any_call(+ "website",+ "url",+ "https://example.com/page1",+ caption="https://example.com/page1",+ type="website",+ )+ scanner.create_node.assert_any_call(+ "website",+ "url",+ "https://example.com/page2",+ caption="https://example.com/page2",+ type="website",+ )+# Verify internal website relationships- scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com',- 'website', 'url', 'https://example.com/page1', 'LINKS_TO')- scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com',- 'website', 'url', 'https://example.com/page2', 'LINKS_TO')-+ scanner.create_relationship.assert_any_call(+ "website",+ "url",+ "https://example.com",+ "website",+ "url",+ "https://example.com/page1",+ "LINKS_TO",+ )+ scanner.create_relationship.assert_any_call(+ "website",+ "url",+ "https://example.com",+ "website",+ "url",+ "https://example.com/page2",+ "LINKS_TO",+ )+# Verify external website nodes were created in callback- scanner.create_node.assert_any_call('website', 'url', 'https://external.com/page',- caption='https://external.com/page', type='website')- scanner.create_node.assert_any_call('website', 'url', 'https://another-external.org/resource',- caption='https://another-external.org/resource', type='website')-+ scanner.create_node.assert_any_call(+ "website",+ "url",+ "https://external.com/page",+ caption="https://external.com/page",+ type="website",+ )+ scanner.create_node.assert_any_call(+ "website",+ "url",+ "https://another-external.org/resource",+ caption="https://another-external.org/resource",+ type="website",+ )+# Verify external domain nodes were created in callback- scanner.create_node.assert_any_call('domain', 'name', 'external.com',- caption='external.com', type='domain')- scanner.create_node.assert_any_call('domain', 'name', 'another-external.org',- caption='another-external.org', type='domain')-+ scanner.create_node.assert_any_call(+ "domain", "name", "external.com", caption="external.com", type="domain"+ )+ scanner.create_node.assert_any_call(+ "domain",+ "name",+ "another-external.org",+ caption="another-external.org",+ type="domain",+ )+# Verify main website to external website relationships- scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com',- 'website', 'url', 'https://external.com/page', 'LINKS_TO')- scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com',- 'website', 'url', 'https://another-external.org/resource', 'LINKS_TO')-+ scanner.create_relationship.assert_any_call(+ "website",+ "url",+ "https://example.com",+ "website",+ "url",+ "https://external.com/page",+ "LINKS_TO",+ )+ scanner.create_relationship.assert_any_call(+ "website",+ "url",+ "https://example.com",+ "website",+ "url",+ "https://another-external.org/resource",+ "LINKS_TO",+ )+# Verify external website to domain relationships- scanner.create_relationship.assert_any_call('website', 'url', 'https://external.com/page',- 'domain', 'name', 'external.com', 'BELONGS_TO_DOMAIN')- scanner.create_relationship.assert_any_call('website', 'url', 'https://another-external.org/resource',- 'domain', 'name', 'another-external.org', 'BELONGS_TO_DOMAIN')-+ scanner.create_relationship.assert_any_call(+ "website",+ "url",+ "https://external.com/page",+ "domain",+ "name",+ "external.com",+ "BELONGS_TO_DOMAIN",+ )+ scanner.create_relationship.assert_any_call(+ "website",+ "url",+ "https://another-external.org/resource",+ "domain",+ "name",+ "another-external.org",+ "BELONGS_TO_DOMAIN",+ )+# Verify main website to external domain relationships- scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com',- 'domain', 'name', 'external.com', 'LINKS_TO_DOMAIN')- scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com',- 'domain', 'name', 'another-external.org', 'LINKS_TO_DOMAIN')+ scanner.create_relationship.assert_any_call(+ "website",+ "url",+ "https://example.com",+ "domain",+ "name",+ "external.com",+ "LINKS_TO_DOMAIN",+ )+ scanner.create_relationship.assert_any_call(+ "website",+ "url",+ "https://example.com",+ "domain",+ "name",+ "another-external.org",+ "LINKS_TO_DOMAIN",+ )@pytest.mark.asyncioasync def test_website_to_links_error_handling_with_neo4j():"""Test that main nodes are still created even when crawling fails."""scanner = WebsiteToLinks(sketch_id="test", scan_id="test")-+# Mock neo4j connection and methodsscanner.neo4j_conn = Mock()scanner.create_node = Mock()scanner.create_relationship = Mock()scanner.log_graph_message = Mock()-+# Mock crawler that raises an exceptiondef mock_crawler_error(*args, **kwargs):raise Exception("Test error")-+websites = [Website(url="https://example.com")]-- with patch('src.transforms.websites.to_links.Crawler', mock_crawler_error):++ with patch("src.transforms.websites.to_links.Crawler", mock_crawler_error):results = await scanner.scan(websites)-+# Verify main website and domain nodes were still created despite error- scanner.create_node.assert_any_call('website', 'url', 'https://example.com',- caption='https://example.com', type='website')- scanner.create_node.assert_any_call('domain', 'name', 'example.com',- caption='example.com', type='domain')-+ scanner.create_node.assert_any_call(+ "website",+ "url",+ "https://example.com",+ caption="https://example.com",+ type="website",+ )+ scanner.create_node.assert_any_call(+ "domain", "name", "example.com", caption="example.com", type="domain"+ )+# Verify main website to domain relationship was created- scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com',- 'domain', 'name', 'example.com', 'BELONGS_TO_DOMAIN')-+ scanner.create_relationship.assert_any_call(+ "website",+ "url",+ "https://example.com",+ "domain",+ "name",+ "example.com",+ "BELONGS_TO_DOMAIN",+ )+# Verify result structureassert len(results) == 1result = results[0]@@ -146,17 +257,19 @@ async def test_website_to_links_error_handling_with_neo4j():def test_postprocess_simplified():"""Test that postprocess now just returns results as-is."""scanner = WebsiteToLinks(sketch_id="test", scan_id="test")-+original_input = [Website(url="https://example.com")]- results = [{- "website": "https://example.com",- "main_domain": "example.com",- "internal_urls": ["https://example.com/page1"],- "external_urls": ["https://external.com/page"],- "external_domains": ["external.com"]- }]-+ results = [+ {+ "website": "https://example.com",+ "main_domain": "example.com",+ "internal_urls": ["https://example.com/page1"],+ "external_urls": ["https://external.com/page"],+ "external_domains": ["external.com"],+ }+ ]+processed_results = scanner.postprocess(results, original_input)-+# Should just return the same results since Neo4j work is done in real-time- assert processed_results == results\ No newline at end of file+ assert processed_results == resultsdiff --git a/flowsint-transforms/tests/tools/network/asnmap.py b/flowsint-transforms/tests/tools/network/asnmap.pyindex f21f1736..12f83fbe 100644--- a/flowsint-transforms/tests/tools/network/asnmap.py+++ b/flowsint-transforms/tests/tools/network/asnmap.py@@ -1,42 +1,52 @@import refrom typing import Dict-from app.tools.network.asnmap import AsnmapTool+from tools.network.asnmap import AsnmapTooltool = AsnmapTool()+def test_name():assert tool.name() == "asnmap"+def test_description():assert tool.description() == "ASN mapping and network reconnaissance tool."-++def test_category():assert tool.category() == "ASN discovery"+def test_image():assert tool.get_image() == "projectdiscovery/asnmap"-++def test_install():tool.install()assert tool.is_installed() == True-++def test_version():tool.install()version = tool.version()# Check that version follows the expected format: v followed by digits and dots- assert re.match(r'^v[\d\.]+$', version)-+ assert re.match(r"^v[\d\.]+$", version)++def test_launch_no_api_key():import pytest+with pytest.raises(KeyError, match="Missing key"):- tool.launch("alliage.io", 'domain')+ tool.launch("alliage.io", "domain")+def test_launch_wrong_type():import pytest+with pytest.raises(ValueError, match="Invalid type: 'domains'"):- tool.launch("alliage.io", 'domains')-+ tool.launch("alliage.io", "domains")++def test_launch():- results = tool.launch("alliage.io", 'domain')+ results = tool.launch("alliage.io", "domain")assert isinstance(results, Dict)-\ No newline at end of filediff --git a/flowsint-transforms/tests/tools/network/httpx.py b/flowsint-transforms/tests/tools/network/httpx.pyindex 5e746acb..63af13a6 100644--- a/flowsint-transforms/tests/tools/network/httpx.py+++ b/flowsint-transforms/tests/tools/network/httpx.py@@ -1,40 +1,50 @@import refrom typing import List-from app.tools.network.httpx import HttpxTool+from tools.network.httpx import HttpxTooltool = HttpxTool()+def test_name():assert tool.name() == "httpx"+def test_description():- assert tool.description() == "An HTTP toolkit that probes services, web servers, and other valuable metadata."-+ assert (+ tool.description()+ == "An HTTP toolkit that probes services, web servers, and other valuable metadata."+ )++def test_category():assert tool.category() == "Web technologies enumeration"+def test_image():assert tool.get_image() == "projectdiscovery/httpx"-++def test_install():tool.install()assert tool.is_installed() == True-++def test_version():tool.install()version = tool.version()# Check that version follows the expected format: v followed by digits and dots- assert re.match(r'^v[\d\.]+$', version)-+ assert re.match(r"^v[\d\.]+$", version)++def test_launch():assert Trueresults = tool.launch("https://alliage.io")print(results)assert isinstance(results, List)-++def test_launch_unreached_host():assert Trueresults = tool.launch("https://this-is-not-a-valid-domain.local")assert isinstance(results, List)assert len(results) == 0-\ No newline at end of filediff --git a/flowsint-transforms/tests/tools/network/reconcrawl.py b/flowsint-transforms/tests/tools/network/reconcrawl.pyindex 1abbf1e4..0bffd125 100644--- a/flowsint-transforms/tests/tools/network/reconcrawl.py+++ b/flowsint-transforms/tests/tools/network/reconcrawl.py@@ -1,20 +1,23 @@-import re-from typing import Dict-from app.tools.network.reconcrawl import ReconCrawlTool+from tools.network.reconcrawl import ReconCrawlTooltool = ReconCrawlTool()+def test_name():assert tool.name() == "reconcrawl"+def test_description():- assert tool.description() == "Emails and phone numbers crawler from websites by analyzing their HTML and embedded scripts."-+ assert (+ tool.description()+ == "Emails and phone numbers crawler from websites by analyzing their HTML and embedded scripts."+ )++def test_category():assert tool.category() == "Crawler"-++def test_install():tool.install()assert tool.is_installed() == True--\ No newline at end of filediff --git a/flowsint-transforms/tests/tools/network/subfinder.py b/flowsint-transforms/tests/tools/network/subfinder.pyindex 444c3444..3219e214 100644--- a/flowsint-transforms/tests/tools/network/subfinder.py+++ b/flowsint-transforms/tests/tools/network/subfinder.py@@ -1,32 +1,38 @@import re-from app.tools.network.subfinder import SubfinderTool+from tools.network.subfinder import SubfinderTooltool = SubfinderTool()+def test_name():assert tool.name() == "subfinder"+def test_description():assert tool.description() == "Fast passive subdomain enumeration tool."-++def test_category():assert tool.category() == "Subdomain enumeration"+def test_image():assert tool.get_image() == "projectdiscovery/subfinder"-++def test_install():tool.install()assert tool.is_installed() == True-++def test_version():tool.install()version = tool.version()# Check that version follows the expected format: v followed by digits and dots- assert re.match(r'^v[\d\.]+$', version)+ assert re.match(r"^v[\d\.]+$", version)+def test_launch():results = tool.launch("alliage.io")assert isinstance(results, list)assert all(isinstance(item, str) for item in results)-\ No newline at end of filediff --git a/flowsint-transforms/tests/tools/organizations/sirene.py b/flowsint-transforms/tests/tools/organizations/sirene.pyindex eb3f79be..762b3b98 100644--- a/flowsint-transforms/tests/tools/organizations/sirene.py+++ b/flowsint-transforms/tests/tools/organizations/sirene.py@@ -1,28 +1,36 @@-import refrom typing import Dict-from app.tools.organizations.sirene import SireneTool+from tools.organizations.sirene import SireneTooltool = SireneTool()+def test_name():assert tool.name() == "sirene"+def test_description():- assert tool.description() == "The Sirene API allows you to query the Sirene directory of businesses and establishments, managed by Insee."-+ assert (+ tool.description()+ == "The Sirene API allows you to query the Sirene directory of businesses and establishments, managed by Insee."+ )++def test_category():assert tool.category() == "Business intelligence"+def test_launch_org():results = tool.launch("blablacar", 1)assert isinstance(results, list)assert all(isinstance(item, Dict) for item in results)+def test_launch_person():results = tool.launch("Karim+Terrache", 1)assert isinstance(results, list)assert all(isinstance(item, Dict) for item in results)-++def test_launch_person_space_format():results = tool.launch("Karim Terrache", 1)assert isinstance(results, list)diff --git a/flowsint-transforms/tests/utils.py b/flowsint-transforms/tests/utils.pyindex d1576d96..d9affb08 100644--- a/flowsint-transforms/tests/utils.py+++ b/flowsint-transforms/tests/utils.py@@ -1,41 +1,40 @@from flowsint_core.core.scanner_base import build_params_model+def test_build_params_model_valid():param_schema = [- {- "name": "ETHERSCAN_API_KEY",- "type": "string",- "description": "The Etherscan API key to use for the transaction lookup.",- "required": True- },- {- "name": "url",- "type": "string",- "description": "Base URL for API",- "required": False,- "default": "https://api.etherscan.io/api"- }-]+ {+ "name": "ETHERSCAN_API_KEY",+ "type": "string",+ "description": "The Etherscan API key to use for the transaction lookup.",+ "required": True,+ },+ {+ "name": "url",+ "type": "string",+ "description": "Base URL for API",+ "required": False,+ "default": "https://api.etherscan.io/api",+ },+ ]ParamsModel = build_params_model(param_schema)validated_params = ParamsModel(ETHERSCAN_API_KEY="clef-123")assert validated_params.ETHERSCAN_API_KEY == "clef-123"assert validated_params.url == "https://api.etherscan.io/api"-++def test_build_params_model_invalid():param_schema = [- {-- },- {- "name": "url",- "type": "string",- "description": "Base URL for API",- "required": False,- "default": "https://api.etherscan.io/api"- }-]+ {},+ {+ "name": "url",+ "type": "string",+ "description": "Base URL for API",+ "required": False,+ "default": "https://api.etherscan.io/api",+ },+ ]ParamsModel = build_params_model(param_schema)validated_params = ParamsModel(ETHERSCAN_API_KEY="clef-123")assert validated_params.ETHERSCAN_API_KEY == "clef-123"assert validated_params.url == "https://api.etherscan.io/api"-\ No newline at end of filediff --git a/flowsint-transforms/utils.py b/flowsint-transforms/utils.pyindex b1e02a8e..238eed7c 100644--- a/flowsint-transforms/utils.py+++ b/flowsint-transforms/utils.py@@ -13,6 +13,7 @@ import inspectfrom typing import Any, Dict, Typefrom pydantic import BaseModel, TypeAdapter+def is_valid_ip(address: str) -> bool:try:ipaddress.ip_address(address)@@ -20,11 +21,13 @@ def is_valid_ip(address: str) -> bool:except ValueError:return False+def is_valid_username(username: str) -> bool:if not re.match(r"^[a-zA-Z0-9_-]{3,30}$", username):return Falsereturn True+def is_valid_email(email: str) -> bool:if not re.match(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$", email):return False@@ -34,26 +37,29 @@ def is_valid_email(email: str) -> bool:def is_valid_domain(url_or_domain: str) -> str:try:- parsed = urlparse(url_or_domain if "://" in url_or_domain else "http://" + url_or_domain)+ parsed = urlparse(+ url_or_domain if "://" in url_or_domain else "http://" + url_or_domain+ )hostname = parsed.hostname or url_or_domain-+if not hostname or "." not in hostname:return False-+if not re.match(r"^[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$", hostname):return False-+return Trueexcept Exception as e:return False+def is_root_domain(domain: str) -> bool:"""Determine if a domain is a root domain or subdomain.-+Args:domain: The domain string to check-+Returns:True if it's a root domain (e.g., example.com), False if it's a subdomain (e.g., sub.example.com)"""@@ -62,20 +68,32 @@ def is_root_domain(domain: str) -> bool:if "://" in domain:parsed = urlparse(domain)domain = parsed.hostname or domain-+# Split by dots- parts = domain.split('.')-+ parts = domain.split(".")+# Handle common country code TLDs that have 2 parts (e.g., .co.uk, .com.au, .org.uk)- common_cc_tlds = ['.co.uk', '.com.au', '.org.uk', '.net.uk', '.gov.uk', '.ac.uk',- '.co.nz', '.com.sg', '.co.jp', '.co.kr', '.com.br', '.com.mx']-+ common_cc_tlds = [+ ".co.uk",+ ".com.au",+ ".org.uk",+ ".net.uk",+ ".gov.uk",+ ".ac.uk",+ ".co.nz",+ ".com.sg",+ ".co.jp",+ ".co.kr",+ ".com.br",+ ".com.mx",+ ]+# Check if the domain ends with a common country code TLDfor cc_tld in common_cc_tlds:if domain.endswith(cc_tld):# For country code TLDs, we need exactly 3 parts (e.g., example.co.uk)return len(parts) == 3-+# For regular TLDs, a root domain has 2 parts (e.g., example.com)# A subdomain has 3 or more parts (e.g., sub.example.com, www.sub.example.com)return len(parts) == 2@@ -83,6 +101,7 @@ def is_root_domain(domain: str) -> bool:# If we can't parse it, assume it's not a root domainreturn False+def is_valid_number(phone: str, region: str = "FR") -> None:"""Validates a phone number. Raises InvalidPhoneNumberError if invalid.@@ -94,19 +113,21 @@ def is_valid_number(phone: str, region: str = "FR") -> None:return Falseexcept NumberParseException:return False-++def parse_asn(asn: str) -> int:if not is_valid_asn(asn):raise ValueError(f"Invalid ASN format: {asn}")return int(re.sub(r"(?i)^AS", "", asn))-+def is_valid_asn(asn: str) -> bool:if not re.fullmatch(r"(AS)?\d+", asn, re.IGNORECASE):return Falseasn_num = int(re.sub(r"(?i)^AS", "", asn))return 0 <= asn_num <= 4294967295+def resolve_type(details: dict, schema_context: dict = None) -> str:if "anyOf" in details:types = []@@ -121,22 +142,23 @@ def resolve_type(details: dict, schema_context: dict = None) -> str:else:types.append(option.get("type", "unknown"))return " | ".join(types)-+if "type" in details:if details["type"] == "array":item_type = resolve_type(details.get("items", {}), schema_context)return f"{item_type}[]"return details["type"]-+# Handle $ref in array items or other contextsif "$ref" in details and schema_context:ref_path = details["$ref"]if ref_path.startswith("#/$defs/"):ref_name = ref_path.split("/")[-1]return ref_name-+return "any"+def extract_input_schema_transform(model: Type[BaseModel]) -> Dict[str, Any]:adapter = TypeAdapter(model)schema = adapter.json_schema()@@ -153,37 +175,30 @@ def extract_input_schema_transform(model: Type[BaseModel]) -> Dict[str, Any]:"outputs": {"type": type_name,"properties": [- {- "name": prop,- "type": resolve_type(info, schema)- }+ {"name": prop, "type": resolve_type(info, schema)}for prop, info in details.get("properties", {}).items()- ]- },- "inputs": {- "type": "",- "properties": []+ ],},+ "inputs": {"type": "", "properties": []},"type": "type","category": model.__name__,}-def get_domain_from_ssl(ip: str, port: int = 443) -> str | None:try:context = ssl.create_default_context()with socket.create_connection((ip, port), timeout=3) as sock:with context.wrap_socket(sock, server_hostname=ip) as ssock:cert = ssock.getpeercert()- subject = cert.get('subject', [])+ subject = cert.get("subject", [])for entry in subject:- if entry[0][0] == 'commonName':+ if entry[0][0] == "commonName":return entry[0][1]# Alternative: check subjectAltName- san = cert.get('subjectAltName', [])+ san = cert.get("subjectAltName", [])for typ, val in san:- if typ == 'DNS':+ if typ == "DNS":return valexcept Exception as e:print(f"SSL extraction failed for {ip}: {e}")@@ -208,12 +223,14 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:scanner_node = node_lookup.get(target_id)if scanner_node and scanner_node["data"]["type"] == "scanner":- scanners.append({- "scanner_name": scanner_node["data"]["name"],- "module": scanner_node["data"]["module"],- "input": source_handle,- "output": target_handle- })+ scanners.append(+ {+ "scanner_name": scanner_node["data"]["name"],+ "module": scanner_node["data"]["module"],+ "input": source_handle,+ "output": target_handle,+ }+ )return {"input": {@@ -221,25 +238,22 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:"outputs": input_output,},"scanners": scanners,- "scanner_names" : [scanner["scanner_name"] for scanner in scanners]-+ "scanner_names": [scanner["scanner_name"] for scanner in scanners],}-++def get_label_color(label: str) -> str:- color_map = {- 'subdomain': '#A5ABB6',- 'domain': '#68BDF6',- 'default': '#A5ABB6'- }-+ color_map = {"subdomain": "#A5ABB6", "domain": "#68BDF6", "default": "#A5ABB6"}+return color_map.get(label, color_map["default"])+def flatten(data_dict, prefix=""):"""Flattens a dictionary to contain only Neo4j-compatible property values.Neo4j supports primitive types (string, number, boolean) and arrays of those types.Args:- data_dict (dict): Dictionary to flatten+ data_dict (dict): Dictionary to flattenReturns:dict: Flattened dictionary with only Neo4j-compatible values"""@@ -250,13 +264,14 @@ def flatten(data_dict, prefix=""):if value is None:continueif isinstance(value, (str, int, float, bool)) or (- isinstance(value, list) and all(isinstance(item, (str, int, float, bool)) for item in value)+ isinstance(value, list)+ and all(isinstance(item, (str, int, float, bool)) for item in value)):key = f"{prefix}{key}"- flattened[key] = value+ flattened[key] = valuereturn flattened-+def get_inline_relationships(nodes: List[Any], edges: List[Any]) -> List[str]:"""Get the inline relationships for a list of nodes and edges.@@ -266,14 +281,15 @@ def get_inline_relationships(nodes: List[Any], edges: List[Any]) -> List[str]:source = next((node for node in nodes if node["id"] == edge["source"]), None)target = next((node for node in nodes if node["id"] == edge["target"]), None)if source and target:- relationships.append({"source":source, "edge":edge, "target":target})+ relationships.append({"source": source, "edge": edge, "target": target})return relationships+def to_json_serializable(obj):"""Convert any object to a JSON-serializable format."""import jsonfrom pydantic import BaseModel-+try:# Test if already JSON serializablejson.dumps(obj)@@ -282,11 +298,15 @@ def to_json_serializable(obj):# Handle common casesif isinstance(obj, BaseModel):# Use mode='json' to ensure all Pydantic types are properly serialized- return obj.model_dump(mode='json') if hasattr(obj, 'model_dump') else obj.dict()+ return (+ obj.model_dump(mode="json")+ if hasattr(obj, "model_dump")+ else obj.dict()+ )elif isinstance(obj, (list, tuple)):return [to_json_serializable(item) for item in obj]elif isinstance(obj, dict):return {key: to_json_serializable(value) for key, value in obj.items()}else:# Convert anything else to string- return str(obj)\ No newline at end of file+ return str(obj)diff --git a/flowsint-types/src/flowsint_types/__init__.py b/flowsint-types/src/flowsint_types/__init__.pyindex 7bf9047c..6d7cf3d2 100644--- a/flowsint-types/src/flowsint_types/__init__.py+++ b/flowsint-types/src/flowsint_types/__init__.py@@ -44,7 +44,7 @@ __author__ = "EliottElek <[email protected]>"__all__ = ["PhysicalAddress",- "Affiliation",+ "Affiliation","Alias","ASN","BankAccount",diff --git a/flowsint-types/src/flowsint_types/address.py b/flowsint-types/src/flowsint_types/address.pyindex e7e84602..91621f27 100644--- a/flowsint-types/src/flowsint_types/address.py+++ b/flowsint-types/src/flowsint_types/address.py@@ -1,12 +1,17 @@from pydantic import BaseModel, Fieldfrom typing import Optional+class PhysicalAddress(BaseModel):"""Represents a physical address with geographical coordinates."""+address: str = Field(..., description="Street address", title="Street Address")city: str = Field(..., description="City name", title="City")country: str = Field(..., description="Country name", title="Country")zip: str = Field(..., description="ZIP or postal code", title="ZIP/Postal Code")- latitude: Optional[float] = Field(None, description="Latitude coordinate of the address", title="Latitude")- longitude: Optional[float] = Field(None, description="Longitude coordinate of the address", title="Longitude")-+ latitude: Optional[float] = Field(+ None, description="Latitude coordinate of the address", title="Latitude"+ )+ longitude: Optional[float] = Field(+ None, description="Longitude coordinate of the address", title="Longitude"+ )diff --git a/flowsint-types/src/flowsint_types/affiliation.py b/flowsint-types/src/flowsint_types/affiliation.pyindex 0c0527f3..9b594f11 100644--- a/flowsint-types/src/flowsint_types/affiliation.py+++ b/flowsint-types/src/flowsint_types/affiliation.py@@ -1,19 +1,55 @@from pydantic import BaseModel, Fieldfrom typing import Optional, List+class Affiliation(BaseModel):"""Represents an organizational affiliation or employment relationship."""- organization: str = Field(..., description="Organization or group name", title="Organization")- role: Optional[str] = Field(None, description="Role or position within organization", title="Role")- start_date: Optional[str] = Field(None, description="Start date of affiliation", title="Start Date")- end_date: Optional[str] = Field(None, description="End date of affiliation", title="End Date")- is_current: Optional[bool] = Field(None, description="Whether affiliation is currently active", title="Is Current")- department: Optional[str] = Field(None, description="Department or division", title="Department")- location: Optional[str] = Field(None, description="Geographic location of affiliation", title="Location")- industry: Optional[str] = Field(None, description="Industry or sector", title="Industry")- description: Optional[str] = Field(None, description="Description of affiliation", title="Description")- source: Optional[str] = Field(None, description="Source of affiliation information", title="Source")- confidence: Optional[float] = Field(None, description="Confidence score for affiliation", title="Confidence")- associated_individuals: Optional[List[str]] = Field(None, description="Individuals with similar affiliations", title="Associated Individuals")- organization_type: Optional[str] = Field(None, description="Type of organization (company, NGO, government, etc.)", title="Organization Type")- hierarchy_level: Optional[str] = Field(None, description="Hierarchical level within organization", title="Hierarchy Level")\ No newline at end of file++ organization: str = Field(+ ..., description="Organization or group name", title="Organization"+ )+ role: Optional[str] = Field(+ None, description="Role or position within organization", title="Role"+ )+ start_date: Optional[str] = Field(+ None, description="Start date of affiliation", title="Start Date"+ )+ end_date: Optional[str] = Field(+ None, description="End date of affiliation", title="End Date"+ )+ is_current: Optional[bool] = Field(+ None, description="Whether affiliation is currently active", title="Is Current"+ )+ department: Optional[str] = Field(+ None, description="Department or division", title="Department"+ )+ location: Optional[str] = Field(+ None, description="Geographic location of affiliation", title="Location"+ )+ industry: Optional[str] = Field(+ None, description="Industry or sector", title="Industry"+ )+ description: Optional[str] = Field(+ None, description="Description of affiliation", title="Description"+ )+ source: Optional[str] = Field(+ None, description="Source of affiliation information", title="Source"+ )+ confidence: Optional[float] = Field(+ None, description="Confidence score for affiliation", title="Confidence"+ )+ associated_individuals: Optional[List[str]] = Field(+ None,+ description="Individuals with similar affiliations",+ title="Associated Individuals",+ )+ organization_type: Optional[str] = Field(+ None,+ description="Type of organization (company, NGO, government, etc.)",+ title="Organization Type",+ )+ hierarchy_level: Optional[str] = Field(+ None,+ description="Hierarchical level within organization",+ title="Hierarchy Level",+ )diff --git a/flowsint-types/src/flowsint_types/alias.py b/flowsint-types/src/flowsint_types/alias.pyindex aab7f7b2..2e766125 100644--- a/flowsint-types/src/flowsint_types/alias.py+++ b/flowsint-types/src/flowsint_types/alias.py@@ -1,17 +1,45 @@from pydantic import BaseModel, Fieldfrom typing import Optional, List+class Alias(BaseModel):"""Represents an alias or alternative name used by an entity."""+alias: str = Field(..., description="Alias or alternative name", title="Alias")- type: Optional[str] = Field(None, description="Type of alias (nickname, code name, etc.)", title="Alias Type")- context: Optional[str] = Field(None, description="Context where alias is used", title="Context")- first_seen: Optional[str] = Field(None, description="First time alias was observed", title="First Seen")- last_seen: Optional[str] = Field(None, description="Last time alias was observed", title="Last Seen")- confidence: Optional[float] = Field(None, description="Confidence score for alias association", title="Confidence")- source: Optional[str] = Field(None, description="Source of alias information", title="Source")- associated_entities: Optional[List[str]] = Field(None, description="Entities associated with this alias", title="Associated Entities")- description: Optional[str] = Field(None, description="Description or notes about the alias", title="Description")- is_active: Optional[bool] = Field(None, description="Whether alias is currently active", title="Is Active")- language: Optional[str] = Field(None, description="Language of the alias", title="Language")- region: Optional[str] = Field(None, description="Geographic region where alias is used", title="Region")\ No newline at end of file+ type: Optional[str] = Field(+ None,+ description="Type of alias (nickname, code name, etc.)",+ title="Alias Type",+ )+ context: Optional[str] = Field(+ None, description="Context where alias is used", title="Context"+ )+ first_seen: Optional[str] = Field(+ None, description="First time alias was observed", title="First Seen"+ )+ last_seen: Optional[str] = Field(+ None, description="Last time alias was observed", title="Last Seen"+ )+ confidence: Optional[float] = Field(+ None, description="Confidence score for alias association", title="Confidence"+ )+ source: Optional[str] = Field(+ None, description="Source of alias information", title="Source"+ )+ associated_entities: Optional[List[str]] = Field(+ None,+ description="Entities associated with this alias",+ title="Associated Entities",+ )+ description: Optional[str] = Field(+ None, description="Description or notes about the alias", title="Description"+ )+ is_active: Optional[bool] = Field(+ None, description="Whether alias is currently active", title="Is Active"+ )+ language: Optional[str] = Field(+ None, description="Language of the alias", title="Language"+ )+ region: Optional[str] = Field(+ None, description="Geographic region where alias is used", title="Region"+ )diff --git a/flowsint-types/src/flowsint_types/asn.py b/flowsint-types/src/flowsint_types/asn.pyindex 61a29015..df8db510 100644--- a/flowsint-types/src/flowsint_types/asn.py+++ b/flowsint-types/src/flowsint_types/asn.py@@ -2,12 +2,29 @@ from typing import List, Optionalfrom pydantic import BaseModel, Fieldfrom .cidr import CIDR+class ASN(BaseModel):"""Represents an Autonomous System Number with associated network information."""- number: int = Field(..., description="Autonomous System Number (e.g., 15169)", title="ASN Number")- name: Optional[str] = Field(None, description="Name of the organization owning the ASN", title="Organization Name")- country: Optional[str] = Field(None, description="ISO 3166-1 alpha-2 country code", title="Country Code")- description: Optional[str] = Field(None, description="Additional information about the ASN", title="Description")- cidrs: List[CIDR] = Field(default_factory=list, description="List of announced CIDR blocks", title="CIDR Blocks")++ number: int = Field(+ ..., description="Autonomous System Number (e.g., 15169)", title="ASN Number"+ )+ name: Optional[str] = Field(+ None,+ description="Name of the organization owning the ASN",+ title="Organization Name",+ )+ country: Optional[str] = Field(+ None, description="ISO 3166-1 alpha-2 country code", title="Country Code"+ )+ description: Optional[str] = Field(+ None, description="Additional information about the ASN", title="Description"+ )+ cidrs: List[CIDR] = Field(+ default_factory=list,+ description="List of announced CIDR blocks",+ title="CIDR Blocks",+ )+ASN.model_rebuild()diff --git a/flowsint-types/src/flowsint_types/bank_account.py b/flowsint-types/src/flowsint_types/bank_account.pyindex fbee130e..bbafc745 100644--- a/flowsint-types/src/flowsint_types/bank_account.py+++ b/flowsint-types/src/flowsint_types/bank_account.py@@ -1,23 +1,63 @@from pydantic import BaseModel, Fieldfrom typing import Optional, List+class BankAccount(BaseModel):"""Represents a bank account with financial and security information."""- account_number: str = Field(..., description="Bank account number", title="Account Number")++ account_number: str = Field(+ ..., description="Bank account number", title="Account Number"+ )bank_name: Optional[str] = Field(None, description="Bank name", title="Bank Name")- account_type: Optional[str] = Field(None, description="Type of account (checking, savings, etc.)", title="Account Type")- routing_number: Optional[str] = Field(None, description="Bank routing number", title="Routing Number")- iban: Optional[str] = Field(None, description="International Bank Account Number", title="IBAN")- swift_code: Optional[str] = Field(None, description="SWIFT/BIC code", title="SWIFT Code")- country: Optional[str] = Field(None, description="Country where account is held", title="Country")- currency: Optional[str] = Field(None, description="Account currency", title="Currency")- balance: Optional[float] = Field(None, description="Account balance", title="Balance")- account_holder: Optional[str] = Field(None, description="Account holder name", title="Account Holder")- status: Optional[str] = Field(None, description="Account status (active, closed, etc.)", title="Status")- opened_date: Optional[str] = Field(None, description="Account opening date", title="Opened Date")- closed_date: Optional[str] = Field(None, description="Account closing date", title="Closed Date")- is_joint: Optional[bool] = Field(None, description="Whether account is joint", title="Is Joint")- associated_individuals: Optional[List[str]] = Field(None, description="Individuals associated with account", title="Associated Individuals")- source: Optional[str] = Field(None, description="Source of account information", title="Source")- is_compromised: Optional[bool] = Field(None, description="Whether account has been compromised", title="Is Compromised")- breach_source: Optional[str] = Field(None, description="Source of breach if compromised", title="Breach Source")\ No newline at end of file+ account_type: Optional[str] = Field(+ None,+ description="Type of account (checking, savings, etc.)",+ title="Account Type",+ )+ routing_number: Optional[str] = Field(+ None, description="Bank routing number", title="Routing Number"+ )+ iban: Optional[str] = Field(+ None, description="International Bank Account Number", title="IBAN"+ )+ swift_code: Optional[str] = Field(+ None, description="SWIFT/BIC code", title="SWIFT Code"+ )+ country: Optional[str] = Field(+ None, description="Country where account is held", title="Country"+ )+ currency: Optional[str] = Field(+ None, description="Account currency", title="Currency"+ )+ balance: Optional[float] = Field(+ None, description="Account balance", title="Balance"+ )+ account_holder: Optional[str] = Field(+ None, description="Account holder name", title="Account Holder"+ )+ status: Optional[str] = Field(+ None, description="Account status (active, closed, etc.)", title="Status"+ )+ opened_date: Optional[str] = Field(+ None, description="Account opening date", title="Opened Date"+ )+ closed_date: Optional[str] = Field(+ None, description="Account closing date", title="Closed Date"+ )+ is_joint: Optional[bool] = Field(+ None, description="Whether account is joint", title="Is Joint"+ )+ associated_individuals: Optional[List[str]] = Field(+ None,+ description="Individuals associated with account",+ title="Associated Individuals",+ )+ source: Optional[str] = Field(+ None, description="Source of account information", title="Source"+ )+ is_compromised: Optional[bool] = Field(+ None, description="Whether account has been compromised", title="Is Compromised"+ )+ breach_source: Optional[str] = Field(+ None, description="Source of breach if compromised", title="Breach Source"+ )diff --git a/flowsint-types/src/flowsint_types/breach.py b/flowsint-types/src/flowsint_types/breach.pyindex 53ae5730..e3b5b050 100644--- a/flowsint-types/src/flowsint_types/breach.py+++ b/flowsint-types/src/flowsint_types/breach.py@@ -1,24 +1,68 @@from typing import List, Optional, Dictfrom pydantic import BaseModel, Field+class Breach(BaseModel):"""Represents a data breach incident with affected accounts and details."""- name: str = Field(..., description="The name of the breach or service", title="Breach Name")- title: Optional[str] = Field(None, description="Title of the breach", title="Breach Title")- domain: Optional[str] = Field(None, description="Domain of the breached service", title="Domain")- breachdate: Optional[str] = Field(None, description="Date of the breach", title="Breach Date")- addeddate: Optional[str] = Field(None, description="Date breach was added to HIBP", title="Added Date")- modifieddate: Optional[str] = Field(None, description="Date breach was last modified", title="Modified Date")- pwncount: Optional[int] = Field(None, description="Number of accounts affected", title="Affected Accounts")- description: Optional[str] = Field(None, description="Description of the breach", title="Description")- logopath: Optional[str] = Field(None, description="Logo path for the breach", title="Logo Path")- dataclasses: Optional[List[str]] = Field(None, description="Types of data compromised", title="Data Classes")- isverified: Optional[bool] = Field(None, description="Whether the breach is verified", title="Is Verified")- isfabricated: Optional[bool] = Field(None, description="Whether the breach is fabricated", title="Is Fabricated")- issensitive: Optional[bool] = Field(None, description="Whether the breach is sensitive", title="Is Sensitive")- isretired: Optional[bool] = Field(None, description="Whether the breach is retired", title="Is Retired")- isspamlist: Optional[bool] = Field(None, description="Whether the breach is a spam list", title="Is Spam List")- ismalware: Optional[bool] = Field(None, description="Whether the breach is related to malware", title="Is Malware")- isstealerlog: Optional[bool] = Field(None, description="Whether the breach is a stealer log", title="Is Stealer Log")- issubscriptionfree: Optional[bool] = Field(None, description="Whether the breach is subscription free", title="Is Subscription Free")- breach: Optional[Dict] = Field(None, description="Full breach data as returned by the API", title="Full Breach Data")\ No newline at end of file++ name: str = Field(+ ..., description="The name of the breach or service", title="Breach Name"+ )+ title: Optional[str] = Field(+ None, description="Title of the breach", title="Breach Title"+ )+ domain: Optional[str] = Field(+ None, description="Domain of the breached service", title="Domain"+ )+ breachdate: Optional[str] = Field(+ None, description="Date of the breach", title="Breach Date"+ )+ addeddate: Optional[str] = Field(+ None, description="Date breach was added to HIBP", title="Added Date"+ )+ modifieddate: Optional[str] = Field(+ None, description="Date breach was last modified", title="Modified Date"+ )+ pwncount: Optional[int] = Field(+ None, description="Number of accounts affected", title="Affected Accounts"+ )+ description: Optional[str] = Field(+ None, description="Description of the breach", title="Description"+ )+ logopath: Optional[str] = Field(+ None, description="Logo path for the breach", title="Logo Path"+ )+ dataclasses: Optional[List[str]] = Field(+ None, description="Types of data compromised", title="Data Classes"+ )+ isverified: Optional[bool] = Field(+ None, description="Whether the breach is verified", title="Is Verified"+ )+ isfabricated: Optional[bool] = Field(+ None, description="Whether the breach is fabricated", title="Is Fabricated"+ )+ issensitive: Optional[bool] = Field(+ None, description="Whether the breach is sensitive", title="Is Sensitive"+ )+ isretired: Optional[bool] = Field(+ None, description="Whether the breach is retired", title="Is Retired"+ )+ isspamlist: Optional[bool] = Field(+ None, description="Whether the breach is a spam list", title="Is Spam List"+ )+ ismalware: Optional[bool] = Field(+ None, description="Whether the breach is related to malware", title="Is Malware"+ )+ isstealerlog: Optional[bool] = Field(+ None, description="Whether the breach is a stealer log", title="Is Stealer Log"+ )+ issubscriptionfree: Optional[bool] = Field(+ None,+ description="Whether the breach is subscription free",+ title="Is Subscription Free",+ )+ breach: Optional[Dict] = Field(+ None,+ description="Full breach data as returned by the API",+ title="Full Breach Data",+ )diff --git a/flowsint-types/src/flowsint_types/cidr.py b/flowsint-types/src/flowsint_types/cidr.pyindex ec58c351..5cbc24bb 100644--- a/flowsint-types/src/flowsint_types/cidr.py+++ b/flowsint-types/src/flowsint_types/cidr.py@@ -1,5 +1,9 @@from pydantic import BaseModel, IPvAnyNetwork, Field+class CIDR(BaseModel):"""Represents a CIDR (Classless Inter-Domain Routing) network block."""- network: IPvAnyNetwork = Field(..., description="CIDR block (e.g., 8.8.8.0/24)", title="Network Block")++ network: IPvAnyNetwork = Field(+ ..., description="CIDR block (e.g., 8.8.8.0/24)", title="Network Block"+ )diff --git a/flowsint-types/src/flowsint_types/credential.py b/flowsint-types/src/flowsint_types/credential.pyindex 38094ad5..a7c75e9c 100644--- a/flowsint-types/src/flowsint_types/credential.py+++ b/flowsint-types/src/flowsint_types/credential.py@@ -1,22 +1,64 @@from pydantic import BaseModel, Fieldfrom typing import Optional, List+class Credential(BaseModel):"""Represents user credentials with compromise and usage information."""+username: str = Field(..., description="Username or identifier", title="Username")- service: Optional[str] = Field(None, description="Service or platform where credential is used", title="Service")- type: Optional[str] = Field(None, description="Type of credential (password, token, key, etc.)", title="Credential Type")- first_seen: Optional[str] = Field(None, description="First time credential was observed", title="First Seen")- last_seen: Optional[str] = Field(None, description="Last time credential was observed", title="Last Seen")- is_active: Optional[bool] = Field(None, description="Whether credential is currently active", title="Is Active")- is_compromised: Optional[bool] = Field(None, description="Whether credential has been compromised", title="Is Compromised")- breach_source: Optional[str] = Field(None, description="Source of breach if compromised", title="Breach Source")- password_hash: Optional[str] = Field(None, description="Hashed password (if available)", title="Password Hash")- hash_type: Optional[str] = Field(None, description="Type of hash algorithm used", title="Hash Type")- associated_emails: Optional[List[str]] = Field(None, description="Associated email addresses", title="Associated Emails")- associated_phones: Optional[List[str]] = Field(None, description="Associated phone numbers", title="Associated Phones")- ip_addresses: Optional[List[str]] = Field(None, description="IP addresses used with credential", title="IP Addresses")- user_agents: Optional[List[str]] = Field(None, description="User agents used with credential", title="User Agents")- locations: Optional[List[str]] = Field(None, description="Geographic locations where credential was used", title="Locations")- description: Optional[str] = Field(None, description="Additional notes about credential", title="Description")- source: Optional[str] = Field(None, description="Source of credential information", title="Source")\ No newline at end of file+ service: Optional[str] = Field(+ None,+ description="Service or platform where credential is used",+ title="Service",+ )+ type: Optional[str] = Field(+ None,+ description="Type of credential (password, token, key, etc.)",+ title="Credential Type",+ )+ first_seen: Optional[str] = Field(+ None, description="First time credential was observed", title="First Seen"+ )+ last_seen: Optional[str] = Field(+ None, description="Last time credential was observed", title="Last Seen"+ )+ is_active: Optional[bool] = Field(+ None, description="Whether credential is currently active", title="Is Active"+ )+ is_compromised: Optional[bool] = Field(+ None,+ description="Whether credential has been compromised",+ title="Is Compromised",+ )+ breach_source: Optional[str] = Field(+ None, description="Source of breach if compromised", title="Breach Source"+ )+ password_hash: Optional[str] = Field(+ None, description="Hashed password (if available)", title="Password Hash"+ )+ hash_type: Optional[str] = Field(+ None, description="Type of hash algorithm used", title="Hash Type"+ )+ associated_emails: Optional[List[str]] = Field(+ None, description="Associated email addresses", title="Associated Emails"+ )+ associated_phones: Optional[List[str]] = Field(+ None, description="Associated phone numbers", title="Associated Phones"+ )+ ip_addresses: Optional[List[str]] = Field(+ None, description="IP addresses used with credential", title="IP Addresses"+ )+ user_agents: Optional[List[str]] = Field(+ None, description="User agents used with credential", title="User Agents"+ )+ locations: Optional[List[str]] = Field(+ None,+ description="Geographic locations where credential was used",+ title="Locations",+ )+ description: Optional[str] = Field(+ None, description="Additional notes about credential", title="Description"+ )+ source: Optional[str] = Field(+ None, description="Source of credential information", title="Source"+ )diff --git a/flowsint-types/src/flowsint_types/credit_card.py b/flowsint-types/src/flowsint_types/credit_card.pyindex a2ab78d1..f6ef1fc8 100644--- a/flowsint-types/src/flowsint_types/credit_card.py+++ b/flowsint-types/src/flowsint_types/credit_card.py@@ -1,24 +1,58 @@from pydantic import BaseModel, Fieldfrom typing import Optional, List+class CreditCard(BaseModel):"""Represents a credit card with financial details and security status."""+card_number: str = Field(..., description="Credit card number", title="Card Number")- card_type: Optional[str] = Field(None, description="Type of card (Visa, Mastercard, etc.)", title="Card Type")+ card_type: Optional[str] = Field(+ None, description="Type of card (Visa, Mastercard, etc.)", title="Card Type"+ )issuer: Optional[str] = Field(None, description="Card issuer bank", title="Issuer")- expiry_date: Optional[str] = Field(None, description="Card expiry date", title="Expiry Date")+ expiry_date: Optional[str] = Field(+ None, description="Card expiry date", title="Expiry Date"+ )cvv: Optional[str] = Field(None, description="Card verification value", title="CVV")- cardholder_name: Optional[str] = Field(None, description="Cardholder name", title="Cardholder Name")- billing_address: Optional[str] = Field(None, description="Billing address", title="Billing Address")- credit_limit: Optional[float] = Field(None, description="Credit limit", title="Credit Limit")- available_credit: Optional[float] = Field(None, description="Available credit", title="Available Credit")- status: Optional[str] = Field(None, description="Card status (active, suspended, etc.)", title="Status")- issued_date: Optional[str] = Field(None, description="Card issue date", title="Issued Date")- is_virtual: Optional[bool] = Field(None, description="Whether card is virtual", title="Is Virtual")- is_prepaid: Optional[bool] = Field(None, description="Whether card is prepaid", title="Is Prepaid")- is_business: Optional[bool] = Field(None, description="Whether card is business card", title="Is Business")- associated_accounts: Optional[List[str]] = Field(None, description="Associated bank accounts", title="Associated Accounts")- source: Optional[str] = Field(None, description="Source of card information", title="Source")- is_compromised: Optional[bool] = Field(None, description="Whether card has been compromised", title="Is Compromised")- breach_source: Optional[str] = Field(None, description="Source of breach if compromised", title="Breach Source")- last_used: Optional[str] = Field(None, description="Last time card was used", title="Last Used")\ No newline at end of file+ cardholder_name: Optional[str] = Field(+ None, description="Cardholder name", title="Cardholder Name"+ )+ billing_address: Optional[str] = Field(+ None, description="Billing address", title="Billing Address"+ )+ credit_limit: Optional[float] = Field(+ None, description="Credit limit", title="Credit Limit"+ )+ available_credit: Optional[float] = Field(+ None, description="Available credit", title="Available Credit"+ )+ status: Optional[str] = Field(+ None, description="Card status (active, suspended, etc.)", title="Status"+ )+ issued_date: Optional[str] = Field(+ None, description="Card issue date", title="Issued Date"+ )+ is_virtual: Optional[bool] = Field(+ None, description="Whether card is virtual", title="Is Virtual"+ )+ is_prepaid: Optional[bool] = Field(+ None, description="Whether card is prepaid", title="Is Prepaid"+ )+ is_business: Optional[bool] = Field(+ None, description="Whether card is business card", title="Is Business"+ )+ associated_accounts: Optional[List[str]] = Field(+ None, description="Associated bank accounts", title="Associated Accounts"+ )+ source: Optional[str] = Field(+ None, description="Source of card information", title="Source"+ )+ is_compromised: Optional[bool] = Field(+ None, description="Whether card has been compromised", title="Is Compromised"+ )+ breach_source: Optional[str] = Field(+ None, description="Source of breach if compromised", title="Breach Source"+ )+ last_used: Optional[str] = Field(+ None, description="Last time card was used", title="Last Used"+ )diff --git a/flowsint-types/src/flowsint_types/device.py b/flowsint-types/src/flowsint_types/device.pyindex d2d04b99..829c7c15 100644--- a/flowsint-types/src/flowsint_types/device.py+++ b/flowsint-types/src/flowsint_types/device.py@@ -1,25 +1,65 @@from pydantic import BaseModel, Fieldfrom typing import Optional, List+class Device(BaseModel):"""Represents a device with hardware, software, and network information."""- device_id: str = Field(..., description="Unique device identifier", title="Device ID")- type: Optional[str] = Field(None, description="Type of device (mobile, desktop, server, etc.)", title="Device Type")- manufacturer: Optional[str] = Field(None, description="Device manufacturer", title="Manufacturer")++ device_id: str = Field(+ ..., description="Unique device identifier", title="Device ID"+ )+ type: Optional[str] = Field(+ None,+ description="Type of device (mobile, desktop, server, etc.)",+ title="Device Type",+ )+ manufacturer: Optional[str] = Field(+ None, description="Device manufacturer", title="Manufacturer"+ )model: Optional[str] = Field(None, description="Device model", title="Model")- os: Optional[str] = Field(None, description="Operating system", title="Operating System")- os_version: Optional[str] = Field(None, description="Operating system version", title="OS Version")+ os: Optional[str] = Field(+ None, description="Operating system", title="Operating System"+ )+ os_version: Optional[str] = Field(+ None, description="Operating system version", title="OS Version"+ )browser: Optional[str] = Field(None, description="Browser used", title="Browser")- browser_version: Optional[str] = Field(None, description="Browser version", title="Browser Version")- screen_resolution: Optional[str] = Field(None, description="Screen resolution", title="Screen Resolution")- user_agent: Optional[str] = Field(None, description="User agent string", title="User Agent")- mac_address: Optional[str] = Field(None, description="MAC address", title="MAC Address")- ip_addresses: Optional[List[str]] = Field(None, description="IP addresses associated with device", title="IP Addresses")- first_seen: Optional[str] = Field(None, description="First time device was observed", title="First Seen")- last_seen: Optional[str] = Field(None, description="Last time device was observed", title="Last Seen")- location: Optional[str] = Field(None, description="Geographic location", title="Location")- is_mobile: Optional[bool] = Field(None, description="Whether device is mobile", title="Is Mobile")- is_tablet: Optional[bool] = Field(None, description="Whether device is a tablet", title="Is Tablet")- is_desktop: Optional[bool] = Field(None, description="Whether device is a desktop", title="Is Desktop")- associated_users: Optional[List[str]] = Field(None, description="Users associated with device", title="Associated Users")- source: Optional[str] = Field(None, description="Source of device information", title="Source")\ No newline at end of file+ browser_version: Optional[str] = Field(+ None, description="Browser version", title="Browser Version"+ )+ screen_resolution: Optional[str] = Field(+ None, description="Screen resolution", title="Screen Resolution"+ )+ user_agent: Optional[str] = Field(+ None, description="User agent string", title="User Agent"+ )+ mac_address: Optional[str] = Field(+ None, description="MAC address", title="MAC Address"+ )+ ip_addresses: Optional[List[str]] = Field(+ None, description="IP addresses associated with device", title="IP Addresses"+ )+ first_seen: Optional[str] = Field(+ None, description="First time device was observed", title="First Seen"+ )+ last_seen: Optional[str] = Field(+ None, description="Last time device was observed", title="Last Seen"+ )+ location: Optional[str] = Field(+ None, description="Geographic location", title="Location"+ )+ is_mobile: Optional[bool] = Field(+ None, description="Whether device is mobile", title="Is Mobile"+ )+ is_tablet: Optional[bool] = Field(+ None, description="Whether device is a tablet", title="Is Tablet"+ )+ is_desktop: Optional[bool] = Field(+ None, description="Whether device is a desktop", title="Is Desktop"+ )+ associated_users: Optional[List[str]] = Field(+ None, description="Users associated with device", title="Associated Users"+ )+ source: Optional[str] = Field(+ None, description="Source of device information", title="Source"+ )diff --git a/flowsint-types/src/flowsint_types/dns_record.py b/flowsint-types/src/flowsint_types/dns_record.pyindex bb5fb1f6..aa20a02c 100644--- a/flowsint-types/src/flowsint_types/dns_record.py+++ b/flowsint-types/src/flowsint_types/dns_record.py@@ -1,20 +1,50 @@from pydantic import BaseModel, Fieldfrom typing import Optional, List+class DNSRecord(BaseModel):"""Represents a DNS record with type, value, and security information."""- record_type: str = Field(..., description="Type of DNS record (A, AAAA, CNAME, MX, etc.)", title="Record Type")++ record_type: str = Field(+ ...,+ description="Type of DNS record (A, AAAA, CNAME, MX, etc.)",+ title="Record Type",+ )name: str = Field(..., description="Domain name", title="Domain Name")value: str = Field(..., description="Record value", title="Record Value")ttl: Optional[int] = Field(None, description="Time to live in seconds", title="TTL")- priority: Optional[int] = Field(None, description="Priority for MX records", title="Priority")- first_seen: Optional[str] = Field(None, description="First time record was observed", title="First Seen")- last_seen: Optional[str] = Field(None, description="Last time record was observed", title="Last Seen")- is_active: Optional[bool] = Field(None, description="Whether record is currently active", title="Is Active")- nameserver: Optional[str] = Field(None, description="Nameserver that provided the record", title="Nameserver")- source: Optional[str] = Field(None, description="Source of DNS information", title="Source")- associated_domains: Optional[List[str]] = Field(None, description="Related domain names", title="Associated Domains")- description: Optional[str] = Field(None, description="Additional notes about the record", title="Description")- is_suspicious: Optional[bool] = Field(None, description="Whether record is suspicious", title="Is Suspicious")- malware_family: Optional[str] = Field(None, description="Malware family if record is malicious", title="Malware Family")- threat_level: Optional[str] = Field(None, description="Threat level assessment", title="Threat Level")\ No newline at end of file+ priority: Optional[int] = Field(+ None, description="Priority for MX records", title="Priority"+ )+ first_seen: Optional[str] = Field(+ None, description="First time record was observed", title="First Seen"+ )+ last_seen: Optional[str] = Field(+ None, description="Last time record was observed", title="Last Seen"+ )+ is_active: Optional[bool] = Field(+ None, description="Whether record is currently active", title="Is Active"+ )+ nameserver: Optional[str] = Field(+ None, description="Nameserver that provided the record", title="Nameserver"+ )+ source: Optional[str] = Field(+ None, description="Source of DNS information", title="Source"+ )+ associated_domains: Optional[List[str]] = Field(+ None, description="Related domain names", title="Associated Domains"+ )+ description: Optional[str] = Field(+ None, description="Additional notes about the record", title="Description"+ )+ is_suspicious: Optional[bool] = Field(+ None, description="Whether record is suspicious", title="Is Suspicious"+ )+ malware_family: Optional[str] = Field(+ None,+ description="Malware family if record is malicious",+ title="Malware Family",+ )+ threat_level: Optional[str] = Field(+ None, description="Threat level assessment", title="Threat Level"+ )diff --git a/flowsint-types/src/flowsint_types/document.py b/flowsint-types/src/flowsint_types/document.pyindex 7c5c8bd6..f8e5f0d6 100644--- a/flowsint-types/src/flowsint_types/document.py+++ b/flowsint-types/src/flowsint_types/document.py@@ -1,24 +1,58 @@from pydantic import BaseModel, Fieldfrom typing import Optional, List+class Document(BaseModel):"""Represents a document with metadata, security, and content information."""+title: str = Field(..., description="Document title", title="Title")- type: Optional[str] = Field(None, description="Type of document (PDF, DOC, etc.)", title="Document Type")- file_size: Optional[int] = Field(None, description="File size in bytes", title="File Size")- created_date: Optional[str] = Field(None, description="Document creation date", title="Created Date")- modified_date: Optional[str] = Field(None, description="Document last modified date", title="Modified Date")+ type: Optional[str] = Field(+ None, description="Type of document (PDF, DOC, etc.)", title="Document Type"+ )+ file_size: Optional[int] = Field(+ None, description="File size in bytes", title="File Size"+ )+ created_date: Optional[str] = Field(+ None, description="Document creation date", title="Created Date"+ )+ modified_date: Optional[str] = Field(+ None, description="Document last modified date", title="Modified Date"+ )author: Optional[str] = Field(None, description="Document author", title="Author")- subject: Optional[str] = Field(None, description="Document subject", title="Subject")- keywords: Optional[List[str]] = Field(None, description="Document keywords", title="Keywords")- language: Optional[str] = Field(None, description="Document language", title="Language")- page_count: Optional[int] = Field(None, description="Number of pages", title="Page Count")+ subject: Optional[str] = Field(+ None, description="Document subject", title="Subject"+ )+ keywords: Optional[List[str]] = Field(+ None, description="Document keywords", title="Keywords"+ )+ language: Optional[str] = Field(+ None, description="Document language", title="Language"+ )+ page_count: Optional[int] = Field(+ None, description="Number of pages", title="Page Count"+ )hash_md5: Optional[str] = Field(None, description="MD5 hash", title="MD5 Hash")hash_sha1: Optional[str] = Field(None, description="SHA1 hash", title="SHA1 Hash")- hash_sha256: Optional[str] = Field(None, description="SHA256 hash", title="SHA256 Hash")- source_url: Optional[str] = Field(None, description="Source URL if downloaded", title="Source URL")- location: Optional[str] = Field(None, description="File location or path", title="Location")- is_encrypted: Optional[bool] = Field(None, description="Whether document is encrypted", title="Is Encrypted")- is_password_protected: Optional[bool] = Field(None, description="Whether document is password protected", title="Is Password Protected")- description: Optional[str] = Field(None, description="Document description", title="Description")- source: Optional[str] = Field(None, description="Source of document information", title="Source")\ No newline at end of file+ hash_sha256: Optional[str] = Field(+ None, description="SHA256 hash", title="SHA256 Hash"+ )+ source_url: Optional[str] = Field(+ None, description="Source URL if downloaded", title="Source URL"+ )+ location: Optional[str] = Field(+ None, description="File location or path", title="Location"+ )+ is_encrypted: Optional[bool] = Field(+ None, description="Whether document is encrypted", title="Is Encrypted"+ )+ is_password_protected: Optional[bool] = Field(+ None,+ description="Whether document is password protected",+ title="Is Password Protected",+ )+ description: Optional[str] = Field(+ None, description="Document description", title="Description"+ )+ source: Optional[str] = Field(+ None, description="Source of document information", title="Source"+ )diff --git a/flowsint-types/src/flowsint_types/domain.py b/flowsint-types/src/flowsint_types/domain.pyindex eaba0822..62231616 100644--- a/flowsint-types/src/flowsint_types/domain.py+++ b/flowsint-types/src/flowsint_types/domain.py@@ -1,10 +1,14 @@from typing import Optionalfrom pydantic import BaseModel, Field+class Domain(BaseModel):"""Represents a domain name and its properties."""+domain: str = Field(..., description="Domain name", title="Domain Name")- root: Optional[bool]= Field(True, description="Is root or not", title="Is Root Domain")--Domain.model_rebuild()+ root: Optional[bool] = Field(+ True, description="Is root or not", title="Is Root Domain"+ )++Domain.model_rebuild()diff --git a/flowsint-types/src/flowsint_types/email.py b/flowsint-types/src/flowsint_types/email.pyindex 9a4818e4..529d9897 100644--- a/flowsint-types/src/flowsint_types/email.py+++ b/flowsint-types/src/flowsint_types/email.py@@ -1,5 +1,7 @@from pydantic import BaseModel, Field+class Email(BaseModel):"""Represents an email address."""- email: str = Field(..., description="Email address", title="Email Address")\ No newline at end of file++ email: str = Field(..., description="Email address", title="Email Address")diff --git a/flowsint-types/src/flowsint_types/file.py b/flowsint-types/src/flowsint_types/file.pyindex 046264d2..608ec3a3 100644--- a/flowsint-types/src/flowsint_types/file.py+++ b/flowsint-types/src/flowsint_types/file.py@@ -1,27 +1,63 @@from pydantic import BaseModel, Fieldfrom typing import Optional, List+class File(BaseModel):"""Represents a file with metadata, type information, and security assessment."""+filename: str = Field(..., description="File name", title="Filename")- file_type: Optional[str] = Field(None, description="File type or extension", title="File Type")- file_size: Optional[int] = Field(None, description="File size in bytes", title="File Size")- created_date: Optional[str] = Field(None, description="File creation date", title="Created Date")- modified_date: Optional[str] = Field(None, description="File last modified date", title="Modified Date")- accessed_date: Optional[str] = Field(None, description="File last accessed date", title="Accessed Date")+ file_type: Optional[str] = Field(+ None, description="File type or extension", title="File Type"+ )+ file_size: Optional[int] = Field(+ None, description="File size in bytes", title="File Size"+ )+ created_date: Optional[str] = Field(+ None, description="File creation date", title="Created Date"+ )+ modified_date: Optional[str] = Field(+ None, description="File last modified date", title="Modified Date"+ )+ accessed_date: Optional[str] = Field(+ None, description="File last accessed date", title="Accessed Date"+ )path: Optional[str] = Field(None, description="File path", title="Path")hash_md5: Optional[str] = Field(None, description="MD5 hash", title="MD5 Hash")hash_sha1: Optional[str] = Field(None, description="SHA1 hash", title="SHA1 Hash")- hash_sha256: Optional[str] = Field(None, description="SHA256 hash", title="SHA256 Hash")- is_executable: Optional[bool] = Field(None, description="Whether file is executable", title="Is Executable")- is_archive: Optional[bool] = Field(None, description="Whether file is an archive", title="Is Archive")- is_image: Optional[bool] = Field(None, description="Whether file is an image", title="Is Image")- is_video: Optional[bool] = Field(None, description="Whether file is a video", title="Is Video")- is_audio: Optional[bool] = Field(None, description="Whether file is an audio file", title="Is Audio")- is_text: Optional[bool] = Field(None, description="Whether file is a text file", title="Is Text")+ hash_sha256: Optional[str] = Field(+ None, description="SHA256 hash", title="SHA256 Hash"+ )+ is_executable: Optional[bool] = Field(+ None, description="Whether file is executable", title="Is Executable"+ )+ is_archive: Optional[bool] = Field(+ None, description="Whether file is an archive", title="Is Archive"+ )+ is_image: Optional[bool] = Field(+ None, description="Whether file is an image", title="Is Image"+ )+ is_video: Optional[bool] = Field(+ None, description="Whether file is a video", title="Is Video"+ )+ is_audio: Optional[bool] = Field(+ None, description="Whether file is an audio file", title="Is Audio"+ )+ is_text: Optional[bool] = Field(+ None, description="Whether file is a text file", title="Is Text"+ )mime_type: Optional[str] = Field(None, description="MIME type", title="MIME Type")- description: Optional[str] = Field(None, description="File description", title="Description")- source: Optional[str] = Field(None, description="Source of file information", title="Source")- is_malicious: Optional[bool] = Field(None, description="Whether file is malicious", title="Is Malicious")- malware_family: Optional[str] = Field(None, description="Malware family if malicious", title="Malware Family")- threat_level: Optional[str] = Field(None, description="Threat level assessment", title="Threat Level")\ No newline at end of file+ description: Optional[str] = Field(+ None, description="File description", title="Description"+ )+ source: Optional[str] = Field(+ None, description="Source of file information", title="Source"+ )+ is_malicious: Optional[bool] = Field(+ None, description="Whether file is malicious", title="Is Malicious"+ )+ malware_family: Optional[str] = Field(+ None, description="Malware family if malicious", title="Malware Family"+ )+ threat_level: Optional[str] = Field(+ None, description="Threat level assessment", title="Threat Level"+ )diff --git a/flowsint-types/src/flowsint_types/gravatar.py b/flowsint-types/src/flowsint_types/gravatar.pyindex eddc149a..cc1cb059 100644--- a/flowsint-types/src/flowsint_types/gravatar.py+++ b/flowsint-types/src/flowsint_types/gravatar.py@@ -1,18 +1,44 @@from pydantic import BaseModel, Field, HttpUrlfrom typing import Optional, List+class Gravatar(BaseModel):"""Represents a Gravatar profile with image and user information."""+src: HttpUrl = Field(..., description="Gravatar image URL", title="Image URL")hash: str = Field(..., description="Gravatar hash", title="Hash")- size: Optional[int] = Field(None, description="Image size in pixels", title="Image Size")- rating: Optional[str] = Field(None, description="Content rating (g, pg, r, x)", title="Content Rating")- default_image: Optional[str] = Field(None, description="Default image type when no gravatar exists", title="Default Image")- force_default: Optional[bool] = Field(None, description="Whether to force default image", title="Force Default")- exists: Optional[bool] = Field(None, description="Whether the gravatar exists", title="Exists")- profile_url: Optional[HttpUrl] = Field(None, description="URL to the Gravatar profile page", title="Profile URL")- display_name: Optional[str] = Field(None, description="Display name from Gravatar profile", title="Display Name")- location: Optional[str] = Field(None, description="Location from Gravatar profile", title="Location")- about_me: Optional[str] = Field(None, description="Bio/about me text from Gravatar profile", title="About Me")- thumbnail_url: Optional[HttpUrl] = Field(None, description="Smaller version of the image", title="Thumbnail URL")- large_url: Optional[HttpUrl] = Field(None, description="Larger version of the image", title="Large Image URL")+ size: Optional[int] = Field(+ None, description="Image size in pixels", title="Image Size"+ )+ rating: Optional[str] = Field(+ None, description="Content rating (g, pg, r, x)", title="Content Rating"+ )+ default_image: Optional[str] = Field(+ None,+ description="Default image type when no gravatar exists",+ title="Default Image",+ )+ force_default: Optional[bool] = Field(+ None, description="Whether to force default image", title="Force Default"+ )+ exists: Optional[bool] = Field(+ None, description="Whether the gravatar exists", title="Exists"+ )+ profile_url: Optional[HttpUrl] = Field(+ None, description="URL to the Gravatar profile page", title="Profile URL"+ )+ display_name: Optional[str] = Field(+ None, description="Display name from Gravatar profile", title="Display Name"+ )+ location: Optional[str] = Field(+ None, description="Location from Gravatar profile", title="Location"+ )+ about_me: Optional[str] = Field(+ None, description="Bio/about me text from Gravatar profile", title="About Me"+ )+ thumbnail_url: Optional[HttpUrl] = Field(+ None, description="Smaller version of the image", title="Thumbnail URL"+ )+ large_url: Optional[HttpUrl] = Field(+ None, description="Larger version of the image", title="Large Image URL"+ )diff --git a/flowsint-types/src/flowsint_types/individual.py b/flowsint-types/src/flowsint_types/individual.pyindex a7693d32..469cfe48 100644--- a/flowsint-types/src/flowsint_types/individual.py+++ b/flowsint-types/src/flowsint_types/individual.py@@ -2,118 +2,307 @@ from pydantic import BaseModel, Fieldfrom typing import Optional, Literal, Listfrom .address import PhysicalAddress+class Individual(BaseModel):"""Represents an individual person with comprehensive personal information."""+# Basic Information- first_name: str = Field(..., description="First name of the individual", title="First Name")- last_name: str = Field(..., description="Last name of the individual", title="Last Name")- full_name: str = Field(..., description="Full name of the individual", title="Full Name")- middle_name: Optional[str] = Field(None, description="Middle name or initial", title="Middle Name")- maiden_name: Optional[str] = Field(None, description="Maiden name (if applicable)", title="Maiden Name")- aliases: Optional[List[str]] = Field(None, description="Known aliases or nicknames", title="Aliases")-+ first_name: str = Field(+ ..., description="First name of the individual", title="First Name"+ )+ last_name: str = Field(+ ..., description="Last name of the individual", title="Last Name"+ )+ full_name: str = Field(+ ..., description="Full name of the individual", title="Full Name"+ )+ middle_name: Optional[str] = Field(+ None, description="Middle name or initial", title="Middle Name"+ )+ maiden_name: Optional[str] = Field(+ None, description="Maiden name (if applicable)", title="Maiden Name"+ )+ aliases: Optional[List[str]] = Field(+ None, description="Known aliases or nicknames", title="Aliases"+ )+# Birth Information- birth_date: Optional[str] = Field(None, description="Date of birth", title="Date of Birth")- birth_place: Optional[PhysicalAddress] = Field(None, description="Place of birth", title="Birth Place")+ birth_date: Optional[str] = Field(+ None, description="Date of birth", title="Date of Birth"+ )+ birth_place: Optional[PhysicalAddress] = Field(+ None, description="Place of birth", title="Birth Place"+ )age: Optional[int] = Field(None, description="Current age", title="Age")-+# Physical Characteristics- gender: Optional[Literal["male", "female", "other"]] = Field(None, description="Gender of the individual", title="Gender")- height: Optional[str] = Field(None, description="Height (e.g., '5ft 10in' or '178cm')", title="Height")- weight: Optional[str] = Field(None, description="Weight (e.g., '170lbs' or '77kg')", title="Weight")- eye_color: Optional[Literal["brown", "blue", "green", "hazel", "gray", "amber", "other"]] = Field(None, description="Eye color", title="Eye Color")- hair_color: Optional[Literal["black", "brown", "blonde", "red", "gray", "white", "bald", "other"]] = Field(None, description="Hair color", title="Hair Color")- skin_tone: Optional[str] = Field(None, description="Skin tone description", title="Skin Tone")- ethnicity: Optional[str] = Field(None, description="Ethnicity or ethnic background", title="Ethnicity")- blood_type: Optional[Literal["A+", "A-", "B+", "B-", "AB+", "AB-", "O+", "O-"]] = Field(None, description="Blood type", title="Blood Type")- distinguishing_marks: Optional[List[str]] = Field(None, description="Distinguishing marks, scars, tattoos, etc.", title="Distinguishing Marks")- disabilities: Optional[List[str]] = Field(None, description="Known disabilities or medical conditions", title="Disabilities")-+ gender: Optional[Literal["male", "female", "other"]] = Field(+ None, description="Gender of the individual", title="Gender"+ )+ height: Optional[str] = Field(+ None, description="Height (e.g., '5ft 10in' or '178cm')", title="Height"+ )+ weight: Optional[str] = Field(+ None, description="Weight (e.g., '170lbs' or '77kg')", title="Weight"+ )+ eye_color: Optional[+ Literal["brown", "blue", "green", "hazel", "gray", "amber", "other"]+ ] = Field(None, description="Eye color", title="Eye Color")+ hair_color: Optional[+ Literal["black", "brown", "blonde", "red", "gray", "white", "bald", "other"]+ ] = Field(None, description="Hair color", title="Hair Color")+ skin_tone: Optional[str] = Field(+ None, description="Skin tone description", title="Skin Tone"+ )+ ethnicity: Optional[str] = Field(+ None, description="Ethnicity or ethnic background", title="Ethnicity"+ )+ blood_type: Optional[Literal["A+", "A-", "B+", "B-", "AB+", "AB-", "O+", "O-"]] = (+ Field(None, description="Blood type", title="Blood Type")+ )+ distinguishing_marks: Optional[List[str]] = Field(+ None,+ description="Distinguishing marks, scars, tattoos, etc.",+ title="Distinguishing Marks",+ )+ disabilities: Optional[List[str]] = Field(+ None,+ description="Known disabilities or medical conditions",+ title="Disabilities",+ )+# Health and Lifestyle- is_smoker: Optional[bool] = Field(None, description="Whether the individual smokes", title="Is Smoker")- is_deceased: Optional[bool] = Field(None, description="Whether the individual is deceased", title="Is Deceased")- death_date: Optional[str] = Field(None, description="Date of death (if applicable)", title="Death Date")- death_place: Optional[PhysicalAddress] = Field(None, description="Place of death (if applicable)", title="Death Place")- cause_of_death: Optional[str] = Field(None, description="Cause of death (if applicable)", title="Cause of Death")-+ is_smoker: Optional[bool] = Field(+ None, description="Whether the individual smokes", title="Is Smoker"+ )+ is_deceased: Optional[bool] = Field(+ None, description="Whether the individual is deceased", title="Is Deceased"+ )+ death_date: Optional[str] = Field(+ None, description="Date of death (if applicable)", title="Death Date"+ )+ death_place: Optional[PhysicalAddress] = Field(+ None, description="Place of death (if applicable)", title="Death Place"+ )+ cause_of_death: Optional[str] = Field(+ None, description="Cause of death (if applicable)", title="Cause of Death"+ )+# Contact Information- phone_numbers: Optional[List[str]] = Field(None, description="Known phone numbers", title="Phone Numbers")- email_addresses: Optional[List[str]] = Field(None, description="Known email addresses", title="Email Addresses")- social_media_profiles: Optional[List[str]] = Field(None, description="Social media profile URLs or usernames", title="Social Media Profiles")-+ phone_numbers: Optional[List[str]] = Field(+ None, description="Known phone numbers", title="Phone Numbers"+ )+ email_addresses: Optional[List[str]] = Field(+ None, description="Known email addresses", title="Email Addresses"+ )+ social_media_profiles: Optional[List[str]] = Field(+ None,+ description="Social media profile URLs or usernames",+ title="Social Media Profiles",+ )+# Location Information- current_address: Optional[PhysicalAddress] = Field(None, description="Current residential address", title="Current Address")- previous_addresses: Optional[List[PhysicalAddress]] = Field(None, description="Previous known addresses", title="Previous Addresses")- nationality: Optional[str] = Field(None, description="Nationality or country of citizenship", title="Nationality")- citizenship: Optional[List[str]] = Field(None, description="Countries of citizenship (dual citizenship possible)", title="Citizenship")- place_of_residence: Optional[str] = Field(None, description="Current place of residence (city/state/country)", title="Place of Residence")-+ current_address: Optional[PhysicalAddress] = Field(+ None, description="Current residential address", title="Current Address"+ )+ previous_addresses: Optional[List[PhysicalAddress]] = Field(+ None, description="Previous known addresses", title="Previous Addresses"+ )+ nationality: Optional[str] = Field(+ None, description="Nationality or country of citizenship", title="Nationality"+ )+ citizenship: Optional[List[str]] = Field(+ None,+ description="Countries of citizenship (dual citizenship possible)",+ title="Citizenship",+ )+ place_of_residence: Optional[str] = Field(+ None,+ description="Current place of residence (city/state/country)",+ title="Place of Residence",+ )+# Identification Documents- passport_number: Optional[str] = Field(None, description="Passport number", title="Passport Number")- drivers_license: Optional[str] = Field(None, description="Driver's license number", title="Driver's License")- social_security_number: Optional[str] = Field(None, description="Social Security Number (SSN)", title="Social Security Number")- national_id: Optional[str] = Field(None, description="National ID number", title="National ID")- voter_id: Optional[str] = Field(None, description="Voter ID number", title="Voter ID")- tax_id: Optional[str] = Field(None, description="Tax identification number", title="Tax ID")-+ passport_number: Optional[str] = Field(+ None, description="Passport number", title="Passport Number"+ )+ drivers_license: Optional[str] = Field(+ None, description="Driver's license number", title="Driver's License"+ )+ social_security_number: Optional[str] = Field(+ None, description="Social Security Number (SSN)", title="Social Security Number"+ )+ national_id: Optional[str] = Field(+ None, description="National ID number", title="National ID"+ )+ voter_id: Optional[str] = Field(+ None, description="Voter ID number", title="Voter ID"+ )+ tax_id: Optional[str] = Field(+ None, description="Tax identification number", title="Tax ID"+ )+# Personal Relationships- marital_status: Optional[Literal["single", "married", "divorced", "widowed", "separated", "domestic_partnership"]] = Field(None, description="Marital status", title="Marital Status")- spouse_name: Optional[str] = Field(None, description="Name of spouse or partner", title="Spouse Name")- children_count: Optional[int] = Field(None, description="Number of children", title="Children Count")- children_names: Optional[List[str]] = Field(None, description="Names of children", title="Children Names")- parents_names: Optional[List[str]] = Field(None, description="Names of parents", title="Parents Names")- emergency_contact: Optional[str] = Field(None, description="Emergency contact information", title="Emergency Contact")-+ marital_status: Optional[+ Literal[+ "single",+ "married",+ "divorced",+ "widowed",+ "separated",+ "domestic_partnership",+ ]+ ] = Field(None, description="Marital status", title="Marital Status")+ spouse_name: Optional[str] = Field(+ None, description="Name of spouse or partner", title="Spouse Name"+ )+ children_count: Optional[int] = Field(+ None, description="Number of children", title="Children Count"+ )+ children_names: Optional[List[str]] = Field(+ None, description="Names of children", title="Children Names"+ )+ parents_names: Optional[List[str]] = Field(+ None, description="Names of parents", title="Parents Names"+ )+ emergency_contact: Optional[str] = Field(+ None, description="Emergency contact information", title="Emergency Contact"+ )+# Professional Information- occupation: Optional[str] = Field(None, description="Current occupation or job title", title="Occupation")- employer: Optional[str] = Field(None, description="Current employer", title="Employer")- employer_address: Optional[PhysicalAddress] = Field(None, description="Employer's address", title="Employer Address")- job_title: Optional[str] = Field(None, description="Current job title or position", title="Job Title")- annual_income: Optional[float] = Field(None, description="Annual income", title="Annual Income")- employment_history: Optional[List[str]] = Field(None, description="Previous employment history", title="Employment History")-+ occupation: Optional[str] = Field(+ None, description="Current occupation or job title", title="Occupation"+ )+ employer: Optional[str] = Field(+ None, description="Current employer", title="Employer"+ )+ employer_address: Optional[PhysicalAddress] = Field(+ None, description="Employer's address", title="Employer Address"+ )+ job_title: Optional[str] = Field(+ None, description="Current job title or position", title="Job Title"+ )+ annual_income: Optional[float] = Field(+ None, description="Annual income", title="Annual Income"+ )+ employment_history: Optional[List[str]] = Field(+ None, description="Previous employment history", title="Employment History"+ )+# Education- education_level: Optional[Literal["elementary", "high_school", "bachelor", "master", "doctorate", "other"]] = Field(None, description="Highest education level achieved", title="Education Level")- schools_attended: Optional[List[str]] = Field(None, description="Schools or universities attended", title="Schools Attended")- degrees: Optional[List[str]] = Field(None, description="Degrees or certifications obtained", title="Degrees")-+ education_level: Optional[+ Literal["elementary", "high_school", "bachelor", "master", "doctorate", "other"]+ ] = Field(+ None, description="Highest education level achieved", title="Education Level"+ )+ schools_attended: Optional[List[str]] = Field(+ None, description="Schools or universities attended", title="Schools Attended"+ )+ degrees: Optional[List[str]] = Field(+ None, description="Degrees or certifications obtained", title="Degrees"+ )+# Personal Characteristics- languages_spoken: Optional[List[str]] = Field(None, description="Languages spoken", title="Languages Spoken")- religion: Optional[str] = Field(None, description="Religious affiliation", title="Religion")- political_affiliation: Optional[str] = Field(None, description="Political party affiliation", title="Political Affiliation")- hobbies: Optional[List[str]] = Field(None, description="Known hobbies or interests", title="Hobbies")- personality_traits: Optional[List[str]] = Field(None, description="Known personality traits", title="Personality Traits")-+ languages_spoken: Optional[List[str]] = Field(+ None, description="Languages spoken", title="Languages Spoken"+ )+ religion: Optional[str] = Field(+ None, description="Religious affiliation", title="Religion"+ )+ political_affiliation: Optional[str] = Field(+ None, description="Political party affiliation", title="Political Affiliation"+ )+ hobbies: Optional[List[str]] = Field(+ None, description="Known hobbies or interests", title="Hobbies"+ )+ personality_traits: Optional[List[str]] = Field(+ None, description="Known personality traits", title="Personality Traits"+ )+# Biometric Information- fingerprints: Optional[str] = Field(None, description="Fingerprint data or hash", title="Fingerprints")- dna_profile: Optional[str] = Field(None, description="DNA profile or hash", title="DNA Profile")- facial_recognition_id: Optional[str] = Field(None, description="Facial recognition system ID", title="Facial Recognition ID")- iris_scan: Optional[str] = Field(None, description="Iris scan data or hash", title="Iris Scan")- voice_print: Optional[str] = Field(None, description="Voice print data or hash", title="Voice Print")-+ fingerprints: Optional[str] = Field(+ None, description="Fingerprint data or hash", title="Fingerprints"+ )+ dna_profile: Optional[str] = Field(+ None, description="DNA profile or hash", title="DNA Profile"+ )+ facial_recognition_id: Optional[str] = Field(+ None, description="Facial recognition system ID", title="Facial Recognition ID"+ )+ iris_scan: Optional[str] = Field(+ None, description="Iris scan data or hash", title="Iris Scan"+ )+ voice_print: Optional[str] = Field(+ None, description="Voice print data or hash", title="Voice Print"+ )+# Financial Information- bank_accounts: Optional[List[str]] = Field(None, description="Known bank account numbers", title="Bank Accounts")- credit_score: Optional[int] = Field(None, description="Credit score", title="Credit Score")- net_worth: Optional[float] = Field(None, description="Estimated net worth", title="Net Worth")- assets: Optional[List[str]] = Field(None, description="Known assets (property, vehicles, etc.)", title="Assets")-+ bank_accounts: Optional[List[str]] = Field(+ None, description="Known bank account numbers", title="Bank Accounts"+ )+ credit_score: Optional[int] = Field(+ None, description="Credit score", title="Credit Score"+ )+ net_worth: Optional[float] = Field(+ None, description="Estimated net worth", title="Net Worth"+ )+ assets: Optional[List[str]] = Field(+ None, description="Known assets (property, vehicles, etc.)", title="Assets"+ )+# Legal Information- criminal_record: Optional[bool] = Field(None, description="Whether individual has a criminal record", title="Criminal Record")- arrest_history: Optional[List[str]] = Field(None, description="Known arrests or charges", title="Arrest History")- convictions: Optional[List[str]] = Field(None, description="Criminal convictions", title="Convictions")- civil_cases: Optional[List[str]] = Field(None, description="Civil court cases", title="Civil Cases")- security_clearance: Optional[str] = Field(None, description="Security clearance level", title="Security Clearance")-+ criminal_record: Optional[bool] = Field(+ None,+ description="Whether individual has a criminal record",+ title="Criminal Record",+ )+ arrest_history: Optional[List[str]] = Field(+ None, description="Known arrests or charges", title="Arrest History"+ )+ convictions: Optional[List[str]] = Field(+ None, description="Criminal convictions", title="Convictions"+ )+ civil_cases: Optional[List[str]] = Field(+ None, description="Civil court cases", title="Civil Cases"+ )+ security_clearance: Optional[str] = Field(+ None, description="Security clearance level", title="Security Clearance"+ )+# Digital Footprint- ip_addresses: Optional[List[str]] = Field(None, description="Known IP addresses", title="IP Addresses")- usernames: Optional[List[str]] = Field(None, description="Known online usernames", title="Usernames")- device_ids: Optional[List[str]] = Field(None, description="Known device identifiers", title="Device IDs")-+ ip_addresses: Optional[List[str]] = Field(+ None, description="Known IP addresses", title="IP Addresses"+ )+ usernames: Optional[List[str]] = Field(+ None, description="Known online usernames", title="Usernames"+ )+ device_ids: Optional[List[str]] = Field(+ None, description="Known device identifiers", title="Device IDs"+ )+# Metadata- first_seen: Optional[str] = Field(None, description="First time individual was observed in system", title="First Seen")- last_seen: Optional[str] = Field(None, description="Last time individual was observed in system", title="Last Seen")- source: Optional[str] = Field(None, description="Source of individual information", title="Source")- confidence: Optional[float] = Field(None, description="Confidence score for individual data", title="Confidence")- notes: Optional[str] = Field(None, description="Additional notes or observations", title="Notes")- src: Optional[str] = Field(None, description="URL to profile picture", title="Profile Picture URL")- last_updated: Optional[str] = Field(None, description="Last update timestamp", title="Last Updated")\ No newline at end of file+ first_seen: Optional[str] = Field(+ None,+ description="First time individual was observed in system",+ title="First Seen",+ )+ last_seen: Optional[str] = Field(+ None,+ description="Last time individual was observed in system",+ title="Last Seen",+ )+ source: Optional[str] = Field(+ None, description="Source of individual information", title="Source"+ )+ confidence: Optional[float] = Field(+ None, description="Confidence score for individual data", title="Confidence"+ )+ notes: Optional[str] = Field(+ None, description="Additional notes or observations", title="Notes"+ )+ src: Optional[str] = Field(+ None, description="URL to profile picture", title="Profile Picture URL"+ )+ last_updated: Optional[str] = Field(+ None, description="Last update timestamp", title="Last Updated"+ )diff --git a/flowsint-types/src/flowsint_types/ip.py b/flowsint-types/src/flowsint_types/ip.pyindex f3fe3d6f..d40c6cd2 100644--- a/flowsint-types/src/flowsint_types/ip.py+++ b/flowsint-types/src/flowsint_types/ip.py@@ -1,11 +1,23 @@from pydantic import BaseModel, Fieldfrom typing import Optional+class Ip(BaseModel):"""Represents an IP address with geolocation and ISP information."""+address: str = Field(..., description="IP address", title="IP Address")- latitude: Optional[float] = Field(None, description="Latitude coordinate of the IP location", title="Latitude")- longitude: Optional[float] = Field(None, description="Longitude coordinate of the IP location", title="Longitude")- country: Optional[str] = Field(None, description="Country where the IP is located", title="Country")- city: Optional[str] = Field(None, description="City where the IP is located", title="City")- isp: Optional[str] = Field(None, description="Internet Service Provider", title="ISP")+ latitude: Optional[float] = Field(+ None, description="Latitude coordinate of the IP location", title="Latitude"+ )+ longitude: Optional[float] = Field(+ None, description="Longitude coordinate of the IP location", title="Longitude"+ )+ country: Optional[str] = Field(+ None, description="Country where the IP is located", title="Country"+ )+ city: Optional[str] = Field(+ None, description="City where the IP is located", title="City"+ )+ isp: Optional[str] = Field(+ None, description="Internet Service Provider", title="ISP"+ )diff --git a/flowsint-types/src/flowsint_types/leak.py b/flowsint-types/src/flowsint_types/leak.pyindex 0aaf97e1..ec392304 100644--- a/flowsint-types/src/flowsint_types/leak.py+++ b/flowsint-types/src/flowsint_types/leak.py@@ -1,7 +1,13 @@from typing import List, Dict, Optionalfrom pydantic import BaseModel, Field+class Leak(BaseModel):"""Represents a data leak or breach with associated data."""- name: str = Field(..., description= "The name of the leak or service brea", title="Leak Name")- leak: Optional[List[Dict]] = Field(None, description="List of data leaks found", title="Leak Data")++ name: str = Field(+ ..., description="The name of the leak or service brea", title="Leak Name"+ )+ leak: Optional[List[Dict]] = Field(+ None, description="List of data leaks found", title="Leak Data"+ )diff --git a/flowsint-types/src/flowsint_types/malware.py b/flowsint-types/src/flowsint_types/malware.pyindex 5e80d25f..b6c3c019 100644--- a/flowsint-types/src/flowsint_types/malware.py+++ b/flowsint-types/src/flowsint_types/malware.py@@ -1,24 +1,60 @@from pydantic import BaseModel, Fieldfrom typing import Optional, List+class Malware(BaseModel):"""Represents malware with family, capabilities, and threat intelligence."""+name: str = Field(..., description="Malware name or identifier", title="Name")family: Optional[str] = Field(None, description="Malware family", title="Family")- type: Optional[str] = Field(None, description="Type of malware (trojan, ransomware, etc.)", title="Type")- description: Optional[str] = Field(None, description="Malware description", title="Description")- first_seen: Optional[str] = Field(None, description="First time malware was observed", title="First Seen")- last_seen: Optional[str] = Field(None, description="Last time malware was observed", title="Last Seen")- is_active: Optional[bool] = Field(None, description="Whether malware is currently active", title="Is Active")- threat_level: Optional[str] = Field(None, description="Threat level assessment", title="Threat Level")- targets: Optional[List[str]] = Field(None, description="Target platforms or systems", title="Targets")- capabilities: Optional[List[str]] = Field(None, description="Malware capabilities", title="Capabilities")- indicators: Optional[List[str]] = Field(None, description="IOCs (Indicators of Compromise)", title="Indicators")- c2_servers: Optional[List[str]] = Field(None, description="Command and control servers", title="C2 Servers")- distribution_methods: Optional[List[str]] = Field(None, description="Distribution methods", title="Distribution Methods")- evasion_techniques: Optional[List[str]] = Field(None, description="Evasion techniques used", title="Evasion Techniques")- persistence_mechanisms: Optional[List[str]] = Field(None, description="Persistence mechanisms", title="Persistence Mechanisms")- encryption_methods: Optional[List[str]] = Field(None, description="Encryption methods used", title="Encryption Methods")- source: Optional[str] = Field(None, description="Source of malware information", title="Source")- analysis_report: Optional[str] = Field(None, description="Link to analysis report", title="Analysis Report")- sample_hashes: Optional[List[str]] = Field(None, description="Sample file hashes", title="Sample Hashes")\ No newline at end of file+ type: Optional[str] = Field(+ None, description="Type of malware (trojan, ransomware, etc.)", title="Type"+ )+ description: Optional[str] = Field(+ None, description="Malware description", title="Description"+ )+ first_seen: Optional[str] = Field(+ None, description="First time malware was observed", title="First Seen"+ )+ last_seen: Optional[str] = Field(+ None, description="Last time malware was observed", title="Last Seen"+ )+ is_active: Optional[bool] = Field(+ None, description="Whether malware is currently active", title="Is Active"+ )+ threat_level: Optional[str] = Field(+ None, description="Threat level assessment", title="Threat Level"+ )+ targets: Optional[List[str]] = Field(+ None, description="Target platforms or systems", title="Targets"+ )+ capabilities: Optional[List[str]] = Field(+ None, description="Malware capabilities", title="Capabilities"+ )+ indicators: Optional[List[str]] = Field(+ None, description="IOCs (Indicators of Compromise)", title="Indicators"+ )+ c2_servers: Optional[List[str]] = Field(+ None, description="Command and control servers", title="C2 Servers"+ )+ distribution_methods: Optional[List[str]] = Field(+ None, description="Distribution methods", title="Distribution Methods"+ )+ evasion_techniques: Optional[List[str]] = Field(+ None, description="Evasion techniques used", title="Evasion Techniques"+ )+ persistence_mechanisms: Optional[List[str]] = Field(+ None, description="Persistence mechanisms", title="Persistence Mechanisms"+ )+ encryption_methods: Optional[List[str]] = Field(+ None, description="Encryption methods used", title="Encryption Methods"+ )+ source: Optional[str] = Field(+ None, description="Source of malware information", title="Source"+ )+ analysis_report: Optional[str] = Field(+ None, description="Link to analysis report", title="Analysis Report"+ )+ sample_hashes: Optional[List[str]] = Field(+ None, description="Sample file hashes", title="Sample Hashes"+ )diff --git a/flowsint-types/src/flowsint_types/message.py b/flowsint-types/src/flowsint_types/message.pyindex b642b030..a16875e3 100644--- a/flowsint-types/src/flowsint_types/message.py+++ b/flowsint-types/src/flowsint_types/message.py@@ -1,24 +1,62 @@from pydantic import BaseModel, Fieldfrom typing import Optional, List+class Message(BaseModel):"""Represents a message with content, metadata, and security analysis."""- message_id: str = Field(..., description="Unique message identifier", title="Message ID")++ message_id: str = Field(+ ..., description="Unique message identifier", title="Message ID"+ )content: str = Field(..., description="Message content", title="Content")sender: Optional[str] = Field(None, description="Message sender", title="Sender")- recipient: Optional[str] = Field(None, description="Message recipient", title="Recipient")- timestamp: Optional[str] = Field(None, description="Message timestamp", title="Timestamp")- platform: Optional[str] = Field(None, description="Platform where message was sent", title="Platform")- message_type: Optional[str] = Field(None, description="Type of message (text, email, chat, etc.)", title="Message Type")- subject: Optional[str] = Field(None, description="Message subject (for emails)", title="Subject")- is_read: Optional[bool] = Field(None, description="Whether message has been read", title="Is Read")- is_reply: Optional[bool] = Field(None, description="Whether message is a reply", title="Is Reply")- parent_message_id: Optional[str] = Field(None, description="Parent message ID if reply", title="Parent Message ID")- attachments: Optional[List[str]] = Field(None, description="List of attachment file names", title="Attachments")- language: Optional[str] = Field(None, description="Message language", title="Language")- sentiment: Optional[str] = Field(None, description="Message sentiment analysis", title="Sentiment")- keywords: Optional[List[str]] = Field(None, description="Extracted keywords", title="Keywords")- entities: Optional[List[str]] = Field(None, description="Named entities found in message", title="Entities")- source: Optional[str] = Field(None, description="Source of message information", title="Source")- is_suspicious: Optional[bool] = Field(None, description="Whether message is suspicious", title="Is Suspicious")- threat_level: Optional[str] = Field(None, description="Threat level assessment", title="Threat Level")\ No newline at end of file+ recipient: Optional[str] = Field(+ None, description="Message recipient", title="Recipient"+ )+ timestamp: Optional[str] = Field(+ None, description="Message timestamp", title="Timestamp"+ )+ platform: Optional[str] = Field(+ None, description="Platform where message was sent", title="Platform"+ )+ message_type: Optional[str] = Field(+ None,+ description="Type of message (text, email, chat, etc.)",+ title="Message Type",+ )+ subject: Optional[str] = Field(+ None, description="Message subject (for emails)", title="Subject"+ )+ is_read: Optional[bool] = Field(+ None, description="Whether message has been read", title="Is Read"+ )+ is_reply: Optional[bool] = Field(+ None, description="Whether message is a reply", title="Is Reply"+ )+ parent_message_id: Optional[str] = Field(+ None, description="Parent message ID if reply", title="Parent Message ID"+ )+ attachments: Optional[List[str]] = Field(+ None, description="List of attachment file names", title="Attachments"+ )+ language: Optional[str] = Field(+ None, description="Message language", title="Language"+ )+ sentiment: Optional[str] = Field(+ None, description="Message sentiment analysis", title="Sentiment"+ )+ keywords: Optional[List[str]] = Field(+ None, description="Extracted keywords", title="Keywords"+ )+ entities: Optional[List[str]] = Field(+ None, description="Named entities found in message", title="Entities"+ )+ source: Optional[str] = Field(+ None, description="Source of message information", title="Source"+ )+ is_suspicious: Optional[bool] = Field(+ None, description="Whether message is suspicious", title="Is Suspicious"+ )+ threat_level: Optional[str] = Field(+ None, description="Threat level assessment", title="Threat Level"+ )diff --git a/flowsint-types/src/flowsint_types/organization.py b/flowsint-types/src/flowsint_types/organization.pyindex 7b610e6f..33302bbb 100644--- a/flowsint-types/src/flowsint_types/organization.py+++ b/flowsint-types/src/flowsint_types/organization.py@@ -3,113 +3,361 @@ from typing import Optional, Listfrom .individual import Individualfrom .address import PhysicalAddress+class Organization(BaseModel):"""Represents an organization with detailed business and administrative information."""+# Basic informationname: str = Field(..., description="Organization name", title="Organization Name")siren: Optional[str] = Field(None, description="SIREN number", title="SIREN Number")- nom_complet: Optional[str] = Field(None, description="Complete name", title="Complete Name")- nom_raison_sociale: Optional[str] = Field(None, description="Company name", title="Company Name")+ nom_complet: Optional[str] = Field(+ None, description="Complete name", title="Complete Name"+ )+ nom_raison_sociale: Optional[str] = Field(+ None, description="Company name", title="Company Name"+ )sigle: Optional[str] = Field(None, description="Acronym", title="Acronym")# Establishment information- nombre_etablissements: Optional[int] = Field(None, description="Number of establishments", title="Number of Establishments")- nombre_etablissements_ouverts: Optional[int] = Field(None, description="Number of open establishments", title="Number of Open Establishments")+ nombre_etablissements: Optional[int] = Field(+ None, description="Number of establishments", title="Number of Establishments"+ )+ nombre_etablissements_ouverts: Optional[int] = Field(+ None,+ description="Number of open establishments",+ title="Number of Open Establishments",+ )# Activity information- activite_principale: Optional[str] = Field(None, description="Main activity code", title="Main Activity Code")- section_activite_principale: Optional[str] = Field(None, description="Main activity section", title="Main Activity Section")+ activite_principale: Optional[str] = Field(+ None, description="Main activity code", title="Main Activity Code"+ )+ section_activite_principale: Optional[str] = Field(+ None, description="Main activity section", title="Main Activity Section"+ )# Company category- categorie_entreprise: Optional[str] = Field(None, description="Company category", title="Company Category")- annee_categorie_entreprise: Optional[str] = Field(None, description="Year of company category", title="Company Category Year")+ categorie_entreprise: Optional[str] = Field(+ None, description="Company category", title="Company Category"+ )+ annee_categorie_entreprise: Optional[str] = Field(+ None, description="Year of company category", title="Company Category Year"+ )# Employment information- caractere_employeur: Optional[str] = Field(None, description="Employer character", title="Employer Character")- tranche_effectif_salarie: Optional[str] = Field(None, description="Employee size range", title="Employee Size Range")- annee_tranche_effectif_salarie: Optional[str] = Field(None, description="Year of employee size range", title="Employee Size Range Year")+ caractere_employeur: Optional[str] = Field(+ None, description="Employer character", title="Employer Character"+ )+ tranche_effectif_salarie: Optional[str] = Field(+ None, description="Employee size range", title="Employee Size Range"+ )+ annee_tranche_effectif_salarie: Optional[str] = Field(+ None,+ description="Year of employee size range",+ title="Employee Size Range Year",+ )# Dates- date_creation: Optional[str] = Field(None, description="Creation date", title="Creation Date")- date_fermeture: Optional[str] = Field(None, description="Closing date", title="Closing Date")- date_mise_a_jour: Optional[str] = Field(None, description="Update date", title="Update Date")- date_mise_a_jour_insee: Optional[str] = Field(None, description="INSEE update date", title="INSEE Update Date")- date_mise_a_jour_rne: Optional[str] = Field(None, description="RNE update date", title="RNE Update Date")+ date_creation: Optional[str] = Field(+ None, description="Creation date", title="Creation Date"+ )+ date_fermeture: Optional[str] = Field(+ None, description="Closing date", title="Closing Date"+ )+ date_mise_a_jour: Optional[str] = Field(+ None, description="Update date", title="Update Date"+ )+ date_mise_a_jour_insee: Optional[str] = Field(+ None, description="INSEE update date", title="INSEE Update Date"+ )+ date_mise_a_jour_rne: Optional[str] = Field(+ None, description="RNE update date", title="RNE Update Date"+ )# Legal information- nature_juridique: Optional[str] = Field(None, description="Legal nature", title="Legal Nature")- etat_administratif: Optional[str] = Field(None, description="Administrative status", title="Administrative Status")- statut_diffusion: Optional[str] = Field(None, description="Diffusion status", title="Diffusion Status")+ nature_juridique: Optional[str] = Field(+ None, description="Legal nature", title="Legal Nature"+ )+ etat_administratif: Optional[str] = Field(+ None, description="Administrative status", title="Administrative Status"+ )+ statut_diffusion: Optional[str] = Field(+ None, description="Diffusion status", title="Diffusion Status"+ )# Siege (Headquarters) information- siege_activite_principale: Optional[str] = Field(None, description="Siege main activity code", title="Headquarters Main Activity")- siege_activite_principale_registre_metier: Optional[str] = Field(None, description="Siege main activity in trade register", title="Headquarters Trade Register Activity")- siege_annee_tranche_effectif_salarie: Optional[str] = Field(None, description="Siege year of employee size range", title="Headquarters Employee Size Year")- siege_adresse: Optional[str] = Field(None, description="Siege address", title="Headquarters Address")- siege_caractere_employeur: Optional[str] = Field(None, description="Siege employer character", title="Headquarters Employer Character")- siege_cedex: Optional[str] = Field(None, description="Siege CEDEX code", title="Headquarters CEDEX")- siege_code_pays_etranger: Optional[str] = Field(None, description="Siege foreign country code", title="Headquarters Foreign Country Code")- siege_code_postal: Optional[str] = Field(None, description="Siege postal code", title="Headquarters Postal Code")- siege_commune: Optional[str] = Field(None, description="Siege municipality code", title="Headquarters Municipality Code")- siege_complement_adresse: Optional[str] = Field(None, description="Siege address complement", title="Headquarters Address Complement")- siege_coordonnees: Optional[str] = Field(None, description="Siege coordinates", title="Headquarters Coordinates")- siege_date_creation: Optional[str] = Field(None, description="Siege creation date", title="Headquarters Creation Date")- siege_date_debut_activite: Optional[str] = Field(None, description="Siege activity start date", title="Headquarters Activity Start Date")- siege_date_fermeture: Optional[str] = Field(None, description="Siege closing date", title="Headquarters Closing Date")- siege_date_mise_a_jour: Optional[str] = Field(None, description="Siege update date", title="Headquarters Update Date")- siege_date_mise_a_jour_insee: Optional[str] = Field(None, description="Siege INSEE update date", title="Headquarters INSEE Update Date")- siege_departement: Optional[str] = Field(None, description="Siege department code", title="Headquarters Department")- siege_distribution_speciale: Optional[str] = Field(None, description="Siege special distribution", title="Headquarters Special Distribution")- siege_epci: Optional[str] = Field(None, description="Siege EPCI code", title="Headquarters EPCI Code")- siege_est_siege: Optional[bool] = Field(None, description="Siege is headquarters", title="Is Headquarters")- siege_etat_administratif: Optional[str] = Field(None, description="Siege administrative status", title="Headquarters Administrative Status")- siege_geo_adresse: Optional[PhysicalAddress] = Field(None, description="Siege geocoded address", title="Headquarters Geocoded Address")- siege_geo_id: Optional[str] = Field(None, description="Siege geographic ID", title="Headquarters Geographic ID")- siege_indice_repetition: Optional[str] = Field(None, description="Siege repetition index", title="Headquarters Repetition Index")- siege_latitude: Optional[str] = Field(None, description="Siege latitude", title="Headquarters Latitude")- siege_libelle_cedex: Optional[str] = Field(None, description="Siege CEDEX label", title="Headquarters CEDEX Label")- siege_libelle_commune: Optional[str] = Field(None, description="Siege municipality label", title="Headquarters Municipality Label")- siege_libelle_commune_etranger: Optional[str] = Field(None, description="Siege foreign municipality label", title="Headquarters Foreign Municipality Label")- siege_libelle_pays_etranger: Optional[str] = Field(None, description="Siege foreign country label", title="Headquarters Foreign Country Label")- siege_libelle_voie: Optional[str] = Field(None, description="Siege street label", title="Headquarters Street Label")- siege_liste_enseignes: Optional[List[str]] = Field(None, description="Siege list of brands", title="Headquarters Brands List")- siege_liste_finess: Optional[List[str]] = Field(None, description="Siege list of FINESS", title="Headquarters FINESS List")- siege_liste_id_bio: Optional[List[str]] = Field(None, description="Siege list of bio IDs", title="Headquarters Bio IDs List")- siege_liste_idcc: Optional[List[str]] = Field(None, description="Siege list of IDCC", title="Headquarters IDCC List")- siege_liste_id_organisme_formation: Optional[List[str]] = Field(None, description="Siege list of training organization IDs", title="Headquarters Training Organization IDs")- siege_liste_rge: Optional[List[str]] = Field(None, description="Siege list of RGE", title="Headquarters RGE List")- siege_liste_uai: Optional[List[str]] = Field(None, description="Siege list of UAI", title="Headquarters UAI List")- siege_longitude: Optional[str] = Field(None, description="Siege longitude", title="Headquarters Longitude")- siege_nom_commercial: Optional[str] = Field(None, description="Siege commercial name", title="Headquarters Commercial Name")- siege_numero_voie: Optional[str] = Field(None, description="Siege street number", title="Headquarters Street Number")- siege_region: Optional[str] = Field(None, description="Siege region code", title="Headquarters Region")- siege_siret: Optional[str] = Field(None, description="Siege SIRET number", title="Headquarters SIRET")- siege_statut_diffusion_etablissement: Optional[str] = Field(None, description="Siege establishment diffusion status", title="Headquarters Establishment Diffusion Status")- siege_tranche_effectif_salarie: Optional[str] = Field(None, description="Siege employee size range", title="Headquarters Employee Size Range")- siege_type_voie: Optional[str] = Field(None, description="Siege street type", title="Headquarters Street Type")+ siege_activite_principale: Optional[str] = Field(+ None, description="Siege main activity code", title="Headquarters Main Activity"+ )+ siege_activite_principale_registre_metier: Optional[str] = Field(+ None,+ description="Siege main activity in trade register",+ title="Headquarters Trade Register Activity",+ )+ siege_annee_tranche_effectif_salarie: Optional[str] = Field(+ None,+ description="Siege year of employee size range",+ title="Headquarters Employee Size Year",+ )+ siege_adresse: Optional[str] = Field(+ None, description="Siege address", title="Headquarters Address"+ )+ siege_caractere_employeur: Optional[str] = Field(+ None,+ description="Siege employer character",+ title="Headquarters Employer Character",+ )+ siege_cedex: Optional[str] = Field(+ None, description="Siege CEDEX code", title="Headquarters CEDEX"+ )+ siege_code_pays_etranger: Optional[str] = Field(+ None,+ description="Siege foreign country code",+ title="Headquarters Foreign Country Code",+ )+ siege_code_postal: Optional[str] = Field(+ None, description="Siege postal code", title="Headquarters Postal Code"+ )+ siege_commune: Optional[str] = Field(+ None,+ description="Siege municipality code",+ title="Headquarters Municipality Code",+ )+ siege_complement_adresse: Optional[str] = Field(+ None,+ description="Siege address complement",+ title="Headquarters Address Complement",+ )+ siege_coordonnees: Optional[str] = Field(+ None, description="Siege coordinates", title="Headquarters Coordinates"+ )+ siege_date_creation: Optional[str] = Field(+ None, description="Siege creation date", title="Headquarters Creation Date"+ )+ siege_date_debut_activite: Optional[str] = Field(+ None,+ description="Siege activity start date",+ title="Headquarters Activity Start Date",+ )+ siege_date_fermeture: Optional[str] = Field(+ None, description="Siege closing date", title="Headquarters Closing Date"+ )+ siege_date_mise_a_jour: Optional[str] = Field(+ None, description="Siege update date", title="Headquarters Update Date"+ )+ siege_date_mise_a_jour_insee: Optional[str] = Field(+ None,+ description="Siege INSEE update date",+ title="Headquarters INSEE Update Date",+ )+ siege_departement: Optional[str] = Field(+ None, description="Siege department code", title="Headquarters Department"+ )+ siege_distribution_speciale: Optional[str] = Field(+ None,+ description="Siege special distribution",+ title="Headquarters Special Distribution",+ )+ siege_epci: Optional[str] = Field(+ None, description="Siege EPCI code", title="Headquarters EPCI Code"+ )+ siege_est_siege: Optional[bool] = Field(+ None, description="Siege is headquarters", title="Is Headquarters"+ )+ siege_etat_administratif: Optional[str] = Field(+ None,+ description="Siege administrative status",+ title="Headquarters Administrative Status",+ )+ siege_geo_adresse: Optional[PhysicalAddress] = Field(+ None,+ description="Siege geocoded address",+ title="Headquarters Geocoded Address",+ )+ siege_geo_id: Optional[str] = Field(+ None, description="Siege geographic ID", title="Headquarters Geographic ID"+ )+ siege_indice_repetition: Optional[str] = Field(+ None,+ description="Siege repetition index",+ title="Headquarters Repetition Index",+ )+ siege_latitude: Optional[str] = Field(+ None, description="Siege latitude", title="Headquarters Latitude"+ )+ siege_libelle_cedex: Optional[str] = Field(+ None, description="Siege CEDEX label", title="Headquarters CEDEX Label"+ )+ siege_libelle_commune: Optional[str] = Field(+ None,+ description="Siege municipality label",+ title="Headquarters Municipality Label",+ )+ siege_libelle_commune_etranger: Optional[str] = Field(+ None,+ description="Siege foreign municipality label",+ title="Headquarters Foreign Municipality Label",+ )+ siege_libelle_pays_etranger: Optional[str] = Field(+ None,+ description="Siege foreign country label",+ title="Headquarters Foreign Country Label",+ )+ siege_libelle_voie: Optional[str] = Field(+ None, description="Siege street label", title="Headquarters Street Label"+ )+ siege_liste_enseignes: Optional[List[str]] = Field(+ None, description="Siege list of brands", title="Headquarters Brands List"+ )+ siege_liste_finess: Optional[List[str]] = Field(+ None, description="Siege list of FINESS", title="Headquarters FINESS List"+ )+ siege_liste_id_bio: Optional[List[str]] = Field(+ None, description="Siege list of bio IDs", title="Headquarters Bio IDs List"+ )+ siege_liste_idcc: Optional[List[str]] = Field(+ None, description="Siege list of IDCC", title="Headquarters IDCC List"+ )+ siege_liste_id_organisme_formation: Optional[List[str]] = Field(+ None,+ description="Siege list of training organization IDs",+ title="Headquarters Training Organization IDs",+ )+ siege_liste_rge: Optional[List[str]] = Field(+ None, description="Siege list of RGE", title="Headquarters RGE List"+ )+ siege_liste_uai: Optional[List[str]] = Field(+ None, description="Siege list of UAI", title="Headquarters UAI List"+ )+ siege_longitude: Optional[str] = Field(+ None, description="Siege longitude", title="Headquarters Longitude"+ )+ siege_nom_commercial: Optional[str] = Field(+ None, description="Siege commercial name", title="Headquarters Commercial Name"+ )+ siege_numero_voie: Optional[str] = Field(+ None, description="Siege street number", title="Headquarters Street Number"+ )+ siege_region: Optional[str] = Field(+ None, description="Siege region code", title="Headquarters Region"+ )+ siege_siret: Optional[str] = Field(+ None, description="Siege SIRET number", title="Headquarters SIRET"+ )+ siege_statut_diffusion_etablissement: Optional[str] = Field(+ None,+ description="Siege establishment diffusion status",+ title="Headquarters Establishment Diffusion Status",+ )+ siege_tranche_effectif_salarie: Optional[str] = Field(+ None,+ description="Siege employee size range",+ title="Headquarters Employee Size Range",+ )+ siege_type_voie: Optional[str] = Field(+ None, description="Siege street type", title="Headquarters Street Type"+ )# Dirigeants (Leaders) - using Individual type- dirigeants: Optional[List[Individual]] = Field(None, description="List of leaders", title="Leaders")+ dirigeants: Optional[List[Individual]] = Field(+ None, description="List of leaders", title="Leaders"+ )# Matching establishments- matching_etablissements: Optional[List[dict]] = Field(None, description="Matching establishments", title="Matching Establishments")+ matching_etablissements: Optional[List[dict]] = Field(+ None, description="Matching establishments", title="Matching Establishments"+ )# Financial information- finances: Optional[dict] = Field(None, description="Financial information", title="Financial Information")+ finances: Optional[dict] = Field(+ None, description="Financial information", title="Financial Information"+ )# Complements (Additional information)- complements_collectivite_territoriale: Optional[str] = Field(None, description="Complements territorial community", title="Territorial Community")- complements_convention_collective_renseignee: Optional[bool] = Field(None, description="Complements collective agreement informed", title="Collective Agreement Informed")- complements_liste_idcc: Optional[List[str]] = Field(None, description="Complements list of IDCC", title="IDCC List")- complements_egapro_renseignee: Optional[bool] = Field(None, description="Complements Egapro informed", title="Egapro Informed")- complements_est_achats_responsables: Optional[bool] = Field(None, description="Complements is responsible purchases", title="Is Responsible Purchases")- complements_est_alim_confiance: Optional[bool] = Field(None, description="Complements is food trust", title="Is Food Trust")- complements_est_association: Optional[bool] = Field(None, description="Complements is association", title="Is Association")- complements_est_bio: Optional[bool] = Field(None, description="Complements is bio", title="Is Bio")- complements_est_entrepreneur_individuel: Optional[bool] = Field(None, description="Complements is individual entrepreneur", title="Is Individual Entrepreneur")- complements_est_entrepreneur_spectacle: Optional[bool] = Field(None, description="Complements is show entrepreneur", title="Is Show Entrepreneur")- complements_est_ess: Optional[bool] = Field(None, description="Complements is ESS", title="Is ESS")- complements_est_finess: Optional[bool] = Field(None, description="Complements is FINESS", title="Is FINESS")- complements_est_organisme_formation: Optional[bool] = Field(None, description="Complements is training organization", title="Is Training Organization")- complements_est_qualiopi: Optional[bool] = Field(None, description="Complements is Qualiopi", title="Is Qualiopi")- complements_liste_id_organisme_formation: Optional[List[str]] = Field(None, description="Complements list of training organization IDs", title="Training Organization IDs")- complements_est_rge: Optional[bool] = Field(None, description="Complements is RGE", title="Is RGE")- complements_est_service_public: Optional[bool] = Field(None, description="Complements is public service", title="Is Public Service")- complements_est_l100_3: Optional[bool] = Field(None, description="Complements is L100-3", title="Is L100-3")- complements_est_siae: Optional[bool] = Field(None, description="Complements is SIAE", title="Is SIAE")- complements_est_societe_mission: Optional[bool] = Field(None, description="Complements is mission company", title="Is Mission Company")- complements_est_uai: Optional[bool] = Field(None, description="Complements is UAI", title="Is UAI")- complements_est_patrimoine_vivant: Optional[bool] = Field(None, description="Complements is living heritage", title="Is Living Heritage")- complements_bilan_ges_renseigne: Optional[bool] = Field(None, description="Complements GES balance informed", title="GES Balance Informed")- complements_identifiant_association: Optional[str] = Field(None, description="Complements association identifier", title="Association Identifier")- complements_statut_entrepreneur_spectacle: Optional[str] = Field(None, description="Complements show entrepreneur status", title="Show Entrepreneur Status")- complements_type_siae: Optional[str] = Field(None, description="Complements SIAE type", title="SIAE Type")\ No newline at end of file+ complements_collectivite_territoriale: Optional[str] = Field(+ None,+ description="Complements territorial community",+ title="Territorial Community",+ )+ complements_convention_collective_renseignee: Optional[bool] = Field(+ None,+ description="Complements collective agreement informed",+ title="Collective Agreement Informed",+ )+ complements_liste_idcc: Optional[List[str]] = Field(+ None, description="Complements list of IDCC", title="IDCC List"+ )+ complements_egapro_renseignee: Optional[bool] = Field(+ None, description="Complements Egapro informed", title="Egapro Informed"+ )+ complements_est_achats_responsables: Optional[bool] = Field(+ None,+ description="Complements is responsible purchases",+ title="Is Responsible Purchases",+ )+ complements_est_alim_confiance: Optional[bool] = Field(+ None, description="Complements is food trust", title="Is Food Trust"+ )+ complements_est_association: Optional[bool] = Field(+ None, description="Complements is association", title="Is Association"+ )+ complements_est_bio: Optional[bool] = Field(+ None, description="Complements is bio", title="Is Bio"+ )+ complements_est_entrepreneur_individuel: Optional[bool] = Field(+ None,+ description="Complements is individual entrepreneur",+ title="Is Individual Entrepreneur",+ )+ complements_est_entrepreneur_spectacle: Optional[bool] = Field(+ None,+ description="Complements is show entrepreneur",+ title="Is Show Entrepreneur",+ )+ complements_est_ess: Optional[bool] = Field(+ None, description="Complements is ESS", title="Is ESS"+ )+ complements_est_finess: Optional[bool] = Field(+ None, description="Complements is FINESS", title="Is FINESS"+ )+ complements_est_organisme_formation: Optional[bool] = Field(+ None,+ description="Complements is training organization",+ title="Is Training Organization",+ )+ complements_est_qualiopi: Optional[bool] = Field(+ None, description="Complements is Qualiopi", title="Is Qualiopi"+ )+ complements_liste_id_organisme_formation: Optional[List[str]] = Field(+ None,+ description="Complements list of training organization IDs",+ title="Training Organization IDs",+ )+ complements_est_rge: Optional[bool] = Field(+ None, description="Complements is RGE", title="Is RGE"+ )+ complements_est_service_public: Optional[bool] = Field(+ None, description="Complements is public service", title="Is Public Service"+ )+ complements_est_l100_3: Optional[bool] = Field(+ None, description="Complements is L100-3", title="Is L100-3"+ )+ complements_est_siae: Optional[bool] = Field(+ None, description="Complements is SIAE", title="Is SIAE"+ )+ complements_est_societe_mission: Optional[bool] = Field(+ None, description="Complements is mission company", title="Is Mission Company"+ )+ complements_est_uai: Optional[bool] = Field(+ None, description="Complements is UAI", title="Is UAI"+ )+ complements_est_patrimoine_vivant: Optional[bool] = Field(+ None, description="Complements is living heritage", title="Is Living Heritage"+ )+ complements_bilan_ges_renseigne: Optional[bool] = Field(+ None,+ description="Complements GES balance informed",+ title="GES Balance Informed",+ )+ complements_identifiant_association: Optional[str] = Field(+ None,+ description="Complements association identifier",+ title="Association Identifier",+ )+ complements_statut_entrepreneur_spectacle: Optional[str] = Field(+ None,+ description="Complements show entrepreneur status",+ title="Show Entrepreneur Status",+ )+ complements_type_siae: Optional[str] = Field(+ None, description="Complements SIAE type", title="SIAE Type"+ )diff --git a/flowsint-types/src/flowsint_types/phone.py b/flowsint-types/src/flowsint_types/phone.pyindex 0c5eb04a..368e1c96 100644--- a/flowsint-types/src/flowsint_types/phone.py+++ b/flowsint-types/src/flowsint_types/phone.py@@ -1,8 +1,14 @@from pydantic import BaseModel, Fieldfrom typing import Optional+class Phone(BaseModel):"""Represents a phone number with country and carrier information."""+number: str = Field(..., description="Phone number", title="Phone Number")- country: Optional[str] = Field(None, description="Country code (ISO 3166-1 alpha-2)", title="Country Code")- carrier: Optional[str] = Field(None, description="Mobile carrier or service provider", title="Carrier")\ No newline at end of file+ country: Optional[str] = Field(+ None, description="Country code (ISO 3166-1 alpha-2)", title="Country Code"+ )+ carrier: Optional[str] = Field(+ None, description="Mobile carrier or service provider", title="Carrier"+ )diff --git a/flowsint-types/src/flowsint_types/phrase.py b/flowsint-types/src/flowsint_types/phrase.pyindex a0b47649..678832c8 100644--- a/flowsint-types/src/flowsint_types/phrase.py+++ b/flowsint-types/src/flowsint_types/phrase.py@@ -1,7 +1,10 @@from pydantic import BaseModel, Fieldfrom typing import Any+class Phrase(BaseModel):"""Represents a phrase or text content."""- text: Any = Field(..., description="The content of the phrase.", title="Phrase text value.")-++ text: Any = Field(+ ..., description="The content of the phrase.", title="Phrase text value."+ )diff --git a/flowsint-types/src/flowsint_types/reputation_score.py b/flowsint-types/src/flowsint_types/reputation_score.pyindex 81cc0928..c08693e9 100644--- a/flowsint-types/src/flowsint_types/reputation_score.py+++ b/flowsint-types/src/flowsint_types/reputation_score.py@@ -1,20 +1,56 @@from pydantic import BaseModel, Fieldfrom typing import Optional, List+class ReputationScore(BaseModel):"""Represents a reputation score for an entity with historical data and trends."""+entity_id: str = Field(..., description="Entity identifier", title="Entity ID")- entity_type: Optional[str] = Field(None, description="Type of entity (domain, IP, email, etc.)", title="Entity Type")- score: Optional[float] = Field(None, description="Reputation score (0-100)", title="Score")- score_type: Optional[str] = Field(None, description="Type of score (trust, spam, malware, etc.)", title="Score Type")- provider: Optional[str] = Field(None, description="Reputation provider", title="Provider")- last_updated: Optional[str] = Field(None, description="Last update timestamp", title="Last Updated")- confidence: Optional[float] = Field(None, description="Confidence in the score", title="Confidence")- factors: Optional[List[str]] = Field(None, description="Factors contributing to score", title="Factors")- category: Optional[str] = Field(None, description="Category (good, suspicious, malicious, etc.)", title="Category")- description: Optional[str] = Field(None, description="Description of reputation", title="Description")- source: Optional[str] = Field(None, description="Source of reputation data", title="Source")- historical_scores: Optional[List[dict]] = Field(None, description="Historical score data", title="Historical Scores")- trend: Optional[str] = Field(None, description="Score trend (improving, declining, stable)", title="Trend")- risk_level: Optional[str] = Field(None, description="Risk level assessment", title="Risk Level")- recommendations: Optional[List[str]] = Field(None, description="Recommendations based on score", title="Recommendations")\ No newline at end of file+ entity_type: Optional[str] = Field(+ None,+ description="Type of entity (domain, IP, email, etc.)",+ title="Entity Type",+ )+ score: Optional[float] = Field(+ None, description="Reputation score (0-100)", title="Score"+ )+ score_type: Optional[str] = Field(+ None,+ description="Type of score (trust, spam, malware, etc.)",+ title="Score Type",+ )+ provider: Optional[str] = Field(+ None, description="Reputation provider", title="Provider"+ )+ last_updated: Optional[str] = Field(+ None, description="Last update timestamp", title="Last Updated"+ )+ confidence: Optional[float] = Field(+ None, description="Confidence in the score", title="Confidence"+ )+ factors: Optional[List[str]] = Field(+ None, description="Factors contributing to score", title="Factors"+ )+ category: Optional[str] = Field(+ None,+ description="Category (good, suspicious, malicious, etc.)",+ title="Category",+ )+ description: Optional[str] = Field(+ None, description="Description of reputation", title="Description"+ )+ source: Optional[str] = Field(+ None, description="Source of reputation data", title="Source"+ )+ historical_scores: Optional[List[dict]] = Field(+ None, description="Historical score data", title="Historical Scores"+ )+ trend: Optional[str] = Field(+ None, description="Score trend (improving, declining, stable)", title="Trend"+ )+ risk_level: Optional[str] = Field(+ None, description="Risk level assessment", title="Risk Level"+ )+ recommendations: Optional[List[str]] = Field(+ None, description="Recommendations based on score", title="Recommendations"+ )diff --git a/flowsint-types/src/flowsint_types/risk_profile.py b/flowsint-types/src/flowsint_types/risk_profile.pyindex 677a672e..38a52326 100644--- a/flowsint-types/src/flowsint_types/risk_profile.py+++ b/flowsint-types/src/flowsint_types/risk_profile.py@@ -1,27 +1,73 @@from pydantic import BaseModel, Fieldfrom typing import Optional, List+class RiskProfile(BaseModel):"""Represents a comprehensive risk assessment profile for an entity."""+entity_id: str = Field(..., description="Entity identifier", title="Entity ID")- entity_type: Optional[str] = Field(None, description="Type of entity", title="Entity Type")- overall_risk_score: Optional[float] = Field(None, description="Overall risk score (0-100)", title="Overall Risk Score")- risk_level: Optional[str] = Field(None, description="Risk level (low, medium, high, critical)", title="Risk Level")- assessment_date: Optional[str] = Field(None, description="Date of risk assessment", title="Assessment Date")- last_updated: Optional[str] = Field(None, description="Last update timestamp", title="Last Updated")- risk_factors: Optional[List[str]] = Field(None, description="Identified risk factors", title="Risk Factors")- threat_actors: Optional[List[str]] = Field(None, description="Associated threat actors", title="Threat Actors")- attack_vectors: Optional[List[str]] = Field(None, description="Potential attack vectors", title="Attack Vectors")- vulnerabilities: Optional[List[str]] = Field(None, description="Identified vulnerabilities", title="Vulnerabilities")- exposure_surface: Optional[str] = Field(None, description="Exposure surface description", title="Exposure Surface")- data_classification: Optional[str] = Field(None, description="Data classification level", title="Data Classification")- compliance_risks: Optional[List[str]] = Field(None, description="Compliance-related risks", title="Compliance Risks")- financial_impact: Optional[str] = Field(None, description="Potential financial impact", title="Financial Impact")- reputation_impact: Optional[str] = Field(None, description="Potential reputation impact", title="Reputation Impact")- operational_impact: Optional[str] = Field(None, description="Potential operational impact", title="Operational Impact")- mitigation_strategies: Optional[List[str]] = Field(None, description="Recommended mitigation strategies", title="Mitigation Strategies")- monitoring_recommendations: Optional[List[str]] = Field(None, description="Monitoring recommendations", title="Monitoring Recommendations")- source: Optional[str] = Field(None, description="Source of risk assessment", title="Source")+ entity_type: Optional[str] = Field(+ None, description="Type of entity", title="Entity Type"+ )+ overall_risk_score: Optional[float] = Field(+ None, description="Overall risk score (0-100)", title="Overall Risk Score"+ )+ risk_level: Optional[str] = Field(+ None, description="Risk level (low, medium, high, critical)", title="Risk Level"+ )+ assessment_date: Optional[str] = Field(+ None, description="Date of risk assessment", title="Assessment Date"+ )+ last_updated: Optional[str] = Field(+ None, description="Last update timestamp", title="Last Updated"+ )+ risk_factors: Optional[List[str]] = Field(+ None, description="Identified risk factors", title="Risk Factors"+ )+ threat_actors: Optional[List[str]] = Field(+ None, description="Associated threat actors", title="Threat Actors"+ )+ attack_vectors: Optional[List[str]] = Field(+ None, description="Potential attack vectors", title="Attack Vectors"+ )+ vulnerabilities: Optional[List[str]] = Field(+ None, description="Identified vulnerabilities", title="Vulnerabilities"+ )+ exposure_surface: Optional[str] = Field(+ None, description="Exposure surface description", title="Exposure Surface"+ )+ data_classification: Optional[str] = Field(+ None, description="Data classification level", title="Data Classification"+ )+ compliance_risks: Optional[List[str]] = Field(+ None, description="Compliance-related risks", title="Compliance Risks"+ )+ financial_impact: Optional[str] = Field(+ None, description="Potential financial impact", title="Financial Impact"+ )+ reputation_impact: Optional[str] = Field(+ None, description="Potential reputation impact", title="Reputation Impact"+ )+ operational_impact: Optional[str] = Field(+ None, description="Potential operational impact", title="Operational Impact"+ )+ mitigation_strategies: Optional[List[str]] = Field(+ None,+ description="Recommended mitigation strategies",+ title="Mitigation Strategies",+ )+ monitoring_recommendations: Optional[List[str]] = Field(+ None,+ description="Monitoring recommendations",+ title="Monitoring Recommendations",+ )+ source: Optional[str] = Field(+ None, description="Source of risk assessment", title="Source"+ )assessor: Optional[str] = Field(None, description="Risk assessor", title="Assessor")- confidence: Optional[float] = Field(None, description="Confidence in assessment", title="Confidence")- next_review_date: Optional[str] = Field(None, description="Next review date", title="Next Review Date")\ No newline at end of file+ confidence: Optional[float] = Field(+ None, description="Confidence in assessment", title="Confidence"+ )+ next_review_date: Optional[str] = Field(+ None, description="Next review date", title="Next Review Date"+ )diff --git a/flowsint-types/src/flowsint_types/script.py b/flowsint-types/src/flowsint_types/script.pyindex 833cfbdf..7d564fe9 100644--- a/flowsint-types/src/flowsint_types/script.py+++ b/flowsint-types/src/flowsint_types/script.py@@ -1,29 +1,65 @@from pydantic import BaseModel, Fieldfrom typing import Optional, List+class Script(BaseModel):"""Represents a script or code file with analysis and security information."""- script_id: str = Field(..., description="Unique script identifier", title="Script ID")++ script_id: str = Field(+ ..., description="Unique script identifier", title="Script ID"+ )name: Optional[str] = Field(None, description="Script name", title="Name")- language: Optional[str] = Field(None, description="Programming language", title="Language")- type: Optional[str] = Field(None, description="Type of script (JavaScript, Python, etc.)", title="Type")+ language: Optional[str] = Field(+ None, description="Programming language", title="Language"+ )+ type: Optional[str] = Field(+ None, description="Type of script (JavaScript, Python, etc.)", title="Type"+ )content: Optional[str] = Field(None, description="Script content", title="Content")- file_path: Optional[str] = Field(None, description="File path if stored", title="File Path")+ file_path: Optional[str] = Field(+ None, description="File path if stored", title="File Path"+ )url: Optional[str] = Field(None, description="URL if loaded from web", title="URL")hash_md5: Optional[str] = Field(None, description="MD5 hash", title="MD5 Hash")hash_sha1: Optional[str] = Field(None, description="SHA1 hash", title="SHA1 Hash")- hash_sha256: Optional[str] = Field(None, description="SHA256 hash", title="SHA256 Hash")- file_size: Optional[int] = Field(None, description="File size in bytes", title="File Size")- created_date: Optional[str] = Field(None, description="Creation date", title="Created Date")- modified_date: Optional[str] = Field(None, description="Last modified date", title="Modified Date")+ hash_sha256: Optional[str] = Field(+ None, description="SHA256 hash", title="SHA256 Hash"+ )+ file_size: Optional[int] = Field(+ None, description="File size in bytes", title="File Size"+ )+ created_date: Optional[str] = Field(+ None, description="Creation date", title="Created Date"+ )+ modified_date: Optional[str] = Field(+ None, description="Last modified date", title="Modified Date"+ )author: Optional[str] = Field(None, description="Script author", title="Author")version: Optional[str] = Field(None, description="Script version", title="Version")- description: Optional[str] = Field(None, description="Script description", title="Description")- dependencies: Optional[List[str]] = Field(None, description="Script dependencies", title="Dependencies")- functions: Optional[List[str]] = Field(None, description="Functions defined in script", title="Functions")- is_malicious: Optional[bool] = Field(None, description="Whether script is malicious", title="Is Malicious")- malware_family: Optional[str] = Field(None, description="Malware family if malicious", title="Malware Family")- threat_level: Optional[str] = Field(None, description="Threat level assessment", title="Threat Level")- source: Optional[str] = Field(None, description="Source of script information", title="Source")- obfuscated: Optional[bool] = Field(None, description="Whether script is obfuscated", title="Obfuscated")- minified: Optional[bool] = Field(None, description="Whether script is minified", title="Minified")\ No newline at end of file+ description: Optional[str] = Field(+ None, description="Script description", title="Description"+ )+ dependencies: Optional[List[str]] = Field(+ None, description="Script dependencies", title="Dependencies"+ )+ functions: Optional[List[str]] = Field(+ None, description="Functions defined in script", title="Functions"+ )+ is_malicious: Optional[bool] = Field(+ None, description="Whether script is malicious", title="Is Malicious"+ )+ malware_family: Optional[str] = Field(+ None, description="Malware family if malicious", title="Malware Family"+ )+ threat_level: Optional[str] = Field(+ None, description="Threat level assessment", title="Threat Level"+ )+ source: Optional[str] = Field(+ None, description="Source of script information", title="Source"+ )+ obfuscated: Optional[bool] = Field(+ None, description="Whether script is obfuscated", title="Obfuscated"+ )+ minified: Optional[bool] = Field(+ None, description="Whether script is minified", title="Minified"+ )diff --git a/flowsint-types/src/flowsint_types/session.py b/flowsint-types/src/flowsint_types/session.pyindex 7d4294bd..9d025e4e 100644--- a/flowsint-types/src/flowsint_types/session.py+++ b/flowsint-types/src/flowsint_types/session.py@@ -1,21 +1,51 @@from pydantic import BaseModel, Fieldfrom typing import Optional, List+class Session(BaseModel):"""Represents a user session with device and activity information."""- session_id: str = Field(..., description="Unique session identifier", title="Session ID")++ session_id: str = Field(+ ..., description="Unique session identifier", title="Session ID"+ )user_id: Optional[str] = Field(None, description="User identifier", title="User ID")- service: Optional[str] = Field(None, description="Service or platform", title="Service")- start_time: Optional[str] = Field(None, description="Session start timestamp", title="Start Time")- end_time: Optional[str] = Field(None, description="Session end timestamp", title="End Time")- duration: Optional[int] = Field(None, description="Session duration in seconds", title="Duration")- ip_address: Optional[str] = Field(None, description="IP address used for session", title="IP Address")- user_agent: Optional[str] = Field(None, description="User agent string", title="User Agent")- location: Optional[str] = Field(None, description="Geographic location", title="Location")- device_type: Optional[str] = Field(None, description="Type of device used", title="Device Type")+ service: Optional[str] = Field(+ None, description="Service or platform", title="Service"+ )+ start_time: Optional[str] = Field(+ None, description="Session start timestamp", title="Start Time"+ )+ end_time: Optional[str] = Field(+ None, description="Session end timestamp", title="End Time"+ )+ duration: Optional[int] = Field(+ None, description="Session duration in seconds", title="Duration"+ )+ ip_address: Optional[str] = Field(+ None, description="IP address used for session", title="IP Address"+ )+ user_agent: Optional[str] = Field(+ None, description="User agent string", title="User Agent"+ )+ location: Optional[str] = Field(+ None, description="Geographic location", title="Location"+ )+ device_type: Optional[str] = Field(+ None, description="Type of device used", title="Device Type"+ )browser: Optional[str] = Field(None, description="Browser used", title="Browser")- os: Optional[str] = Field(None, description="Operating system", title="Operating System")- is_active: Optional[bool] = Field(None, description="Whether session is currently active", title="Is Active")- is_suspicious: Optional[bool] = Field(None, description="Whether session is suspicious", title="Is Suspicious")- activities: Optional[List[str]] = Field(None, description="Activities performed during session", title="Activities")- source: Optional[str] = Field(None, description="Source of session information", title="Source")\ No newline at end of file+ os: Optional[str] = Field(+ None, description="Operating system", title="Operating System"+ )+ is_active: Optional[bool] = Field(+ None, description="Whether session is currently active", title="Is Active"+ )+ is_suspicious: Optional[bool] = Field(+ None, description="Whether session is suspicious", title="Is Suspicious"+ )+ activities: Optional[List[str]] = Field(+ None, description="Activities performed during session", title="Activities"+ )+ source: Optional[str] = Field(+ None, description="Source of session information", title="Source"+ )diff --git a/flowsint-types/src/flowsint_types/social.py b/flowsint-types/src/flowsint_types/social.pyindex 37726c1e..fe9a23b5 100644--- a/flowsint-types/src/flowsint_types/social.py+++ b/flowsint-types/src/flowsint_types/social.py@@ -1,14 +1,31 @@from pydantic import BaseModel, Fieldfrom typing import Optional+class SocialProfile(BaseModel):"""Represents a social media profile with engagement metrics."""- username: str = Field(..., description="Username on the social platform", title="Username")- profile_url: Optional[str] = Field(None, description="URL to the user's profile page", title="Profile URL")- platform: Optional[str] = Field(None, description="Name of the social media platform", title="Platform")- profile_picture_url: Optional[str] = Field(None, description="URL to the user's profile picture", title="Profile Picture")- bio: Optional[str] = Field(None, description="User's biography or description", title="Bio")- followers_count: Optional[int] = Field(None, description="Number of followers", title="Followers Count")- following_count: Optional[int] = Field(None, description="Number of users being followed", title="Following Count")- posts_count: Optional[int] = Field(None, description="Number of posts made by the user", title="Posts Count")-\ No newline at end of file++ username: str = Field(+ ..., description="Username on the social platform", title="Username"+ )+ profile_url: Optional[str] = Field(+ None, description="URL to the user's profile page", title="Profile URL"+ )+ platform: Optional[str] = Field(+ None, description="Name of the social media platform", title="Platform"+ )+ profile_picture_url: Optional[str] = Field(+ None, description="URL to the user's profile picture", title="Profile Picture"+ )+ bio: Optional[str] = Field(+ None, description="User's biography or description", title="Bio"+ )+ followers_count: Optional[int] = Field(+ None, description="Number of followers", title="Followers Count"+ )+ following_count: Optional[int] = Field(+ None, description="Number of users being followed", title="Following Count"+ )+ posts_count: Optional[int] = Field(+ None, description="Number of posts made by the user", title="Posts Count"+ )diff --git a/flowsint-types/src/flowsint_types/ssl_certificate.py b/flowsint-types/src/flowsint_types/ssl_certificate.pyindex 5f3ccabb..3e33c652 100644--- a/flowsint-types/src/flowsint_types/ssl_certificate.py+++ b/flowsint-types/src/flowsint_types/ssl_certificate.py@@ -1,25 +1,71 @@from pydantic import BaseModel, Fieldfrom typing import Optional, List+class SSLCertificate(BaseModel):"""Represents an SSL/TLS certificate with validation and security details."""- subject: str = Field(..., description="Certificate subject (domain name)", title="Subject")- issuer: Optional[str] = Field(None, description="Certificate authority that issued the certificate", title="Issuer")- serial_number: Optional[str] = Field(None, description="Certificate serial number", title="Serial Number")- valid_from: Optional[str] = Field(None, description="Certificate validity start date", title="Valid From")- valid_until: Optional[str] = Field(None, description="Certificate validity end date", title="Valid Until")- signature_algorithm: Optional[str] = Field(None, description="Signature algorithm used", title="Signature Algorithm")- key_size: Optional[int] = Field(None, description="Key size in bits", title="Key Size")- key_type: Optional[str] = Field(None, description="Type of key (RSA, ECDSA, etc.)", title="Key Type")- san_domains: Optional[List[str]] = Field(None, description="Subject Alternative Names", title="SAN Domains")- is_valid: Optional[bool] = Field(None, description="Whether certificate is currently valid", title="Is Valid")- is_expired: Optional[bool] = Field(None, description="Whether certificate has expired", title="Is Expired")- is_self_signed: Optional[bool] = Field(None, description="Whether certificate is self-signed", title="Is Self Signed")- is_wildcard: Optional[bool] = Field(None, description="Whether certificate is a wildcard certificate", title="Is Wildcard")- ocsp_status: Optional[str] = Field(None, description="OCSP revocation status", title="OCSP Status")- crl_status: Optional[str] = Field(None, description="Certificate Revocation List status", title="CRL Status")- first_seen: Optional[str] = Field(None, description="First time certificate was observed", title="First Seen")- last_seen: Optional[str] = Field(None, description="Last time certificate was observed", title="Last Seen")- source: Optional[str] = Field(None, description="Source of certificate information", title="Source")- fingerprint_sha1: Optional[str] = Field(None, description="SHA1 fingerprint", title="SHA1 Fingerprint")- fingerprint_sha256: Optional[str] = Field(None, description="SHA256 fingerprint", title="SHA256 Fingerprint")\ No newline at end of file++ subject: str = Field(+ ..., description="Certificate subject (domain name)", title="Subject"+ )+ issuer: Optional[str] = Field(+ None,+ description="Certificate authority that issued the certificate",+ title="Issuer",+ )+ serial_number: Optional[str] = Field(+ None, description="Certificate serial number", title="Serial Number"+ )+ valid_from: Optional[str] = Field(+ None, description="Certificate validity start date", title="Valid From"+ )+ valid_until: Optional[str] = Field(+ None, description="Certificate validity end date", title="Valid Until"+ )+ signature_algorithm: Optional[str] = Field(+ None, description="Signature algorithm used", title="Signature Algorithm"+ )+ key_size: Optional[int] = Field(+ None, description="Key size in bits", title="Key Size"+ )+ key_type: Optional[str] = Field(+ None, description="Type of key (RSA, ECDSA, etc.)", title="Key Type"+ )+ san_domains: Optional[List[str]] = Field(+ None, description="Subject Alternative Names", title="SAN Domains"+ )+ is_valid: Optional[bool] = Field(+ None, description="Whether certificate is currently valid", title="Is Valid"+ )+ is_expired: Optional[bool] = Field(+ None, description="Whether certificate has expired", title="Is Expired"+ )+ is_self_signed: Optional[bool] = Field(+ None, description="Whether certificate is self-signed", title="Is Self Signed"+ )+ is_wildcard: Optional[bool] = Field(+ None,+ description="Whether certificate is a wildcard certificate",+ title="Is Wildcard",+ )+ ocsp_status: Optional[str] = Field(+ None, description="OCSP revocation status", title="OCSP Status"+ )+ crl_status: Optional[str] = Field(+ None, description="Certificate Revocation List status", title="CRL Status"+ )+ first_seen: Optional[str] = Field(+ None, description="First time certificate was observed", title="First Seen"+ )+ last_seen: Optional[str] = Field(+ None, description="Last time certificate was observed", title="Last Seen"+ )+ source: Optional[str] = Field(+ None, description="Source of certificate information", title="Source"+ )+ fingerprint_sha1: Optional[str] = Field(+ None, description="SHA1 fingerprint", title="SHA1 Fingerprint"+ )+ fingerprint_sha256: Optional[str] = Field(+ None, description="SHA256 fingerprint", title="SHA256 Fingerprint"+ )diff --git a/flowsint-types/src/flowsint_types/username.py b/flowsint-types/src/flowsint_types/username.pyindex 7c73bf39..76389513 100644--- a/flowsint-types/src/flowsint_types/username.py+++ b/flowsint-types/src/flowsint_types/username.py@@ -1,22 +1,54 @@from pydantic import BaseModel, Fieldfrom typing import Optional, List+class Username(BaseModel):"""Represents a username or handle on a platform with associated profile information."""+username: str = Field(..., description="Username or handle", title="Username")- platform: Optional[str] = Field(None, description="Platform or service where username is used", title="Platform")- display_name: Optional[str] = Field(None, description="Display name associated with username", title="Display Name")- created_at: Optional[str] = Field(None, description="Account creation date", title="Creation Date")- last_seen: Optional[str] = Field(None, description="Last activity timestamp", title="Last Seen")- followers_count: Optional[int] = Field(None, description="Number of followers", title="Followers Count")- following_count: Optional[int] = Field(None, description="Number of accounts being followed", title="Following Count")- posts_count: Optional[int] = Field(None, description="Number of posts or content items", title="Posts Count")- verified: Optional[bool] = Field(None, description="Whether account is verified", title="Verified")- bio: Optional[str] = Field(None, description="User biography or description", title="Bio")+ platform: Optional[str] = Field(+ None, description="Platform or service where username is used", title="Platform"+ )+ display_name: Optional[str] = Field(+ None, description="Display name associated with username", title="Display Name"+ )+ created_at: Optional[str] = Field(+ None, description="Account creation date", title="Creation Date"+ )+ last_seen: Optional[str] = Field(+ None, description="Last activity timestamp", title="Last Seen"+ )+ followers_count: Optional[int] = Field(+ None, description="Number of followers", title="Followers Count"+ )+ following_count: Optional[int] = Field(+ None, description="Number of accounts being followed", title="Following Count"+ )+ posts_count: Optional[int] = Field(+ None, description="Number of posts or content items", title="Posts Count"+ )+ verified: Optional[bool] = Field(+ None, description="Whether account is verified", title="Verified"+ )+ bio: Optional[str] = Field(+ None, description="User biography or description", title="Bio"+ )location: Optional[str] = Field(None, description="User location", title="Location")- website: Optional[str] = Field(None, description="User website URL", title="Website")- avatar_url: Optional[str] = Field(None, description="Profile picture URL", title="Avatar URL")- is_private: Optional[bool] = Field(None, description="Whether account is private", title="Is Private")- is_suspended: Optional[bool] = Field(None, description="Whether account is suspended", title="Is Suspended")- associated_emails: Optional[List[str]] = Field(None, description="Associated email addresses", title="Associated Emails")- associated_phones: Optional[List[str]] = Field(None, description="Associated phone numbers", title="Associated Phones")\ No newline at end of file+ website: Optional[str] = Field(+ None, description="User website URL", title="Website"+ )+ avatar_url: Optional[str] = Field(+ None, description="Profile picture URL", title="Avatar URL"+ )+ is_private: Optional[bool] = Field(+ None, description="Whether account is private", title="Is Private"+ )+ is_suspended: Optional[bool] = Field(+ None, description="Whether account is suspended", title="Is Suspended"+ )+ associated_emails: Optional[List[str]] = Field(+ None, description="Associated email addresses", title="Associated Emails"+ )+ associated_phones: Optional[List[str]] = Field(+ None, description="Associated phone numbers", title="Associated Phones"+ )diff --git a/flowsint-types/src/flowsint_types/wallet.py b/flowsint-types/src/flowsint_types/wallet.pyindex 09f1f3fc..6fd47b66 100644--- a/flowsint-types/src/flowsint_types/wallet.py+++ b/flowsint-types/src/flowsint_types/wallet.py@@ -1,54 +1,135 @@from typing import Optionalfrom pydantic import BaseModel, Field, HttpUrl+class CryptoWallet(BaseModel):"""Represents a cryptocurrency wallet."""+address: str = Field(..., description="Wallet address", title="Wallet Address")- node_id: Optional[str] = Field(None, description="Wallet Explorer node ID", title="Node ID")+ node_id: Optional[str] = Field(+ None, description="Wallet Explorer node ID", title="Node ID"+ )+class CryptoWalletTransaction(BaseModel):"""Represents a cryptocurrency transaction."""- source: CryptoWallet = Field(..., description="Source wallet", title="Source Wallet")- target: Optional[CryptoWallet] = Field(None, description="Target wallet", title="Target Wallet")- hash: Optional[str] = Field(None, description="Transaction hash", title="Transaction Hash")- value: Optional[float] = Field(None, description="Transaction value in cryptocurrency", title="Value")- amount: Optional[float] = Field(None, description="Transaction amount in cryptocurrency", title="Amount")- amount_usd: Optional[float] = Field(None, description="Transaction amount in USD", title="Amount USD")++ source: CryptoWallet = Field(+ ..., description="Source wallet", title="Source Wallet"+ )+ target: Optional[CryptoWallet] = Field(+ None, description="Target wallet", title="Target Wallet"+ )+ hash: Optional[str] = Field(+ None, description="Transaction hash", title="Transaction Hash"+ )+ value: Optional[float] = Field(+ None, description="Transaction value in cryptocurrency", title="Value"+ )+ amount: Optional[float] = Field(+ None, description="Transaction amount in cryptocurrency", title="Amount"+ )+ amount_usd: Optional[float] = Field(+ None, description="Transaction amount in USD", title="Amount USD"+ )date: Optional[str] = Field(None, description="Transaction date", title="Date")- hop: Optional[int] = Field(None, description="Hop distance from original wallet", title="Hop Distance")- timestamp: Optional[str] = Field(None, description="Transaction timestamp (unix epoch)", title="Timestamp")- block_number: Optional[int] = Field(None, description="Block number", title="Block Number")- block_hash: Optional[str] = Field(None, description="Block hash", title="Block Hash")+ hop: Optional[int] = Field(+ None, description="Hop distance from original wallet", title="Hop Distance"+ )+ timestamp: Optional[str] = Field(+ None, description="Transaction timestamp (unix epoch)", title="Timestamp"+ )+ block_number: Optional[int] = Field(+ None, description="Block number", title="Block Number"+ )+ block_hash: Optional[str] = Field(+ None, description="Block hash", title="Block Hash"+ )nonce: Optional[int] = Field(None, description="Transaction nonce", title="Nonce")- transaction_index: Optional[int] = Field(None, description="Transaction index in block", title="Transaction Index")+ transaction_index: Optional[int] = Field(+ None, description="Transaction index in block", title="Transaction Index"+ )gas: Optional[int] = Field(None, description="Gas provided", title="Gas")- gas_price: Optional[int] = Field(None, description="Gas price in wei", title="Gas Price")+ gas_price: Optional[int] = Field(+ None, description="Gas price in wei", title="Gas Price"+ )gas_used: Optional[int] = Field(None, description="Gas used", title="Gas Used")- cumulative_gas_used: Optional[int] = Field(None, description="Cumulative gas used", title="Cumulative Gas Used")+ cumulative_gas_used: Optional[int] = Field(+ None, description="Cumulative gas used", title="Cumulative Gas Used"+ )input: Optional[str] = Field(None, description="Input data", title="Input Data")- contract_address: Optional[str] = Field(None, description="Contract address", title="Contract Address")+ contract_address: Optional[str] = Field(+ None, description="Contract address", title="Contract Address"+ )method_id: Optional[str] = Field(None, description="Method ID", title="Method ID")- function_name: Optional[str] = Field(None, description="Function name", title="Function Name")- confirmations: Optional[int] = Field(None, description="Number of confirmations", title="Confirmations")- is_error: Optional[bool] = Field(None, description="Whether the transaction resulted in an error", title="Is Error")- txreceipt_status: Optional[str] = Field(None, description="Transaction receipt status", title="Transaction Receipt Status")- error_message: Optional[str] = Field(None, description="Error message if transaction failed", title="Error Message")+ function_name: Optional[str] = Field(+ None, description="Function name", title="Function Name"+ )+ confirmations: Optional[int] = Field(+ None, description="Number of confirmations", title="Confirmations"+ )+ is_error: Optional[bool] = Field(+ None,+ description="Whether the transaction resulted in an error",+ title="Is Error",+ )+ txreceipt_status: Optional[str] = Field(+ None,+ description="Transaction receipt status",+ title="Transaction Receipt Status",+ )+ error_message: Optional[str] = Field(+ None, description="Error message if transaction failed", title="Error Message"+ )class CryptoNFT(BaseModel):"""Represents a Non-Fungible Token (NFT) held or minted by a wallet."""+wallet: CryptoWallet = Field(..., description="Source wallet", title="Wallet")- contract_address: str = Field(..., description="Address of the NFT smart contract (ERC-721/1155)", title="Contract Address")- token_id: str = Field(..., description="Unique token ID of the NFT within the contract", title="Token ID")- collection_name: Optional[str] = Field(None, description="Name of the NFT collection", title="Collection Name")- metadata_url: Optional[HttpUrl] = Field(None, description="URL to the metadata JSON or IPFS resource", title="Metadata URL")- image_url: Optional[HttpUrl] = Field(None, description="URL to the image or media representing the NFT", title="Image URL")- name: Optional[str] = Field(None, description="Name or title of the NFT", title="NFT Name")- description: Optional[str] = Field(None, description="Text description of the NFT", title="Description")- owner_address: Optional[str] = Field(None, description="Current owner of the NFT", title="Owner Address")- creator_address: Optional[str] = Field(None, description="Original minter or creator address", title="Creator Address")- last_transfer_date: Optional[str] = Field(None, description="Date of last transfer or update (ISO format)", title="Last Transfer Date")- node_id: Optional[str] = Field(None, description="NFT node ID in the Explorer graph", title="Node ID")+ contract_address: str = Field(+ ...,+ description="Address of the NFT smart contract (ERC-721/1155)",+ title="Contract Address",+ )+ token_id: str = Field(+ ...,+ description="Unique token ID of the NFT within the contract",+ title="Token ID",+ )+ collection_name: Optional[str] = Field(+ None, description="Name of the NFT collection", title="Collection Name"+ )+ metadata_url: Optional[HttpUrl] = Field(+ None,+ description="URL to the metadata JSON or IPFS resource",+ title="Metadata URL",+ )+ image_url: Optional[HttpUrl] = Field(+ None,+ description="URL to the image or media representing the NFT",+ title="Image URL",+ )+ name: Optional[str] = Field(+ None, description="Name or title of the NFT", title="NFT Name"+ )+ description: Optional[str] = Field(+ None, description="Text description of the NFT", title="Description"+ )+ owner_address: Optional[str] = Field(+ None, description="Current owner of the NFT", title="Owner Address"+ )+ creator_address: Optional[str] = Field(+ None, description="Original minter or creator address", title="Creator Address"+ )+ last_transfer_date: Optional[str] = Field(+ None,+ description="Date of last transfer or update (ISO format)",+ title="Last Transfer Date",+ )+ node_id: Optional[str] = Field(+ None, description="NFT node ID in the Explorer graph", title="Node ID"+ )@propertydef uid(self):@@ -58,4 +139,4 @@ class CryptoNFT(BaseModel):# Update forward referencesCryptoWallet.model_rebuild()CryptoWalletTransaction.model_rebuild()-CryptoNFT.model_rebuild()\ No newline at end of file+CryptoNFT.model_rebuild()diff --git a/flowsint-types/src/flowsint_types/weapon.py b/flowsint-types/src/flowsint_types/weapon.pyindex 417ce6b3..ee0f4249 100644--- a/flowsint-types/src/flowsint_types/weapon.py+++ b/flowsint-types/src/flowsint_types/weapon.py@@ -1,35 +1,114 @@from pydantic import BaseModel, Fieldfrom typing import Literal, Optional, List+class Weapon(BaseModel):"""Represents a weapon with detailed specifications and forensic information."""+name: str = Field(..., description="Weapon name or identifier", title="Name")- type: Optional[Literal["firearm", "melee", "explosive", "bladed", "blunt", "chemical", "biological", "radiological", "nuclear", "other"]] = Field(None, description="Weapon category (handgun, rifle, shotgun, knife, etc.)", title="Category")- description: Optional[str] = Field(None, description="Weapon description", title="Description")- manufacturer: Optional[str] = Field(None, description="Weapon manufacturer", title="Manufacturer")+ type: Optional[+ Literal[+ "firearm",+ "melee",+ "explosive",+ "bladed",+ "blunt",+ "chemical",+ "biological",+ "radiological",+ "nuclear",+ "other",+ ]+ ] = Field(+ None,+ description="Weapon category (handgun, rifle, shotgun, knife, etc.)",+ title="Category",+ )+ description: Optional[str] = Field(+ None, description="Weapon description", title="Description"+ )+ manufacturer: Optional[str] = Field(+ None, description="Weapon manufacturer", title="Manufacturer"+ )model: Optional[str] = Field(None, description="Weapon model", title="Model")- caliber: Optional[str] = Field(None, description="Caliber or ammunition type", title="Caliber")- serial_number: Optional[str] = Field(None, description="Weapon serial number", title="Serial Number")- year_manufactured: Optional[int] = Field(None, description="Year weapon was manufactured", title="Year Manufactured")- country_of_origin: Optional[str] = Field(None, description="Country where weapon was manufactured", title="Country of Origin")- legal_status: Optional[str] = Field(None, description="Legal status (legal, illegal, restricted, etc.)", title="Legal Status")- registration_status: Optional[str] = Field(None, description="Registration status", title="Registration Status")- condition: Optional[str] = Field(None, description="Physical condition of the weapon", title="Condition")- modifications: Optional[List[str]] = Field(None, description="Any modifications made to the weapon", title="Modifications")- accessories: Optional[List[str]] = Field(None, description="Attached accessories", title="Accessories")- capacity: Optional[str] = Field(None, description="Magazine capacity or ammunition capacity", title="Capacity")- barrel_length: Optional[str] = Field(None, description="Barrel length", title="Barrel Length")- overall_length: Optional[str] = Field(None, description="Overall weapon length", title="Overall Length")+ caliber: Optional[str] = Field(+ None, description="Caliber or ammunition type", title="Caliber"+ )+ serial_number: Optional[str] = Field(+ None, description="Weapon serial number", title="Serial Number"+ )+ year_manufactured: Optional[int] = Field(+ None, description="Year weapon was manufactured", title="Year Manufactured"+ )+ country_of_origin: Optional[str] = Field(+ None,+ description="Country where weapon was manufactured",+ title="Country of Origin",+ )+ legal_status: Optional[str] = Field(+ None,+ description="Legal status (legal, illegal, restricted, etc.)",+ title="Legal Status",+ )+ registration_status: Optional[str] = Field(+ None, description="Registration status", title="Registration Status"+ )+ condition: Optional[str] = Field(+ None, description="Physical condition of the weapon", title="Condition"+ )+ modifications: Optional[List[str]] = Field(+ None, description="Any modifications made to the weapon", title="Modifications"+ )+ accessories: Optional[List[str]] = Field(+ None, description="Attached accessories", title="Accessories"+ )+ capacity: Optional[str] = Field(+ None, description="Magazine capacity or ammunition capacity", title="Capacity"+ )+ barrel_length: Optional[str] = Field(+ None, description="Barrel length", title="Barrel Length"+ )+ overall_length: Optional[str] = Field(+ None, description="Overall weapon length", title="Overall Length"+ )weight: Optional[str] = Field(None, description="Weapon weight", title="Weight")- action_type: Optional[str] = Field(None, description="Action type (semi-automatic, bolt-action, etc.)", title="Action Type")- firing_mode: Optional[str] = Field(None, description="Firing mode (single shot, burst, full auto, etc.)", title="Firing Mode")- safety_features: Optional[List[str]] = Field(None, description="Safety features present", title="Safety Features")- source: Optional[str] = Field(None, description="Source of weapon information", title="Source")- location_found: Optional[str] = Field(None, description="Location where weapon was found", title="Location Found")- date_found: Optional[str] = Field(None, description="Date weapon was found", title="Date Found")- evidence_number: Optional[str] = Field(None, description="Evidence number or case reference", title="Evidence Number")- owner: Optional[str] = Field(None, description="Registered owner or possessor", title="Owner")- purchase_date: Optional[str] = Field(None, description="Date weapon was purchased", title="Purchase Date")- purchase_location: Optional[str] = Field(None, description="Location where weapon was purchased", title="Purchase Location")+ action_type: Optional[str] = Field(+ None,+ description="Action type (semi-automatic, bolt-action, etc.)",+ title="Action Type",+ )+ firing_mode: Optional[str] = Field(+ None,+ description="Firing mode (single shot, burst, full auto, etc.)",+ title="Firing Mode",+ )+ safety_features: Optional[List[str]] = Field(+ None, description="Safety features present", title="Safety Features"+ )+ source: Optional[str] = Field(+ None, description="Source of weapon information", title="Source"+ )+ location_found: Optional[str] = Field(+ None, description="Location where weapon was found", title="Location Found"+ )+ date_found: Optional[str] = Field(+ None, description="Date weapon was found", title="Date Found"+ )+ evidence_number: Optional[str] = Field(+ None, description="Evidence number or case reference", title="Evidence Number"+ )+ owner: Optional[str] = Field(+ None, description="Registered owner or possessor", title="Owner"+ )+ purchase_date: Optional[str] = Field(+ None, description="Date weapon was purchased", title="Purchase Date"+ )+ purchase_location: Optional[str] = Field(+ None,+ description="Location where weapon was purchased",+ title="Purchase Location",+ )price: Optional[float] = Field(None, description="Purchase price", title="Price")- notes: Optional[str] = Field(None, description="Additional notes or observations", title="Notes")\ No newline at end of file+ notes: Optional[str] = Field(+ None, description="Additional notes or observations", title="Notes"+ )diff --git a/flowsint-types/src/flowsint_types/web_tracker.py b/flowsint-types/src/flowsint_types/web_tracker.pyindex 09f9386b..36ba0450 100644--- a/flowsint-types/src/flowsint_types/web_tracker.py+++ b/flowsint-types/src/flowsint_types/web_tracker.py@@ -1,23 +1,59 @@from pydantic import BaseModel, Fieldfrom typing import Optional, List+class WebTracker(BaseModel):"""Represents a web tracking technology with privacy and compliance information."""- tracker_id: str = Field(..., description="Unique tracker identifier", title="Tracker ID")++ tracker_id: str = Field(+ ..., description="Unique tracker identifier", title="Tracker ID"+ )name: Optional[str] = Field(None, description="Tracker name", title="Name")- type: Optional[str] = Field(None, description="Type of tracker (analytics, advertising, etc.)", title="Type")- domain: Optional[str] = Field(None, description="Domain where tracker is deployed", title="Domain")- script_url: Optional[str] = Field(None, description="URL of tracking script", title="Script URL")- company: Optional[str] = Field(None, description="Company providing the tracker", title="Company")- purpose: Optional[str] = Field(None, description="Purpose of tracking", title="Purpose")- data_collected: Optional[List[str]] = Field(None, description="Types of data collected", title="Data Collected")- privacy_policy: Optional[str] = Field(None, description="Privacy policy URL", title="Privacy Policy")- opt_out_url: Optional[str] = Field(None, description="Opt-out URL", title="Opt-out URL")- first_seen: Optional[str] = Field(None, description="First time tracker was observed", title="First Seen")- last_seen: Optional[str] = Field(None, description="Last time tracker was observed", title="Last Seen")- is_active: Optional[bool] = Field(None, description="Whether tracker is currently active", title="Is Active")- is_third_party: Optional[bool] = Field(None, description="Whether tracker is third-party", title="Is Third Party")- cookie_duration: Optional[int] = Field(None, description="Cookie duration in days", title="Cookie Duration")- gdpr_compliant: Optional[bool] = Field(None, description="Whether tracker is GDPR compliant", title="GDPR Compliant")- source: Optional[str] = Field(None, description="Source of tracker information", title="Source")- risk_level: Optional[str] = Field(None, description="Privacy risk level", title="Risk Level")\ No newline at end of file+ type: Optional[str] = Field(+ None, description="Type of tracker (analytics, advertising, etc.)", title="Type"+ )+ domain: Optional[str] = Field(+ None, description="Domain where tracker is deployed", title="Domain"+ )+ script_url: Optional[str] = Field(+ None, description="URL of tracking script", title="Script URL"+ )+ company: Optional[str] = Field(+ None, description="Company providing the tracker", title="Company"+ )+ purpose: Optional[str] = Field(+ None, description="Purpose of tracking", title="Purpose"+ )+ data_collected: Optional[List[str]] = Field(+ None, description="Types of data collected", title="Data Collected"+ )+ privacy_policy: Optional[str] = Field(+ None, description="Privacy policy URL", title="Privacy Policy"+ )+ opt_out_url: Optional[str] = Field(+ None, description="Opt-out URL", title="Opt-out URL"+ )+ first_seen: Optional[str] = Field(+ None, description="First time tracker was observed", title="First Seen"+ )+ last_seen: Optional[str] = Field(+ None, description="Last time tracker was observed", title="Last Seen"+ )+ is_active: Optional[bool] = Field(+ None, description="Whether tracker is currently active", title="Is Active"+ )+ is_third_party: Optional[bool] = Field(+ None, description="Whether tracker is third-party", title="Is Third Party"+ )+ cookie_duration: Optional[int] = Field(+ None, description="Cookie duration in days", title="Cookie Duration"+ )+ gdpr_compliant: Optional[bool] = Field(+ None, description="Whether tracker is GDPR compliant", title="GDPR Compliant"+ )+ source: Optional[str] = Field(+ None, description="Source of tracker information", title="Source"+ )+ risk_level: Optional[str] = Field(+ None, description="Privacy risk level", title="Risk Level"+ )diff --git a/flowsint-types/src/flowsint_types/website.py b/flowsint-types/src/flowsint_types/website.pyindex 15868c02..3e986360 100644--- a/flowsint-types/src/flowsint_types/website.py+++ b/flowsint-types/src/flowsint_types/website.py@@ -2,9 +2,19 @@ from typing import List, Optionalfrom pydantic import BaseModel, Field, HttpUrlfrom .domain import Domain+class Website(BaseModel):"""Represents a website with its URL, domain, and redirect information."""- url: HttpUrl = Field(..., description="Full URL of the website", title="Website URL")- redirects: Optional[List[HttpUrl]] = Field([], description="List of redirects from the website", title="Redirects")- domain: Optional[Domain] = Field(None, description="Domain information for the website", title="Domain")- active: Optional[bool] = Field(False, description="Whether the website is active", title="Is Active")\ No newline at end of file++ url: HttpUrl = Field(+ ..., description="Full URL of the website", title="Website URL"+ )+ redirects: Optional[List[HttpUrl]] = Field(+ [], description="List of redirects from the website", title="Redirects"+ )+ domain: Optional[Domain] = Field(+ None, description="Domain information for the website", title="Domain"+ )+ active: Optional[bool] = Field(+ False, description="Whether the website is active", title="Is Active"+ )diff --git a/flowsint-types/src/flowsint_types/whois.py b/flowsint-types/src/flowsint_types/whois.pyindex cab163d3..f19e6b56 100644--- a/flowsint-types/src/flowsint_types/whois.py+++ b/flowsint-types/src/flowsint_types/whois.py@@ -2,13 +2,31 @@ from typing import Optionalfrom pydantic import BaseModel, Fieldfrom .email import Email+class Whois(BaseModel):"""Represents WHOIS domain registration information."""+domain: str = Field(..., description="Domain name", title="Domain")- registrar: Optional[str] = Field(None, description="Domain registrar name", title="Registrar")- org: Optional[str] = Field(None, description="Organization name associated with the domain", title="Organization")- city: Optional[str] = Field(None, description="City where the domain is registered", title="City")- country: Optional[str] = Field(None, description="Country where the domain is registered", title="Country")- email: Optional[Email] = Field(None, description="Contact email for the domain", title="Contact Email")- creation_date: Optional[str] = Field(None, description="Date when the domain was created", title="Creation Date")- expiration_date: Optional[str] = Field(None, description="Date when the domain expires", title="Expiration Date")+ registrar: Optional[str] = Field(+ None, description="Domain registrar name", title="Registrar"+ )+ org: Optional[str] = Field(+ None,+ description="Organization name associated with the domain",+ title="Organization",+ )+ city: Optional[str] = Field(+ None, description="City where the domain is registered", title="City"+ )+ country: Optional[str] = Field(+ None, description="Country where the domain is registered", title="Country"+ )+ email: Optional[Email] = Field(+ None, description="Contact email for the domain", title="Contact Email"+ )+ creation_date: Optional[str] = Field(+ None, description="Date when the domain was created", title="Creation Date"+ )+ expiration_date: Optional[str] = Field(+ None, description="Date when the domain expires", title="Expiration Date"+ )