A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat(api): update logic
dextmorgn8b3663e2bbc7
diff --git a/flowsint-api/app/api/routes/custom_types.py b/flowsint-api/app/api/routes/custom_types.pynew file mode 100644index 00000000..612920f0--- /dev/null+++ b/flowsint-api/app/api/routes/custom_types.py@@ -0,0 +1,263 @@+"""API routes for custom types management."""+from uuid import UUID+from typing import List+from fastapi import APIRouter, HTTPException, Depends, status+from sqlalchemy.orm import Session+from flowsint_core.core.postgre_db import get_db+from flowsint_core.core.models import CustomType, Profile+from app.api.deps import get_current_user+from app.api.schemas.custom_type import (+ CustomTypeCreate,+ CustomTypeUpdate,+ CustomTypeRead,+ CustomTypeValidatePayload,+ CustomTypeValidateResponse,+)+from app.utils.custom_types import (+ validate_json_schema,+ validate_payload_against_schema,+ calculate_schema_checksum,+)+++router = APIRouter()++[email protected]("", response_model=CustomTypeRead, status_code=status.HTTP_201_CREATED)+def create_custom_type(+ custom_type: CustomTypeCreate,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):+ """+ Create a new custom type.++ Validates the JSON Schema and stores it in the database.+ """+ # Validate the JSON Schema+ validate_json_schema(custom_type.json_schema)++ # Calculate checksum+ checksum = calculate_schema_checksum(custom_type.json_schema)++ # Check for duplicate name for this user+ existing = (+ db.query(CustomType)+ .filter(+ CustomType.owner_id == current_user.id,+ CustomType.name == custom_type.name+ )+ .first()+ )+ if existing:+ raise HTTPException(+ status_code=400,+ detail=f"Custom type with name '{custom_type.name}' already exists"+ )++ # Create the custom type+ db_custom_type = CustomType(+ name=custom_type.name,+ owner_id=current_user.id,+ schema=custom_type.json_schema,+ description=custom_type.description,+ status=custom_type.status,+ checksum=checksum,+ )++ db.add(db_custom_type)+ db.commit()+ db.refresh(db_custom_type)++ return db_custom_type++[email protected]("", response_model=List[CustomTypeRead])+def list_custom_types(+ status: str = None,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):+ """+ List all custom types for the current user.++ Can be filtered by status (draft, published, archived).+ """+ query = db.query(CustomType).filter(CustomType.owner_id == current_user.id)++ if status:+ if status not in ["draft", "published", "archived"]:+ raise HTTPException(+ status_code=400,+ detail="Status must be one of: draft, published, archived"+ )+ query = query.filter(CustomType.status == status)++ custom_types = query.order_by(CustomType.created_at.desc()).all()+ return custom_types++[email protected]("/{id}", response_model=CustomTypeRead)+def get_custom_type(+ id: UUID,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):+ """Get a specific custom type by ID."""+ custom_type = (+ db.query(CustomType)+ .filter(CustomType.id == id, CustomType.owner_id == current_user.id)+ .first()+ )++ if not custom_type:+ raise HTTPException(+ status_code=404,+ detail="Custom type not found"+ )++ return custom_type++[email protected]("/{id}/schema")+def get_custom_type_schema(+ id: UUID,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):+ """Get the raw JSON Schema for a custom type."""+ custom_type = (+ db.query(CustomType)+ .filter(CustomType.id == id, CustomType.owner_id == current_user.id)+ .first()+ )++ if not custom_type:+ raise HTTPException(+ status_code=404,+ detail="Custom type not found"+ )++ return custom_type.schema++[email protected]("/{id}", response_model=CustomTypeRead)+def update_custom_type(+ id: UUID,+ update_data: CustomTypeUpdate,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):+ """+ Update a custom type.++ If the schema is changed, a new checksum is calculated.+ """+ custom_type = (+ db.query(CustomType)+ .filter(CustomType.id == id, CustomType.owner_id == current_user.id)+ .first()+ )++ if not custom_type:+ raise HTTPException(+ status_code=404,+ detail="Custom type not found"+ )++ # Update fields+ if update_data.name is not None:+ # Check for duplicate name+ existing = (+ db.query(CustomType)+ .filter(+ CustomType.owner_id == current_user.id,+ CustomType.name == update_data.name,+ CustomType.id != id+ )+ .first()+ )+ if existing:+ raise HTTPException(+ status_code=400,+ detail=f"Custom type with name '{update_data.name}' already exists"+ )+ custom_type.name = update_data.name++ if update_data.json_schema is not None:+ # Validate the new schema+ validate_json_schema(update_data.json_schema)+ custom_type.schema = update_data.json_schema+ custom_type.checksum = calculate_schema_checksum(update_data.json_schema)++ if update_data.description is not None:+ custom_type.description = update_data.description++ if update_data.status is not None:+ custom_type.status = update_data.status++ db.commit()+ db.refresh(custom_type)++ return custom_type++[email protected]("/{id}", status_code=status.HTTP_204_NO_CONTENT)+def delete_custom_type(+ id: UUID,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):+ """Delete a custom type."""+ custom_type = (+ db.query(CustomType)+ .filter(CustomType.id == id, CustomType.owner_id == current_user.id)+ .first()+ )++ if not custom_type:+ raise HTTPException(+ status_code=404,+ detail="Custom type not found"+ )++ db.delete(custom_type)+ db.commit()++ return None++[email protected]("/{id}/validate", response_model=CustomTypeValidateResponse)+def validate_payload(+ id: UUID,+ payload_data: CustomTypeValidatePayload,+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),+):+ """+ Validate a payload against a custom type's schema.++ This is useful for testing before publishing a type.+ """+ custom_type = (+ db.query(CustomType)+ .filter(CustomType.id == id, CustomType.owner_id == current_user.id)+ .first()+ )++ if not custom_type:+ raise HTTPException(+ status_code=404,+ detail="Custom type not found"+ )++ # Validate the payload against the schema+ is_valid, errors = validate_payload_against_schema(+ payload_data.payload,+ custom_type.schema+ )++ return CustomTypeValidateResponse(+ valid=is_valid,+ errors=errors if not is_valid else None+ )diff --git a/flowsint-api/app/api/routes/flows.py b/flowsint-api/app/api/routes/flows.pyindex a6bd3a16..b19940c6 100644--- a/flowsint-api/app/api/routes/flows.py+++ b/flowsint-api/app/api/routes/flows.py@@ -10,8 +10,9 @@ from flowsint_types import Domain, Phrase, Ip, SocialProfile, Organization, Emaifrom flowsint_core.core.types import Node, Edge, FlowStep, FlowBranchfrom sqlalchemy.orm import Sessionfrom flowsint_core.core.postgre_db import get_db-from flowsint_core.core.models import Flow, Profile+from flowsint_core.core.models import Flow, Profile, CustomTypefrom app.api.deps import get_current_user+from sqlalchemy import funcfrom app.api.schemas.flow import FlowRead, FlowCreate, FlowUpdatefrom flowsint_types import (ASN,@@ -48,25 +49,40 @@ class launchFlowPayload(BaseModel):router = APIRouter()-# Get the list of all flows@router.get("/", response_model=List[FlowRead])def get_flows(category: Optional[str] = Query(None),db: Session = Depends(get_db),current_user: Profile = Depends(get_current_user),):- query = db.query(Flow)-- if category is not None and category != "undefined":- # Case-insensitive filtering by checking if any category matches (case-insensitive)- flows = query.all()+ if not category or category.lower() == "undefined":+ return db.query(Flow).order_by(Flow.last_updated_at.desc()).all()++ custom_type = (+ db.query(CustomType)+ .filter(+ CustomType.owner_id == current_user.id,+ CustomType.status == "published",+ func.lower(CustomType.name) == category.lower(),+ )+ .first()+ )+ if custom_type:+ flows = db.query(Flow).order_by(Flow.last_updated_at.desc()).all()return [- flow+ {+ **(flow.to_dict() if hasattr(flow, "to_dict") else flow.__dict__),+ "wobblyType": True,+ }for flow in flows- if any(cat.lower() == category.lower() for cat in flow.category)]-- return query.order_by(Flow.last_updated_at.desc()).all()++ flows = db.query(Flow).order_by(Flow.last_updated_at.desc()).all()+ return [+ flow+ for flow in flows+ if any(cat.lower() == category.lower() for cat in flow.category)+ ]# Returns the "raw_materials" for the flow editordiff --git a/flowsint-api/app/api/routes/transforms.py b/flowsint-api/app/api/routes/transforms.pyindex cb058620..9bc2ea10 100644--- a/flowsint-api/app/api/routes/transforms.py+++ b/flowsint-api/app/api/routes/transforms.py@@ -4,8 +4,11 @@ from pydantic import BaseModelfrom flowsint_core.core.registry import TransformRegistryfrom flowsint_core.core.celery import celeryfrom flowsint_core.core.types import Node, Edge, FlowBranch-from flowsint_core.core.models import Profile+from flowsint_core.core.models import CustomType, Profilefrom app.api.deps import get_current_user+from flowsint_core.core.postgre_db import get_db+from sqlalchemy.orm import Session+from sqlalchemy import funcclass FlowComputationRequest(BaseModel):@@ -36,15 +39,26 @@ router = APIRouter()@router.get("/")def get_transforms(category: Optional[str] = Query(None),- # current_user: Profile = Depends(get_current_user),+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user),):- if category is not None and category != "undefined":- transforms = TransformRegistry.list_by_input_type(- category, exclude=["n8n_connector"]+ if not category or category.lower() == "undefined":+ return TransformRegistry.list(exclude=["n8n_connector"])+ # Si catégorie custom+ custom_type = (+ db.query(CustomType)+ .filter(+ CustomType.owner_id == current_user.id,+ CustomType.status == "published",+ func.lower(CustomType.name) == category.lower(),)- else:- transforms = TransformRegistry.list(exclude=["n8n_connector"])- return transforms+ .first()+ )++ if custom_type:+ return TransformRegistry.list(exclude=["n8n_connector"], wobbly_type=True)++ return TransformRegistry.list_by_input_type(category, exclude=["n8n_connector"])@router.post("/{transform_name}/launch")diff --git a/flowsint-api/app/api/routes/types.py b/flowsint-api/app/api/routes/types.pyindex 25287904..1ab2cf6e 100644--- a/flowsint-api/app/api/routes/types.py+++ b/flowsint-api/app/api/routes/types.py@@ -1,7 +1,11 @@from typing import Any, Dict, Optional, Typefrom uuid import uuid4-from fastapi import APIRouter+from fastapi import APIRouter, Dependsfrom pydantic import BaseModel, TypeAdapter+from sqlalchemy.orm import Session+from flowsint_core.core.postgre_db import get_db+from flowsint_core.core.models import CustomType, Profile+from app.api.deps import get_current_userfrom flowsint_types import (Domain,Ip,@@ -44,7 +48,10 @@ router = APIRouter()# Returns the "types" for the sketches@router.get("/")-async def get_types_list():+async def get_types_list(+ db: Session = Depends(get_db),+ current_user: Profile = Depends(get_current_user)+):types = [{"id": uuid4(),@@ -201,6 +208,58 @@ async def get_types_list():},]+ # Add custom types+ custom_types = (+ db.query(CustomType)+ .filter(+ CustomType.owner_id == current_user.id,+ CustomType.status == "published" # Only show published custom types+ )+ .all()+ )++ if custom_types:+ custom_types_children = []+ for custom_type in custom_types:+ # Extract the label_key from the schema (use first required field or first property)+ schema = custom_type.schema+ properties = schema.get("properties", {})+ required = schema.get("required", [])++ # Try to use the first required field, or the first property+ label_key = required[0] if required else list(properties.keys())[0] if properties else "value"++ custom_types_children.append({+ "id": custom_type.id,+ "type": custom_type.name,+ "key": custom_type.name.lower(),+ "label_key": label_key,+ "icon": "custom",+ "label": custom_type.name,+ "description": custom_type.description or "",+ "fields": [+ {+ "name": prop,+ "label": info.get("title", prop),+ "description": info.get("description", ""),+ "type": "text",+ "required": prop in required+ }+ for prop, info in properties.items()+ ],+ "custom": True # Mark as custom type+ })++ types.append({+ "id": uuid4(),+ "type": "custom_types_category",+ "key": "custom_types",+ "icon": "custom",+ "label": "Custom types",+ "fields": [],+ "children": custom_types_children,+ })+return typesdiff --git a/flowsint-api/app/api/schemas/custom_type.py b/flowsint-api/app/api/schemas/custom_type.pynew file mode 100644index 00000000..2a1b7d03--- /dev/null+++ b/flowsint-api/app/api/schemas/custom_type.py@@ -0,0 +1,67 @@+from typing import Optional, Dict, Any+from pydantic import BaseModel, UUID4, Field, field_validator+from datetime import datetime+from .base import ORMBase+++class CustomTypeCreate(BaseModel):+ """Schema for creating a new custom type."""+ name: str = Field(..., min_length=1, max_length=255, description="Name of the custom type")+ json_schema: Dict[str, Any] = Field(..., description="JSON Schema definition", alias="schema")+ description: Optional[str] = Field(None, description="Optional description of the custom type")+ status: Optional[str] = Field("draft", description="Status: draft or published")++ class Config:+ populate_by_name = True++ @field_validator('status')+ @classmethod+ def validate_status(cls, v: str) -> str:+ if v not in ["draft", "published", "archived"]:+ raise ValueError("Status must be one of: draft, published, archived")+ return v+++class CustomTypeUpdate(BaseModel):+ """Schema for updating an existing custom type."""+ name: Optional[str] = Field(None, min_length=1, max_length=255)+ json_schema: Optional[Dict[str, Any]] = Field(None, alias="schema")+ description: Optional[str] = None+ status: Optional[str] = None++ class Config:+ populate_by_name = True++ @field_validator('status')+ @classmethod+ def validate_status(cls, v: Optional[str]) -> Optional[str]:+ if v is not None and v not in ["draft", "published", "archived"]:+ raise ValueError("Status must be one of: draft, published, archived")+ return v+++class CustomTypeRead(ORMBase):+ """Schema for reading a custom type."""+ id: UUID4+ name: str+ owner_id: UUID4+ json_schema: Dict[str, Any] = Field(..., alias="schema")+ status: str+ checksum: Optional[str]+ description: Optional[str]+ created_at: datetime+ updated_at: datetime++ class Config:+ populate_by_name = True+++class CustomTypeValidatePayload(BaseModel):+ """Schema for validating a payload against a custom type schema."""+ payload: Dict[str, Any] = Field(..., description="Data to validate against the schema")+++class CustomTypeValidateResponse(BaseModel):+ """Response schema for validation."""+ valid: bool+ errors: Optional[list[str]] = Nonediff --git a/flowsint-api/app/main.py b/flowsint-api/app/main.pyindex 3f821e44..49f751be 100644--- a/flowsint-api/app/main.py+++ b/flowsint-api/app/main.py@@ -16,6 +16,7 @@ from app.api.routes import chatfrom app.api.routes import scanfrom app.api.routes import keysfrom app.api.routes import types+from app.api.routes import custom_typesload_dotenv()@@ -52,3 +53,4 @@ app.include_router(chat.router, prefix="/api/chats", tags=["chats"])app.include_router(scan.router, prefix="/api/scans", tags=["scans"])app.include_router(keys.router, prefix="/api/keys", tags=["keys"])app.include_router(types.router, prefix="/api/types", tags=["types"])+app.include_router(custom_types.router, prefix="/api/custom-types", tags=["custom-types"])diff --git a/flowsint-api/app/utils/__init__.py b/flowsint-api/app/utils/__init__.pynew file mode 100644index 00000000..02a66aa8--- /dev/null+++ b/flowsint-api/app/utils/__init__.py@@ -0,0 +1 @@+"""Utility modules for the API."""diff --git a/flowsint-api/app/utils/custom_types.py b/flowsint-api/app/utils/custom_types.pynew file mode 100644index 00000000..eac344a8--- /dev/null+++ b/flowsint-api/app/utils/custom_types.py@@ -0,0 +1,215 @@+"""Utilities for handling custom types: validation and dynamic model creation."""+import hashlib+import json+from typing import Any, Dict, Type+from pydantic import BaseModel, create_model, EmailStr, Field, field_validator+from jsonschema import Draft7Validator, ValidationError as JSONSchemaValidationError+from fastapi import HTTPException+++# Mapping of JSON Schema types to Python types+TYPE_MAP = {+ "string": str,+ "integer": int,+ "number": float,+ "boolean": bool,+ "object": dict,+ "array": list,+ "null": type(None),+}++# Mapping of JSON Schema formats to Pydantic types+FORMAT_MAP = {+ "email": EmailStr,+ "uri": str,+ "uuid": str,+ "date": str,+ "date-time": str,+}++# Whitelist of allowed JSON Schema types for security+ALLOWED_TYPES = {"string", "integer", "number", "boolean", "object", "array"}+++def validate_json_schema(schema: Dict[str, Any]) -> None:+ """+ Validate that a schema is a valid JSON Schema.++ Args:+ schema: The JSON Schema to validate++ Raises:+ HTTPException: If the schema is invalid+ """+ try:+ Draft7Validator.check_schema(schema)+ except JSONSchemaValidationError as e:+ raise HTTPException(+ status_code=400,+ detail=f"Invalid JSON Schema: {e.message}"+ )++ # Additional security checks+ _check_schema_security(schema)+++def _check_schema_security(schema: Dict[str, Any]) -> None:+ """+ Check schema for security issues.++ Args:+ schema: The JSON Schema to check++ Raises:+ HTTPException: If security issues are found+ """+ # Check if type is in whitelist+ schema_type = schema.get("type")+ if schema_type and schema_type not in ALLOWED_TYPES:+ raise HTTPException(+ status_code=400,+ detail=f"Type '{schema_type}' is not allowed. Allowed types: {ALLOWED_TYPES}"+ )++ # Check properties recursively+ properties = schema.get("properties", {})+ for prop_name, prop_schema in properties.items():+ if isinstance(prop_schema, dict):+ prop_type = prop_schema.get("type")+ if prop_type and prop_type not in ALLOWED_TYPES:+ raise HTTPException(+ status_code=400,+ detail=f"Type '{prop_type}' in property '{prop_name}' is not allowed"+ )+++def validate_payload_against_schema(payload: Dict[str, Any], schema: Dict[str, Any]) -> tuple[bool, list[str]]:+ """+ Validate a payload against a JSON Schema.++ Args:+ payload: The data to validate+ schema: The JSON Schema to validate against++ Returns:+ Tuple of (is_valid, error_messages)+ """+ validator = Draft7Validator(schema)+ errors = list(validator.iter_errors(payload))++ if errors:+ error_messages = [+ f"{'.'.join(str(p) for p in error.path)}: {error.message}"+ for error in errors+ ]+ return False, error_messages++ return True, []+++def calculate_schema_checksum(schema: Dict[str, Any]) -> str:+ """+ Calculate a checksum for a schema to detect changes.++ Args:+ schema: The JSON Schema++ Returns:+ SHA256 checksum of the schema+ """+ schema_str = json.dumps(schema, sort_keys=True)+ return hashlib.sha256(schema_str.encode()).hexdigest()+++def jsonschema_to_pydantic(schema: Dict[str, Any], model_name: str = "DynamicModel") -> Type[BaseModel]:+ """+ Convert a JSON Schema to a Pydantic model.++ Args:+ schema: The JSON Schema definition+ model_name: Name for the generated Pydantic model++ Returns:+ A dynamically created Pydantic model class++ Raises:+ HTTPException: If conversion fails+ """+ try:+ fields = {}+ properties = schema.get("properties", {})+ required_fields = set(schema.get("required", []))++ for field_name, field_schema in properties.items():+ python_type = _get_python_type(field_schema)++ # Determine if field is required+ if field_name in required_fields:+ default = ... # Required field+ else:+ default = field_schema.get("default", None)++ # Add field constraints if present+ field_kwargs = {}+ if "description" in field_schema:+ field_kwargs["description"] = field_schema["description"]+ if "minimum" in field_schema:+ field_kwargs["ge"] = field_schema["minimum"]+ if "maximum" in field_schema:+ field_kwargs["le"] = field_schema["maximum"]+ if "minLength" in field_schema:+ field_kwargs["min_length"] = field_schema["minLength"]+ if "maxLength" in field_schema:+ field_kwargs["max_length"] = field_schema["maxLength"]++ if field_kwargs:+ fields[field_name] = (python_type, Field(default, **field_kwargs))+ else:+ fields[field_name] = (python_type, default)++ # Create the dynamic model+ dynamic_model = create_model(model_name, **fields)+ return dynamic_model++ except Exception as e:+ raise HTTPException(+ status_code=400,+ detail=f"Failed to convert JSON Schema to Pydantic model: {str(e)}"+ )+++def _get_python_type(field_schema: Dict[str, Any]) -> Type:+ """+ Get the Python type for a JSON Schema field.++ Args:+ field_schema: The JSON Schema field definition++ Returns:+ The corresponding Python type+ """+ # Handle format first (e.g., email)+ field_format = field_schema.get("format")+ if field_format and field_format in FORMAT_MAP:+ return FORMAT_MAP[field_format]++ # Handle type+ field_type = field_schema.get("type")+ if field_type:+ if isinstance(field_type, list):+ # Handle union types (e.g., ["string", "null"])+ # For simplicity, take the first non-null type+ for t in field_type:+ if t != "null":+ return TYPE_MAP.get(t, Any)+ return Any+ return TYPE_MAP.get(field_type, Any)++ # Handle anyOf (common in Pydantic-generated schemas)+ if "anyOf" in field_schema:+ any_of = field_schema["anyOf"]+ for option in any_of:+ if option.get("type") != "null":+ return _get_python_type(option)++ return Any