Skip to content

A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.

feat(api): update logic

dextmorgn8b3663e2bbc7
diff --git a/flowsint-api/app/api/routes/custom_types.py b/flowsint-api/app/api/routes/custom_types.py
new file mode 100644
index 00000000..612920f0
--- /dev/null
+++ b/flowsint-api/app/api/routes/custom_types.py
@@ -0,0 +1,263 @@
+"""API routes for custom types management."""
+from uuid import UUID
+from typing import List
+from fastapi import APIRouter, HTTPException, Depends, status
+from sqlalchemy.orm import Session
+from flowsint_core.core.postgre_db import get_db
+from flowsint_core.core.models import CustomType, Profile
+from app.api.deps import get_current_user
+from app.api.schemas.custom_type import (
+ CustomTypeCreate,
+ CustomTypeUpdate,
+ CustomTypeRead,
+ CustomTypeValidatePayload,
+ CustomTypeValidateResponse,
+)
+from app.utils.custom_types import (
+ validate_json_schema,
+ validate_payload_against_schema,
+ calculate_schema_checksum,
+)
+
+
+router = APIRouter()
+
+
[email protected]("", response_model=CustomTypeRead, status_code=status.HTTP_201_CREATED)
+def create_custom_type(
+ custom_type: CustomTypeCreate,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
+ """
+ Create a new custom type.
+
+ Validates the JSON Schema and stores it in the database.
+ """
+ # Validate the JSON Schema
+ validate_json_schema(custom_type.json_schema)
+
+ # Calculate checksum
+ checksum = calculate_schema_checksum(custom_type.json_schema)
+
+ # Check for duplicate name for this user
+ existing = (
+ db.query(CustomType)
+ .filter(
+ CustomType.owner_id == current_user.id,
+ CustomType.name == custom_type.name
+ )
+ .first()
+ )
+ if existing:
+ raise HTTPException(
+ status_code=400,
+ detail=f"Custom type with name '{custom_type.name}' already exists"
+ )
+
+ # Create the custom type
+ db_custom_type = CustomType(
+ name=custom_type.name,
+ owner_id=current_user.id,
+ schema=custom_type.json_schema,
+ description=custom_type.description,
+ status=custom_type.status,
+ checksum=checksum,
+ )
+
+ db.add(db_custom_type)
+ db.commit()
+ db.refresh(db_custom_type)
+
+ return db_custom_type
+
+
[email protected]("", response_model=List[CustomTypeRead])
+def list_custom_types(
+ status: str = None,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
+ """
+ List all custom types for the current user.
+
+ Can be filtered by status (draft, published, archived).
+ """
+ query = db.query(CustomType).filter(CustomType.owner_id == current_user.id)
+
+ if status:
+ if status not in ["draft", "published", "archived"]:
+ raise HTTPException(
+ status_code=400,
+ detail="Status must be one of: draft, published, archived"
+ )
+ query = query.filter(CustomType.status == status)
+
+ custom_types = query.order_by(CustomType.created_at.desc()).all()
+ return custom_types
+
+
[email protected]("/{id}", response_model=CustomTypeRead)
+def get_custom_type(
+ id: UUID,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
+ """Get a specific custom type by ID."""
+ custom_type = (
+ db.query(CustomType)
+ .filter(CustomType.id == id, CustomType.owner_id == current_user.id)
+ .first()
+ )
+
+ if not custom_type:
+ raise HTTPException(
+ status_code=404,
+ detail="Custom type not found"
+ )
+
+ return custom_type
+
+
[email protected]("/{id}/schema")
+def get_custom_type_schema(
+ id: UUID,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
+ """Get the raw JSON Schema for a custom type."""
+ custom_type = (
+ db.query(CustomType)
+ .filter(CustomType.id == id, CustomType.owner_id == current_user.id)
+ .first()
+ )
+
+ if not custom_type:
+ raise HTTPException(
+ status_code=404,
+ detail="Custom type not found"
+ )
+
+ return custom_type.schema
+
+
[email protected]("/{id}", response_model=CustomTypeRead)
+def update_custom_type(
+ id: UUID,
+ update_data: CustomTypeUpdate,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
+ """
+ Update a custom type.
+
+ If the schema is changed, a new checksum is calculated.
+ """
+ custom_type = (
+ db.query(CustomType)
+ .filter(CustomType.id == id, CustomType.owner_id == current_user.id)
+ .first()
+ )
+
+ if not custom_type:
+ raise HTTPException(
+ status_code=404,
+ detail="Custom type not found"
+ )
+
+ # Update fields
+ if update_data.name is not None:
+ # Check for duplicate name
+ existing = (
+ db.query(CustomType)
+ .filter(
+ CustomType.owner_id == current_user.id,
+ CustomType.name == update_data.name,
+ CustomType.id != id
+ )
+ .first()
+ )
+ if existing:
+ raise HTTPException(
+ status_code=400,
+ detail=f"Custom type with name '{update_data.name}' already exists"
+ )
+ custom_type.name = update_data.name
+
+ if update_data.json_schema is not None:
+ # Validate the new schema
+ validate_json_schema(update_data.json_schema)
+ custom_type.schema = update_data.json_schema
+ custom_type.checksum = calculate_schema_checksum(update_data.json_schema)
+
+ if update_data.description is not None:
+ custom_type.description = update_data.description
+
+ if update_data.status is not None:
+ custom_type.status = update_data.status
+
+ db.commit()
+ db.refresh(custom_type)
+
+ return custom_type
+
+
[email protected]("/{id}", status_code=status.HTTP_204_NO_CONTENT)
+def delete_custom_type(
+ id: UUID,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
+ """Delete a custom type."""
+ custom_type = (
+ db.query(CustomType)
+ .filter(CustomType.id == id, CustomType.owner_id == current_user.id)
+ .first()
+ )
+
+ if not custom_type:
+ raise HTTPException(
+ status_code=404,
+ detail="Custom type not found"
+ )
+
+ db.delete(custom_type)
+ db.commit()
+
+ return None
+
+
[email protected]("/{id}/validate", response_model=CustomTypeValidateResponse)
+def validate_payload(
+ id: UUID,
+ payload_data: CustomTypeValidatePayload,
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
+):
+ """
+ Validate a payload against a custom type's schema.
+
+ This is useful for testing before publishing a type.
+ """
+ custom_type = (
+ db.query(CustomType)
+ .filter(CustomType.id == id, CustomType.owner_id == current_user.id)
+ .first()
+ )
+
+ if not custom_type:
+ raise HTTPException(
+ status_code=404,
+ detail="Custom type not found"
+ )
+
+ # Validate the payload against the schema
+ is_valid, errors = validate_payload_against_schema(
+ payload_data.payload,
+ custom_type.schema
+ )
+
+ return CustomTypeValidateResponse(
+ valid=is_valid,
+ errors=errors if not is_valid else None
+ )
diff --git a/flowsint-api/app/api/routes/flows.py b/flowsint-api/app/api/routes/flows.py
index a6bd3a16..b19940c6 100644
--- a/flowsint-api/app/api/routes/flows.py
+++ b/flowsint-api/app/api/routes/flows.py
@@ -10,8 +10,9 @@ from flowsint_types import Domain, Phrase, Ip, SocialProfile, Organization, Emai
from flowsint_core.core.types import Node, Edge, FlowStep, FlowBranch
from sqlalchemy.orm import Session
from flowsint_core.core.postgre_db import get_db
-from flowsint_core.core.models import Flow, Profile
+from flowsint_core.core.models import Flow, Profile, CustomType
from app.api.deps import get_current_user
+from sqlalchemy import func
from app.api.schemas.flow import FlowRead, FlowCreate, FlowUpdate
from flowsint_types import (
ASN,
@@ -48,25 +49,40 @@ class launchFlowPayload(BaseModel):
router = APIRouter()
-# Get the list of all flows
@router.get("/", response_model=List[FlowRead])
def get_flows(
category: Optional[str] = Query(None),
db: Session = Depends(get_db),
current_user: Profile = Depends(get_current_user),
):
- query = db.query(Flow)
-
- if category is not None and category != "undefined":
- # Case-insensitive filtering by checking if any category matches (case-insensitive)
- flows = query.all()
+ if not category or category.lower() == "undefined":
+ return db.query(Flow).order_by(Flow.last_updated_at.desc()).all()
+
+ custom_type = (
+ db.query(CustomType)
+ .filter(
+ CustomType.owner_id == current_user.id,
+ CustomType.status == "published",
+ func.lower(CustomType.name) == category.lower(),
+ )
+ .first()
+ )
+ if custom_type:
+ flows = db.query(Flow).order_by(Flow.last_updated_at.desc()).all()
return [
- flow
+ {
+ **(flow.to_dict() if hasattr(flow, "to_dict") else flow.__dict__),
+ "wobblyType": True,
+ }
for flow in flows
- if any(cat.lower() == category.lower() for cat in flow.category)
]
-
- return query.order_by(Flow.last_updated_at.desc()).all()
+
+ flows = db.query(Flow).order_by(Flow.last_updated_at.desc()).all()
+ return [
+ flow
+ for flow in flows
+ if any(cat.lower() == category.lower() for cat in flow.category)
+ ]
# Returns the "raw_materials" for the flow editor
diff --git a/flowsint-api/app/api/routes/transforms.py b/flowsint-api/app/api/routes/transforms.py
index cb058620..9bc2ea10 100644
--- a/flowsint-api/app/api/routes/transforms.py
+++ b/flowsint-api/app/api/routes/transforms.py
@@ -4,8 +4,11 @@ from pydantic import BaseModel
from flowsint_core.core.registry import TransformRegistry
from flowsint_core.core.celery import celery
from flowsint_core.core.types import Node, Edge, FlowBranch
-from flowsint_core.core.models import Profile
+from flowsint_core.core.models import CustomType, Profile
from app.api.deps import get_current_user
+from flowsint_core.core.postgre_db import get_db
+from sqlalchemy.orm import Session
+from sqlalchemy import func
class FlowComputationRequest(BaseModel):
@@ -36,15 +39,26 @@ router = APIRouter()
@router.get("/")
def get_transforms(
category: Optional[str] = Query(None),
- # current_user: Profile = Depends(get_current_user),
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user),
):
- if category is not None and category != "undefined":
- transforms = TransformRegistry.list_by_input_type(
- category, exclude=["n8n_connector"]
+ if not category or category.lower() == "undefined":
+ return TransformRegistry.list(exclude=["n8n_connector"])
+ # Si catégorie custom
+ custom_type = (
+ db.query(CustomType)
+ .filter(
+ CustomType.owner_id == current_user.id,
+ CustomType.status == "published",
+ func.lower(CustomType.name) == category.lower(),
)
- else:
- transforms = TransformRegistry.list(exclude=["n8n_connector"])
- return transforms
+ .first()
+ )
+
+ if custom_type:
+ return TransformRegistry.list(exclude=["n8n_connector"], wobbly_type=True)
+
+ return TransformRegistry.list_by_input_type(category, exclude=["n8n_connector"])
@router.post("/{transform_name}/launch")
diff --git a/flowsint-api/app/api/routes/types.py b/flowsint-api/app/api/routes/types.py
index 25287904..1ab2cf6e 100644
--- a/flowsint-api/app/api/routes/types.py
+++ b/flowsint-api/app/api/routes/types.py
@@ -1,7 +1,11 @@
from typing import Any, Dict, Optional, Type
from uuid import uuid4
-from fastapi import APIRouter
+from fastapi import APIRouter, Depends
from pydantic import BaseModel, TypeAdapter
+from sqlalchemy.orm import Session
+from flowsint_core.core.postgre_db import get_db
+from flowsint_core.core.models import CustomType, Profile
+from app.api.deps import get_current_user
from flowsint_types import (
Domain,
Ip,
@@ -44,7 +48,10 @@ router = APIRouter()
# Returns the "types" for the sketches
@router.get("/")
-async def get_types_list():
+async def get_types_list(
+ db: Session = Depends(get_db),
+ current_user: Profile = Depends(get_current_user)
+):
types = [
{
"id": uuid4(),
@@ -201,6 +208,58 @@ async def get_types_list():
},
]
+ # Add custom types
+ custom_types = (
+ db.query(CustomType)
+ .filter(
+ CustomType.owner_id == current_user.id,
+ CustomType.status == "published" # Only show published custom types
+ )
+ .all()
+ )
+
+ if custom_types:
+ custom_types_children = []
+ for custom_type in custom_types:
+ # Extract the label_key from the schema (use first required field or first property)
+ schema = custom_type.schema
+ properties = schema.get("properties", {})
+ required = schema.get("required", [])
+
+ # Try to use the first required field, or the first property
+ label_key = required[0] if required else list(properties.keys())[0] if properties else "value"
+
+ custom_types_children.append({
+ "id": custom_type.id,
+ "type": custom_type.name,
+ "key": custom_type.name.lower(),
+ "label_key": label_key,
+ "icon": "custom",
+ "label": custom_type.name,
+ "description": custom_type.description or "",
+ "fields": [
+ {
+ "name": prop,
+ "label": info.get("title", prop),
+ "description": info.get("description", ""),
+ "type": "text",
+ "required": prop in required
+ }
+ for prop, info in properties.items()
+ ],
+ "custom": True # Mark as custom type
+ })
+
+ types.append({
+ "id": uuid4(),
+ "type": "custom_types_category",
+ "key": "custom_types",
+ "icon": "custom",
+ "label": "Custom types",
+ "fields": [],
+ "children": custom_types_children,
+ })
+
return types
diff --git a/flowsint-api/app/api/schemas/custom_type.py b/flowsint-api/app/api/schemas/custom_type.py
new file mode 100644
index 00000000..2a1b7d03
--- /dev/null
+++ b/flowsint-api/app/api/schemas/custom_type.py
@@ -0,0 +1,67 @@
+from typing import Optional, Dict, Any
+from pydantic import BaseModel, UUID4, Field, field_validator
+from datetime import datetime
+from .base import ORMBase
+
+
+class CustomTypeCreate(BaseModel):
+ """Schema for creating a new custom type."""
+ name: str = Field(..., min_length=1, max_length=255, description="Name of the custom type")
+ json_schema: Dict[str, Any] = Field(..., description="JSON Schema definition", alias="schema")
+ description: Optional[str] = Field(None, description="Optional description of the custom type")
+ status: Optional[str] = Field("draft", description="Status: draft or published")
+
+ class Config:
+ populate_by_name = True
+
+ @field_validator('status')
+ @classmethod
+ def validate_status(cls, v: str) -> str:
+ if v not in ["draft", "published", "archived"]:
+ raise ValueError("Status must be one of: draft, published, archived")
+ return v
+
+
+class CustomTypeUpdate(BaseModel):
+ """Schema for updating an existing custom type."""
+ name: Optional[str] = Field(None, min_length=1, max_length=255)
+ json_schema: Optional[Dict[str, Any]] = Field(None, alias="schema")
+ description: Optional[str] = None
+ status: Optional[str] = None
+
+ class Config:
+ populate_by_name = True
+
+ @field_validator('status')
+ @classmethod
+ def validate_status(cls, v: Optional[str]) -> Optional[str]:
+ if v is not None and v not in ["draft", "published", "archived"]:
+ raise ValueError("Status must be one of: draft, published, archived")
+ return v
+
+
+class CustomTypeRead(ORMBase):
+ """Schema for reading a custom type."""
+ id: UUID4
+ name: str
+ owner_id: UUID4
+ json_schema: Dict[str, Any] = Field(..., alias="schema")
+ status: str
+ checksum: Optional[str]
+ description: Optional[str]
+ created_at: datetime
+ updated_at: datetime
+
+ class Config:
+ populate_by_name = True
+
+
+class CustomTypeValidatePayload(BaseModel):
+ """Schema for validating a payload against a custom type schema."""
+ payload: Dict[str, Any] = Field(..., description="Data to validate against the schema")
+
+
+class CustomTypeValidateResponse(BaseModel):
+ """Response schema for validation."""
+ valid: bool
+ errors: Optional[list[str]] = None
diff --git a/flowsint-api/app/main.py b/flowsint-api/app/main.py
index 3f821e44..49f751be 100644
--- a/flowsint-api/app/main.py
+++ b/flowsint-api/app/main.py
@@ -16,6 +16,7 @@ from app.api.routes import chat
from app.api.routes import scan
from app.api.routes import keys
from app.api.routes import types
+from app.api.routes import custom_types
load_dotenv()
@@ -52,3 +53,4 @@ app.include_router(chat.router, prefix="/api/chats", tags=["chats"])
app.include_router(scan.router, prefix="/api/scans", tags=["scans"])
app.include_router(keys.router, prefix="/api/keys", tags=["keys"])
app.include_router(types.router, prefix="/api/types", tags=["types"])
+app.include_router(custom_types.router, prefix="/api/custom-types", tags=["custom-types"])
diff --git a/flowsint-api/app/utils/__init__.py b/flowsint-api/app/utils/__init__.py
new file mode 100644
index 00000000..02a66aa8
--- /dev/null
+++ b/flowsint-api/app/utils/__init__.py
@@ -0,0 +1 @@
+"""Utility modules for the API."""
diff --git a/flowsint-api/app/utils/custom_types.py b/flowsint-api/app/utils/custom_types.py
new file mode 100644
index 00000000..eac344a8
--- /dev/null
+++ b/flowsint-api/app/utils/custom_types.py
@@ -0,0 +1,215 @@
+"""Utilities for handling custom types: validation and dynamic model creation."""
+import hashlib
+import json
+from typing import Any, Dict, Type
+from pydantic import BaseModel, create_model, EmailStr, Field, field_validator
+from jsonschema import Draft7Validator, ValidationError as JSONSchemaValidationError
+from fastapi import HTTPException
+
+
+# Mapping of JSON Schema types to Python types
+TYPE_MAP = {
+ "string": str,
+ "integer": int,
+ "number": float,
+ "boolean": bool,
+ "object": dict,
+ "array": list,
+ "null": type(None),
+}
+
+# Mapping of JSON Schema formats to Pydantic types
+FORMAT_MAP = {
+ "email": EmailStr,
+ "uri": str,
+ "uuid": str,
+ "date": str,
+ "date-time": str,
+}
+
+# Whitelist of allowed JSON Schema types for security
+ALLOWED_TYPES = {"string", "integer", "number", "boolean", "object", "array"}
+
+
+def validate_json_schema(schema: Dict[str, Any]) -> None:
+ """
+ Validate that a schema is a valid JSON Schema.
+
+ Args:
+ schema: The JSON Schema to validate
+
+ Raises:
+ HTTPException: If the schema is invalid
+ """
+ try:
+ Draft7Validator.check_schema(schema)
+ except JSONSchemaValidationError as e:
+ raise HTTPException(
+ status_code=400,
+ detail=f"Invalid JSON Schema: {e.message}"
+ )
+
+ # Additional security checks
+ _check_schema_security(schema)
+
+
+def _check_schema_security(schema: Dict[str, Any]) -> None:
+ """
+ Check schema for security issues.
+
+ Args:
+ schema: The JSON Schema to check
+
+ Raises:
+ HTTPException: If security issues are found
+ """
+ # Check if type is in whitelist
+ schema_type = schema.get("type")
+ if schema_type and schema_type not in ALLOWED_TYPES:
+ raise HTTPException(
+ status_code=400,
+ detail=f"Type '{schema_type}' is not allowed. Allowed types: {ALLOWED_TYPES}"
+ )
+
+ # Check properties recursively
+ properties = schema.get("properties", {})
+ for prop_name, prop_schema in properties.items():
+ if isinstance(prop_schema, dict):
+ prop_type = prop_schema.get("type")
+ if prop_type and prop_type not in ALLOWED_TYPES:
+ raise HTTPException(
+ status_code=400,
+ detail=f"Type '{prop_type}' in property '{prop_name}' is not allowed"
+ )
+
+
+def validate_payload_against_schema(payload: Dict[str, Any], schema: Dict[str, Any]) -> tuple[bool, list[str]]:
+ """
+ Validate a payload against a JSON Schema.
+
+ Args:
+ payload: The data to validate
+ schema: The JSON Schema to validate against
+
+ Returns:
+ Tuple of (is_valid, error_messages)
+ """
+ validator = Draft7Validator(schema)
+ errors = list(validator.iter_errors(payload))
+
+ if errors:
+ error_messages = [
+ f"{'.'.join(str(p) for p in error.path)}: {error.message}"
+ for error in errors
+ ]
+ return False, error_messages
+
+ return True, []
+
+
+def calculate_schema_checksum(schema: Dict[str, Any]) -> str:
+ """
+ Calculate a checksum for a schema to detect changes.
+
+ Args:
+ schema: The JSON Schema
+
+ Returns:
+ SHA256 checksum of the schema
+ """
+ schema_str = json.dumps(schema, sort_keys=True)
+ return hashlib.sha256(schema_str.encode()).hexdigest()
+
+
+def jsonschema_to_pydantic(schema: Dict[str, Any], model_name: str = "DynamicModel") -> Type[BaseModel]:
+ """
+ Convert a JSON Schema to a Pydantic model.
+
+ Args:
+ schema: The JSON Schema definition
+ model_name: Name for the generated Pydantic model
+
+ Returns:
+ A dynamically created Pydantic model class
+
+ Raises:
+ HTTPException: If conversion fails
+ """
+ try:
+ fields = {}
+ properties = schema.get("properties", {})
+ required_fields = set(schema.get("required", []))
+
+ for field_name, field_schema in properties.items():
+ python_type = _get_python_type(field_schema)
+
+ # Determine if field is required
+ if field_name in required_fields:
+ default = ... # Required field
+ else:
+ default = field_schema.get("default", None)
+
+ # Add field constraints if present
+ field_kwargs = {}
+ if "description" in field_schema:
+ field_kwargs["description"] = field_schema["description"]
+ if "minimum" in field_schema:
+ field_kwargs["ge"] = field_schema["minimum"]
+ if "maximum" in field_schema:
+ field_kwargs["le"] = field_schema["maximum"]
+ if "minLength" in field_schema:
+ field_kwargs["min_length"] = field_schema["minLength"]
+ if "maxLength" in field_schema:
+ field_kwargs["max_length"] = field_schema["maxLength"]
+
+ if field_kwargs:
+ fields[field_name] = (python_type, Field(default, **field_kwargs))
+ else:
+ fields[field_name] = (python_type, default)
+
+ # Create the dynamic model
+ dynamic_model = create_model(model_name, **fields)
+ return dynamic_model
+
+ except Exception as e:
+ raise HTTPException(
+ status_code=400,
+ detail=f"Failed to convert JSON Schema to Pydantic model: {str(e)}"
+ )
+
+
+def _get_python_type(field_schema: Dict[str, Any]) -> Type:
+ """
+ Get the Python type for a JSON Schema field.
+
+ Args:
+ field_schema: The JSON Schema field definition
+
+ Returns:
+ The corresponding Python type
+ """
+ # Handle format first (e.g., email)
+ field_format = field_schema.get("format")
+ if field_format and field_format in FORMAT_MAP:
+ return FORMAT_MAP[field_format]
+
+ # Handle type
+ field_type = field_schema.get("type")
+ if field_type:
+ if isinstance(field_type, list):
+ # Handle union types (e.g., ["string", "null"])
+ # For simplicity, take the first non-null type
+ for t in field_type:
+ if t != "null":
+ return TYPE_MAP.get(t, Any)
+ return Any
+ return TYPE_MAP.get(field_type, Any)
+
+ # Handle anyOf (common in Pydantic-generated schemas)
+ if "anyOf" in field_schema:
+ any_of = field_schema["anyOf"]
+ for option in any_of:
+ if option.get("type") != "null":
+ return _get_python_type(option)
+
+ return Any