A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
refactor: remove occurences of scanner
dextmorgn980bb85af476
diff --git a/README.md b/README.mdindex 73aec6c3..ad696571 100644--- a/README.md+++ b/README.md@@ -64,7 +64,7 @@ Core utilities and base classes used by all other modules:- Authentication and authorization- Logging and event handling- Configuration management-- Base classes for scanners and tools+- Base classes for transforms and tools- Utility functions### flowsint-types@@ -81,11 +81,11 @@ Pydantic models for all data types:Transform modules that process data:-- Domain scanners (subdomains, WHOIS, resolution)-- IP scanners (geolocation, ASN lookup)-- Social media scanners (Maigret, Sherlock)-- Email scanners (breaches, Gravatar)-- Crypto scanners (transactions, NFTs)+- Domain transforms (subdomains, WHOIS, resolution)+- IP transforms (geolocation, ASN lookup)+- Social media transforms (Maigret, Sherlock)+- Email transforms (breaches, Gravatar)+- Crypto transforms (transactions, NFTs)- And many more...### flowsint-apidiff --git a/flowsint-api/README.md b/flowsint-api/README.mdindex 7e233ddc..2b553006 100644--- a/flowsint-api/README.md+++ b/flowsint-api/README.md@@ -50,4 +50,4 @@ The crawler can be configured with:## Usage-The API provides REST endpoints for various scanning operations. See the individual scanner modules for specific usage examples.+The API provides REST endpoints for various scanning operations. See the individual transform modules for specific usage examples.diff --git a/flowsint-api/alembic/versions/2da47dbd4a52_add_cascade_delete_to_scans_and_logs.py b/flowsint-api/alembic/versions/2da47dbd4a52_add_cascade_delete_to_scans_and_logs.pyindex ca0ee0b0..dc4590d7 100644--- a/flowsint-api/alembic/versions/2da47dbd4a52_add_cascade_delete_to_scans_and_logs.py+++ b/flowsint-api/alembic/versions/2da47dbd4a52_add_cascade_delete_to_scans_and_logs.py@@ -22,7 +22,7 @@ def upgrade() -> None:"""Upgrade schema."""# ### commands auto generated by Alembic - please adjust! ###op.alter_column('scans', 'status',- existing_type=postgresql.ENUM('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', name='scannerstatus'),+ existing_type=postgresql.ENUM('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', name='transformstatus'),nullable=True)op.drop_constraint('scans_sketch_id_fkey', 'scans', type_='foreignkey')op.create_foreign_key(None, 'scans', 'sketches', ['sketch_id'], ['id'], onupdate='CASCADE', ondelete='CASCADE')@@ -35,6 +35,6 @@ def downgrade() -> None:op.drop_constraint(None, 'scans', type_='foreignkey')op.create_foreign_key('scans_sketch_id_fkey', 'scans', 'sketches', ['sketch_id'], ['id'])op.alter_column('scans', 'status',- existing_type=postgresql.ENUM('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', name='scannerstatus'),+ existing_type=postgresql.ENUM('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', name='transformstatus'),nullable=False)# ### end Alembic commands ###diff --git a/flowsint-api/alembic/versions/6dfa83113ad7_change_content_colum_of_log_to_json.py b/flowsint-api/alembic/versions/6dfa83113ad7_change_content_colum_of_log_to_json.pyindex 5c102f30..8a8766c3 100644--- a/flowsint-api/alembic/versions/6dfa83113ad7_change_content_colum_of_log_to_json.py+++ b/flowsint-api/alembic/versions/6dfa83113ad7_change_content_colum_of_log_to_json.py@@ -68,8 +68,8 @@ def downgrade() -> None:# 1. Revert 'scans.status' to old ENUM typeop.execute("""ALTER TABLE scans- ALTER COLUMN status TYPE scannerstatus- USING status::text::scannerstatus+ ALTER COLUMN status TYPE transformstatus+ USING status::text::transformstatus""")# 2. Revert 'logs.type' back to VARCHARdiff --git a/flowsint-api/alembic/versions/71a3e5b4db2a_update_scan_status_enum.py b/flowsint-api/alembic/versions/71a3e5b4db2a_update_scan_status_enum.pyindex b57033ae..d8d7cce7 100644--- a/flowsint-api/alembic/versions/71a3e5b4db2a_update_scan_status_enum.py+++ b/flowsint-api/alembic/versions/71a3e5b4db2a_update_scan_status_enum.py@@ -21,7 +21,7 @@ depends_on: Union[str, Sequence[str], None] = Nonedef upgrade() -> None:"""Upgrade schema."""# Create the enum type first using raw SQL to ensure it exists- op.execute("CREATE TYPE scannerstatus AS ENUM ('PENDING', 'RUNNING', 'COMPLETED', 'FAILED')")+ op.execute("CREATE TYPE transformstatus AS ENUM ('PENDING', 'RUNNING', 'COMPLETED', 'FAILED')")# Add new columnsop.add_column('scans', sa.Column('started_at', sa.DateTime(), nullable=True))@@ -30,17 +30,17 @@ def upgrade() -> None:op.add_column('scans', sa.Column('details', sa.JSON(), nullable=True))# Add new status column with enum type- op.add_column('scans', sa.Column('status_new', postgresql.ENUM('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', name='scannerstatus'), nullable=True))+ op.add_column('scans', sa.Column('status_new', postgresql.ENUM('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', name='transformstatus'), nullable=True))# Copy data from old status to new status with proper castingop.execute("""UPDATE scansSET status_new = CASE- WHEN status = 'PENDING' THEN 'PENDING'::scannerstatus- WHEN status = 'RUNNING' THEN 'RUNNING'::scannerstatus- WHEN status = 'COMPLETED' THEN 'COMPLETED'::scannerstatus- WHEN status = 'FAILED' THEN 'FAILED'::scannerstatus- ELSE 'PENDING'::scannerstatus+ WHEN status = 'PENDING' THEN 'PENDING'::transformstatus+ WHEN status = 'RUNNING' THEN 'RUNNING'::transformstatus+ WHEN status = 'COMPLETED' THEN 'COMPLETED'::transformstatus+ WHEN status = 'FAILED' THEN 'FAILED'::transformstatus+ ELSE 'PENDING'::transformstatusEND""")@@ -100,4 +100,4 @@ def downgrade() -> None:op.drop_column('scans', 'started_at')# Drop the enum type- op.execute("DROP TYPE scannerstatus")\ No newline at end of file+ op.execute("DROP TYPE transformstatus")\ No newline at end of filediff --git a/flowsint-api/app/api/routes/events.py b/flowsint-api/app/api/routes/events.pyindex 92b3f260..201aedb1 100644--- a/flowsint-api/app/api/routes/events.py+++ b/flowsint-api/app/api/routes/events.py@@ -105,9 +105,9 @@ async def stream_events(continue# Handle different types of events- if isinstance(data, dict) and data.get("type") == "scanner_complete":- # Send scanner completion event- yield json.dumps({"event": "scanner_complete", "data": data})+ if isinstance(data, dict) and data.get("type") == "transform_complete":+ # Send transform completion event+ yield json.dumps({"event": "transform_complete", "data": data})else:# Send regular log eventyield json.dumps({"event": "log", "data": data})diff --git a/flowsint-api/app/api/routes/flows.py b/flowsint-api/app/api/routes/flows.pyindex b7da55d9..7bae3698 100644--- a/flowsint-api/app/api/routes/flows.py+++ b/flowsint-api/app/api/routes/flows.py@@ -72,27 +72,27 @@ def get_flows(# Returns the "raw_materials" for the flow editor@router.get("/raw_materials")async def get_material_list():- scanners = TransformRegistry.list_by_categories()- scanner_categories = {+ transforms = TransformRegistry.list_by_categories()+ transform_categories = {category: [{- "class_name": scanner.get("class_name"),- "category": scanner.get("category"),- "name": scanner.get("name"),- "module": scanner.get("module"),- "documentation": scanner.get("documentation"),- "description": scanner.get("description"),- "inputs": scanner.get("inputs"),- "outputs": scanner.get("outputs"),- "type": "scanner",- "params": scanner.get("params"),- "params_schema": scanner.get("params_schema"),- "required_params": scanner.get("required_params"),- "icon": scanner.get("icon"),+ "class_name": transform.get("class_name"),+ "category": transform.get("category"),+ "name": transform.get("name"),+ "module": transform.get("module"),+ "documentation": transform.get("documentation"),+ "description": transform.get("description"),+ "inputs": transform.get("inputs"),+ "outputs": transform.get("outputs"),+ "type": "transform",+ "params": transform.get("params"),+ "params_schema": transform.get("params_schema"),+ "required_params": transform.get("required_params"),+ "icon": transform.get("icon"),}- for scanner in scanner_list+ for transform in transform_list]- for category, scanner_list in scanners.items()+ for category, transform_list in transforms.items()}object_inputs = [@@ -112,11 +112,11 @@ async def get_material_list():extract_input_schema_flow(CryptoNFT),]- # Put types first, then add all scanner categories- flattened_scanners = {"types": object_inputs}- flattened_scanners.update(scanner_categories)+ # Put types first, then add all transform categories+ flattened_transforms = {"types": object_inputs}+ flattened_transforms.update(transform_categories)- return {"items": flattened_scanners}+ return {"items": flattened_transforms}# Returns the "raw_materials" for the flow editor@@ -296,8 +296,8 @@ def compute_flow_branches(node_map = {node.id: node for node in nodes}branches = []branch_counter = 0- # Track scanner outputs across all branches- scanner_outputs = {}+ # Track transform outputs across all branches+ transform_outputs = {}def calculate_path_length(start_node: str, visited: set = None) -> int:"""Calculate the shortest possible path length from a node to any leaf"""@@ -340,7 +340,7 @@ def compute_flow_branches(params=node_params,inputs={} if is_input_node else input_data,outputs=outputs,- type="type" if is_input_node else "scanner",+ type="type" if is_input_node else "transform",status="pending",branchId=branch_id,depth=depth,@@ -376,13 +376,13 @@ def compute_flow_branches()current_outputs = {first_output_name: initial_value}else:- # Check if we already have outputs for this scanner- if current_node_id in scanner_outputs:- current_outputs = scanner_outputs[current_node_id]+ # Check if we already have outputs for this transform+ if current_node_id in transform_outputs:+ current_outputs = transform_outputs[current_node_id]else:current_outputs = process_node_data(current_node, input_data)# Store the outputs for future use- scanner_outputs[current_node_id] = current_outputs+ transform_outputs[current_node_id] = current_outputs# Extract node parametersnode_params = current_node.data.get("params", {})@@ -493,49 +493,49 @@ def process_node_data(node: Node, inputs: Dict[str, Any]) -> Dict[str, Any]:for output in output_types:output_name = output.get("name", "output")class_name = node.data.get("class_name", "")- # For simulation purposes, we'll return a placeholder value based on the scanner type- if class_name in ["ReverseResolveScanner", "ResolveScanner"]:- # IP/Domain resolution scanners+ # For simulation purposes, we'll return a placeholder value based on the transform type+ if class_name in ["ReverseResolveTransform", "ResolveTransform"]:+ # IP/Domain resolution transformsoutputs[output_name] = ("192.168.1.1" if "ip" in output_name.lower() else "example.com")- elif class_name == "SubdomainScanner":- # Subdomain scanner+ elif class_name == "SubdomainTransform":+ # Subdomain transformoutputs[output_name] = f"sub.{inputs.get('input', 'example.com')}"- elif class_name == "WhoisScanner":- # WHOIS scanner+ elif class_name == "WhoisTransform":+ # WHOIS transformoutputs[output_name] = {"domain": inputs.get("input", "example.com"),"registrar": "Example Registrar","creation_date": "2020-01-01",}- elif class_name == "IpToInfosScanner":- # Geolocation scanner+ elif class_name == "IpToInfosTransform":+ # Geolocation transformoutputs[output_name] = {"country": "France","city": "Paris","coordinates": {"lat": 48.8566, "lon": 2.3522},}- elif class_name == "MaigretScanner":- # Social media scanner+ elif class_name == "MaigretTransform":+ # Social media transformoutputs[output_name] = {"username": inputs.get("input", "user123"),"platforms": ["twitter", "github", "linkedin"],}- elif class_name == "HoleheScanner":- # Email verification scanner+ elif class_name == "HoleheTransform":+ # Email verification transformoutputs[output_name] = {"email": inputs.get("input", "[email protected]"),"exists": True,"platforms": ["gmail", "github"],}- elif class_name == "SireneScanner":- # Organization scanner+ elif class_name == "SireneTransform":+ # Organization transformoutputs[output_name] = {"name": inputs.get("input", "Example Corp"),"siret": "12345678901234",@@ -543,7 +543,7 @@ def process_node_data(node: Node, inputs: Dict[str, Any]) -> Dict[str, Any]:}else:- # For unknown scanners, pass through the input+ # For unknown transforms, pass through the inputoutputs[output_name] = inputs.get("input") or f"flowed_{output_name}"return outputsdiff --git a/flowsint-api/app/utils.py b/flowsint-api/app/utils.pyindex 5a6441ac..1a0bd16d 100644--- a/flowsint-api/app/utils.py+++ b/flowsint-api/app/utils.py@@ -215,18 +215,18 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:input_output = input_node["data"]["outputs"]node_lookup = {node["id"]: node for node in nodes}- scanners = []+ transforms = []for edge in edges:target_id = edge["target"]source_handle = edge["sourceHandle"]target_handle = edge["targetHandle"]- scanner_node = node_lookup.get(target_id)- if scanner_node and scanner_node["data"]["type"] == "scanner":- scanners.append(+ transform_node = node_lookup.get(target_id)+ if transform_node and transform_node["data"]["type"] == "transform":+ transforms.append({- "transform_name": scanner_node["data"]["name"],- "module": scanner_node["data"]["module"],+ "transform_name": transform_node["data"]["name"],+ "module": transform_node["data"]["module"],"input": source_handle,"output": target_handle,}@@ -237,8 +237,8 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:"name": input_node["data"]["name"],"outputs": input_output,},- "scanners": scanners,- "transform_names": [scanner["transform_name"] for scanner in scanners],+ "transforms": transforms,+ "transform_names": [transform["transform_name"] for transform in transforms],}diff --git a/flowsint-app/src/renderer/src/components/auth-debug.tsx b/flowsint-app/src/renderer/src/components/auth-debug.tsxdeleted file mode 100644index e08a887d..00000000--- a/flowsint-app/src/renderer/src/components/auth-debug.tsx+++ /dev/null@@ -1,17 +0,0 @@-import { useAuthStore } from '@/stores/auth-store';-import { isAuthenticated } from '@/lib/auth-utils';--export function AuthDebug() {- const authState = useAuthStore();-- return (- <div className="fixed bottom-4 right-4 p-4 bg-black/80 text-white rounded-lg shadow-lg z-50">- <h3 className="font-bold mb-2">Auth State Debug</h3>- <div className="space-y-1 text-sm">- <p>Is Authenticated: {isAuthenticated() ? '✅' : '❌'}</p>- <p>Has Token: {authState.token ? '✅' : '❌'}</p>- <p>User: {authState.user ? authState.user.username : 'None'}</p>- </div>- </div>- );-}\ No newline at end of filediff --git a/flowsint-app/src/renderer/src/components/flows/editor.tsx b/flowsint-app/src/renderer/src/components/flows/editor.tsxindex 52ffc4ff..535b36b0 100644--- a/flowsint-app/src/renderer/src/components/flows/editor.tsx+++ b/flowsint-app/src/renderer/src/components/flows/editor.tsx@@ -18,9 +18,9 @@ import {import "@xyflow/react/dist/style.css"import { Play, Pause, SkipForward, RefreshCw } from "lucide-react"import { Button } from "@/components/ui/button"-import ScannerNode from "./scanner-node"+import TransformNode from "./transform-node"import TypeNode from "./type-node"-import { type ScannerNodeData } from "@/types/transform"+import { type TransformNodeData } from "@/types/transform"import { FlowControls } from "./controls"import { getDagreLayoutedElements } from "@/lib/utils"import { toast } from "sonner"@@ -42,11 +42,11 @@ import ParamsDialog from "./params-dialog"import FlowSheet from "./flow-sheet"import ContextMenu from "./context-menu"import { useNodesDisplaySettings } from "@/stores/node-display-settings"-import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query"+import { useMutation, useQueryClient } from "@tanstack/react-query"import { queryKeys } from "@/api/query-keys"const nodeTypes: NodeTypes = {- scanner: ScannerNode,+ transform: TransformNode,type: TypeNode,}@@ -111,13 +111,13 @@ const FlowEditor = memo(({ initialEdges, initialNodes, theme, flow }: FlowEditor// Update flow mutationconst updateFlowMutation = useMutation({- mutationFn: ({ flowId, body }: { flowId: string; body: BodyInit }) =>+ mutationFn: ({ flowId, body }: { flowId: string; body: BodyInit }) =>flowService.update(flowId, body),onSuccess: () => {toast.success("Transform saved successfully.")// Invalidate the flow detail queryif (flowId) {- queryClient.invalidateQueries({+ queryClient.invalidateQueries({queryKey: queryKeys.flows.detail(flowId)})}@@ -134,7 +134,7 @@ const FlowEditor = memo(({ initialEdges, initialNodes, theme, flow }: FlowEditorrouter.navigate({ to: "/dashboard/flows" })toast.success("Flow deleted successfully.")// Invalidate flows list- queryClient.invalidateQueries({+ queryClient.invalidateQueries({queryKey: queryKeys.flows.list})},@@ -145,7 +145,7 @@ const FlowEditor = memo(({ initialEdges, initialNodes, theme, flow }: FlowEditor// Compute flow mutationconst computeFlowMutation = useMutation({- mutationFn: ({ flowId, body }: { flowId: string; body: BodyInit }) =>+ mutationFn: ({ flowId, body }: { flowId: string; body: BodyInit }) =>flowService.compute(flowId, body),onSuccess: (response) => {setFlowBranches(response.flowBranches)@@ -229,11 +229,11 @@ const FlowEditor = memo(({ initialEdges, initialNodes, theme, flow }: FlowEditorevent.preventDefault()if (!reactFlowWrapper.current || !reactFlowInstance) returnconst reactFlowBounds = reactFlowWrapper.current.getBoundingClientRect()- const scannerData = JSON.parse(event.dataTransfer.getData("application/json")) as ScannerNodeData & {+ const transformData = JSON.parse(event.dataTransfer.getData("application/json")) as TransformNodeData & {category: string}- if (scannerData.type === "type") {+ if (transformData.type === "type") {const existsType = nodes.find((node) => node.data.type === "type")if (existsType) {return toast.error("Only one type node is allowed")@@ -244,26 +244,26 @@ const FlowEditor = memo(({ initialEdges, initialNodes, theme, flow }: FlowEditory: event.clientY - reactFlowBounds.top,})const newNode: FlowNode = {- id: `${scannerData.name}-${Date.now()}`,- type: scannerData.type === "type" ? "type" : "scanner",+ id: `${transformData.name}-${Date.now()}`,+ type: transformData.type === "type" ? "type" : "transform",position,data: {- id: scannerData.id,- class_name: scannerData.class_name,- module: scannerData.module || "",- key: scannerData.name,- color: colors[scannerData.category.toLowerCase()] || "#94a3b8",- name: scannerData.name,- category: scannerData.category,- type: scannerData.type,- inputs: scannerData.inputs,- outputs: scannerData.outputs,- documentation: scannerData.documentation,- description: scannerData.description,- required_params: scannerData.required_params,- params: scannerData.params,- params_schema: scannerData.params_schema,- icon: scannerData.icon+ id: transformData.id,+ class_name: transformData.class_name,+ module: transformData.module || "",+ key: transformData.name,+ color: colors[transformData.category.toLowerCase()] || "#94a3b8",+ name: transformData.name,+ category: transformData.category,+ type: transformData.type,+ inputs: transformData.inputs,+ outputs: transformData.outputs,+ documentation: transformData.documentation,+ description: transformData.description,+ required_params: transformData.required_params,+ params: transformData.params,+ params_schema: transformData.params_schema,+ icon: transformData.icon},}const updatedNodes = [...nodes, newNode]@@ -335,7 +335,7 @@ const FlowEditor = memo(({ initialEdges, initialNodes, theme, flow }: FlowEditortoast.error("Make sure your transform contains an input type.")return}-+const body = JSON.stringify({name: name,description: description,@@ -345,7 +345,7 @@ const FlowEditor = memo(({ initialEdges, initialNodes, theme, flow }: FlowEditoredges,},})-+if (flowId) {await updateFlowMutation.mutateAsync({ flowId, body })} else {diff --git a/flowsint-app/src/renderer/src/components/flows/flow-sheet.tsx b/flowsint-app/src/renderer/src/components/flows/flow-sheet.tsxindex 5b204b4d..ea2a29b5 100644--- a/flowsint-app/src/renderer/src/components/flows/flow-sheet.tsx+++ b/flowsint-app/src/renderer/src/components/flows/flow-sheet.tsx@@ -11,7 +11,7 @@ import { memo, useCallback, useMemo, useState } from "react"import { TriangleAlert, Loader2, ArrowRight } from "lucide-react"import { TooltipProvider, Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"import { useNodesDisplaySettings } from "@/stores/node-display-settings"-import { Scanner } from "@/types/transform"+import { Transform } from "@/types/transform"import { useQuery } from "@tanstack/react-query"import { Input } from "../ui/input"import { Alert, AlertDescription } from "../ui/alert"@@ -33,48 +33,48 @@ const FlowSheet = ({ onLayout }: { onLayout: () => void }) => {})const [searchTerm, setSearchTerm] = useState<string>("")- const filteredScanners = useMemo(() => {+ const filteredTransforms = useMemo(() => {if (!materials?.items || !selectedNode) return []if (!searchTerm.trim()) {return materials?.items}const term = searchTerm.toLowerCase()- return materials?.items.filter((scanner: Scanner) => {+ return materials?.items.filter((transform: Transform) => {return (- scanner.class_name.toLowerCase().includes(term) ||- scanner.name.toLowerCase().includes(term) ||- scanner.module.toLowerCase().includes(term) ||- scanner.inputs.type.toLowerCase().includes(term) ||- scanner.outputs.type.toLowerCase().includes(term) ||- (scanner.documentation && scanner.documentation.toLowerCase().includes(term))+ transform.class_name.toLowerCase().includes(term) ||+ transform.name.toLowerCase().includes(term) ||+ transform.module.toLowerCase().includes(term) ||+ transform.inputs.type.toLowerCase().includes(term) ||+ transform.outputs.type.toLowerCase().includes(term) ||+ (transform.documentation && transform.documentation.toLowerCase().includes(term)))})}, [searchTerm, materials?.items, selectedNode])- const handleClick = useCallback((scanner: Scanner) => {+ const handleClick = useCallback((transform: Transform) => {if (!selectedNode) returnconst position = { x: selectedNode.position.x + 350, y: selectedNode.position.y }const newNode: FlowNode = {- id: `${scanner.name}-${Date.now()}`,- type: scanner.type === "type" ? "type" : "scanner",+ id: `${transform.name}-${Date.now()}`,+ type: transform.type === "type" ? "type" : "transform",position,data: {- id: scanner.id,- class_name: scanner.class_name,- module: scanner.module || "",- key: scanner.name,- color: colors[scanner.category.toLowerCase()] || "#94a3b8",- name: scanner.name,- category: scanner.category,- type: scanner.type,- inputs: scanner.inputs,- outputs: scanner.outputs,- documentation: scanner.documentation,- description: scanner.description,- required_params: scanner.required_params,- params: scanner.params,- params_schema: scanner.params_schema,- icon: scanner.icon+ id: transform.id,+ class_name: transform.class_name,+ module: transform.module || "",+ key: transform.name,+ color: colors[transform.category.toLowerCase()] || "#94a3b8",+ name: transform.name,+ category: transform.category,+ type: transform.type,+ inputs: transform.inputs,+ outputs: transform.outputs,+ documentation: transform.documentation,+ description: transform.description,+ required_params: transform.required_params,+ params: transform.params,+ params_schema: transform.params_schema,+ icon: transform.icon},}setNodes((prev) => ([...prev, newNode]))@@ -84,7 +84,7 @@ const FlowSheet = ({ onLayout }: { onLayout: () => void }) => {source: selectedNode.id,target: newNode.id,sourceHandle: selectedNode.data.outputs.type,- targetHandle: scanner.inputs.type+ targetHandle: transform.inputs.type}setEdges((prev) => ([...prev, connection]))// onLayout && onLayout()@@ -125,13 +125,13 @@ const FlowSheet = ({ onLayout }: { onLayout: () => void }) => {{!isLoading && !error && (<div className="flex flex-col gap-2">- {filteredScanners.length === 0 ? (+ {filteredTransforms.length === 0 ? (<div className="text-center py-8 text-muted-foreground">{searchTerm.trim() ? 'No transforms found matching your search.' : 'No transforms available for this node type.'}</div>) : (- filteredScanners.map((scanner: Scanner) => (- <ScannerItem key={scanner.id} scanner={scanner} onClick={() => handleClick(scanner)} />+ filteredTransforms.map((transform: Transform) => (+ <TransformItem key={transform.id} transform={transform} onClick={() => handleClick(transform)} />)))}</div>@@ -146,22 +146,22 @@ const FlowSheet = ({ onLayout }: { onLayout: () => void }) => {export default FlowSheet-// Custom equality function for ScannerItem-function areEqual(prevProps: { scanner: Scanner }, nextProps: { scanner: Scanner }) {+// Custom equality function for TransformItem+function areEqual(prevProps: { transform: Transform }, nextProps: { transform: Transform }) {return (- prevProps.scanner.class_name === nextProps.scanner.class_name &&- prevProps.scanner.name === nextProps.scanner.name &&- prevProps.scanner.module === nextProps.scanner.module &&- prevProps.scanner.documentation === nextProps.scanner.documentation+ prevProps.transform.class_name === nextProps.transform.class_name &&+ prevProps.transform.name === nextProps.transform.name &&+ prevProps.transform.module === nextProps.transform.module &&+ prevProps.transform.documentation === nextProps.transform.documentation)}-// Memoized scanner item component for the sidebar-const ScannerItem = memo(({ scanner, onClick }: { scanner: Scanner, onClick: () => void }) => {+// Memoized transform item component for the sidebar+const TransformItem = memo(({ transform, onClick }: { transform: Transform, onClick: () => void }) => {const colors = useNodesDisplaySettings(s => s.colors)- const borderInputColor = colors[scanner.inputs.type.toLowerCase()]- const borderOutputColor = colors[scanner.outputs.type.toLowerCase()]- const Icon = scanner.type === "type" ? useIcon(scanner.outputs.type.toLowerCase() as string, null) : scanner.icon ? useIcon(scanner.icon, null) : null+ const borderInputColor = colors[transform.inputs.type.toLowerCase()]+ const borderOutputColor = colors[transform.outputs.type.toLowerCase()]+ const Icon = transform.type === "type" ? useIcon(transform.outputs.type.toLowerCase() as string, null) : transform.icon ? useIcon(transform.icon, null) : nullreturn (<TooltipProvider>@@ -175,24 +175,24 @@ const ScannerItem = memo(({ scanner, onClick }: { scanner: Scanner, onClick: ()<div className="space-y-1 truncate text-left"><div className="flex items-center gap-2 truncate text-ellipsis">{Icon && <Icon size={24} />}- <h3 className="text-sm font-medium truncate text-ellipsis">{scanner.class_name}</h3>+ <h3 className="text-sm font-medium truncate text-ellipsis">{transform.class_name}</h3></div>- <p className="text-sm font-normal opacity-60 truncate text-ellipsis">{scanner.description}</p>+ <p className="text-sm font-normal opacity-60 truncate text-ellipsis">{transform.description}</p><div className="mt-2 flex items-center gap-2 text-xs"><div className="flex items-center gap-1"><span className="text-muted-foreground">Takes</span>- <span className="font-bold truncate text-ellipsis">{scanner.inputs.type}</span>+ <span className="font-bold truncate text-ellipsis">{transform.inputs.type}</span></div><span><ArrowRight className="h-3 w-3" /></span><div className="flex items-center gap-1"><span className="text-muted-foreground">Returns</span>- <span className="font-bold truncate text-ellipsis">{scanner.outputs.type}</span>+ <span className="font-bold truncate text-ellipsis">{transform.outputs.type}</span></div></div></div></div></div>- {scanner.required_params &&+ {transform.required_params &&<div className="absolute bottom-3 right-3"><Tooltip><TooltipTrigger asChild>@@ -208,4 +208,4 @@ const ScannerItem = memo(({ scanner, onClick }: { scanner: Scanner, onClick: ())}, areEqual)-ScannerItem.displayName = "ScannerItem"+TransformItem.displayName = "TransformItem"diff --git a/flowsint-app/src/renderer/src/components/flows/params-dialog.tsx b/flowsint-app/src/renderer/src/components/flows/params-dialog.tsxindex cccf3845..5e306e10 100644--- a/flowsint-app/src/renderer/src/components/flows/params-dialog.tsx+++ b/flowsint-app/src/renderer/src/components/flows/params-dialog.tsx@@ -2,7 +2,7 @@ import { useFlowStore } from '@/stores/flow-store'import { DialogHeader, DialogFooter, Dialog, DialogContent, DialogTitle } from '../ui/dialog'import { Button } from '../ui/button'import { useCallback, useState, useEffect } from 'react'-import { ScannerParamSchemaItem } from '@/types'+import { TransformParamSchemaItem } from '@/types'import { Label } from '../ui/label'import { Input } from '../ui/input'import KeySelector from '../keys/key-select'@@ -78,7 +78,7 @@ const ParamsDialog = () => {</TabsList><TabsContent value="parameters" className="space-y-4 mt-4"><div className="grid gap-4">- {selectedNode?.data?.params_schema?.map((param: ScannerParamSchemaItem) => (+ {selectedNode?.data?.params_schema?.map((param: TransformParamSchemaItem) => (<div className="space-y-2" key={param.name}><div className="flex items-start flex-col"><Label htmlFor={param.name} className="text-sm font-medium">diff --git a/flowsint-app/src/renderer/src/components/flows/raw-material.tsx b/flowsint-app/src/renderer/src/components/flows/raw-material.tsxindex 03e8cf7e..bd718255 100644--- a/flowsint-app/src/renderer/src/components/flows/raw-material.tsx+++ b/flowsint-app/src/renderer/src/components/flows/raw-material.tsx@@ -1,7 +1,7 @@import { useMemo, useState } from "react"import { Search, X } from "lucide-react"-import ScannerItem from "./scanner-item"-import { type Scanner } from "@/types/transform"+import TransformItem from "./transform-list-item"+import { type Transform } from "@/types/transform"import { Input } from "../ui/input"import { Button } from "../ui/button"import { useQuery } from "@tanstack/react-query"@@ -17,9 +17,9 @@ export default function RawMaterial() {})const [searchTerm, setSearchTerm] = useState<string>("")- const filteredScanners = useMemo(() => {+ const filteredTransforms = useMemo(() => {if (!materials?.items) return {}- const result: Record<string, Scanner[]> = {}+ const result: Record<string, Transform[]> = {}if (!searchTerm.trim()) {return materials?.items}@@ -50,7 +50,7 @@ export default function RawMaterial() {<Search className="absolute left-2 top-2.5 h-4 w-4 text-muted-foreground" /><Inputtype="text"- placeholder="Search scanners..."+ placeholder="Search transforms..."className="pl-8 !border border-border"value={searchTerm}onChange={(e) => setSearchTerm(e.target.value)}@@ -67,24 +67,24 @@ export default function RawMaterial() {)}</div><div className="flex-1 w-full">- {Object.entries(filteredScanners).map(([category, scanners]) => (+ {Object.entries(filteredTransforms).map(([category, transforms]) => (<div key={category} className="space-y-2 w-full"><h3 className="text-sm font-medium capitalize mt-4">{category.replace("_", " ")}</h3><div className="space-y-2">{/* @ts-ignore */}- {scanners.map((scanner: Scanner) => (- <ScannerItem- key={scanner.name}- scanner={scanner}- category={scanner.category}+ {transforms.map((transform: Transform) => (+ <TransformItem+ key={transform.name}+ transform={transform}+ category={transform.category}/>))}</div></div>))}- {Object.keys(filteredScanners).length === 0 && (+ {Object.keys(filteredTransforms).length === 0 && (<div className="text-center py-4 text-muted-foreground">- No scanners found matching "{searchTerm}"+ No transforms found matching "{searchTerm}"</div>)}</div>diff --git a/flowsint-app/src/renderer/src/components/flows/scanner-item.tsx b/flowsint-app/src/renderer/src/components/flows/transform-list-item.tsxsimilarity index 75%rename from flowsint-app/src/renderer/src/components/flows/scanner-item.tsxrename to flowsint-app/src/renderer/src/components/flows/transform-list-item.tsxindex 862692d1..441c3cfe 100644--- a/flowsint-app/src/renderer/src/components/flows/scanner-item.tsx+++ b/flowsint-app/src/renderer/src/components/flows/transform-list-item.tsx@@ -7,41 +7,41 @@ import { TooltipProvider, Tooltip, TooltipContent, TooltipTrigger } from "@/compimport { Dialog, DialogContent, DialogHeader, DialogTitle, DialogTrigger } from "@/components/ui/dialog"import { useNodesDisplaySettings } from "@/stores/node-display-settings"import { Badge } from "../ui/badge"-import { type ScannerItemProps } from "@/types/transform"+import { type TransformItemProps } from "@/types/transform"import { useIcon } from "@/hooks/use-icon"-// Custom equality function for ScannerItem-function areEqual(prevProps: ScannerItemProps, nextProps: ScannerItemProps) {+// Custom equality function for TransformItem+function areEqual(prevProps: TransformItemProps, nextProps: TransformItemProps) {return (- prevProps.scanner.class_name === nextProps.scanner.class_name &&- prevProps.scanner.name === nextProps.scanner.name &&- prevProps.scanner.module === nextProps.scanner.module &&- prevProps.scanner.documentation === nextProps.scanner.documentation &&- prevProps.scanner.description === nextProps.scanner.description &&+ prevProps.transform.class_name === nextProps.transform.class_name &&+ prevProps.transform.name === nextProps.transform.name &&+ prevProps.transform.module === nextProps.transform.module &&+ prevProps.transform.documentation === nextProps.transform.documentation &&+ prevProps.transform.description === nextProps.transform.description &&prevProps.category === nextProps.category)}-// Memoized scanner item component for the sidebar-const ScannerItem = memo(({ scanner, category }: ScannerItemProps) => {+// Memoized transform item component for the sidebar+const TransformItem = memo(({ transform, category }: TransformItemProps) => {const colors = useNodesDisplaySettings(s => s.colors)- const borderInputColor = colors[scanner.inputs.type.toLowerCase()]- const borderOutputColor = colors[scanner.outputs.type.toLowerCase()]- const Icon = scanner.type === "type" ? useIcon(scanner.outputs.type.toLowerCase() as string, null) : scanner.icon ? useIcon(scanner.icon, null) : null+ const borderInputColor = colors[transform.inputs.type.toLowerCase()]+ const borderOutputColor = colors[transform.outputs.type.toLowerCase()]+ const Icon = transform.type === "type" ? useIcon(transform.outputs.type.toLowerCase() as string, null) : transform.icon ? useIcon(transform.icon, null) : nullconst [isDialogOpen, setIsDialogOpen] = useState(false)// Handler for drag start - using useCallback to prevent recreation on each renderconst onDragStart = useCallback((event: React.DragEvent<HTMLDivElement>) => {- const data = { ...scanner, category }+ const data = { ...transform, category }event.dataTransfer.setData("application/json", JSON.stringify(data))event.dataTransfer.effectAllowed = "move"},- [scanner, category],+ [transform, category],)- const isConfigurationRequired = scanner.required_params+ const isConfigurationRequired = transform.required_paramsreturn (<TooltipProvider>@@ -60,18 +60,18 @@ const ScannerItem = memo(({ scanner, category }: ScannerItemProps) => {<div className="space-y-1 truncate"><div className="flex items-center gap-2 truncate text-ellipsis">{Icon && <Icon size={24} />}- <h3 className="text-sm font-medium truncate text-ellipsis">{scanner.class_name}</h3>+ <h3 className="text-sm font-medium truncate text-ellipsis">{transform.class_name}</h3></div>- <p className="text-xs font-normal truncate text-ellipsis opacity-60">{scanner.description}</p>- {scanner.type !== "type" &&+ <p className="text-xs font-normal truncate text-ellipsis opacity-60">{transform.description}</p>+ {transform.type !== "type" &&<div className="mt-2 text-xs"><div className="flex items-center gap-1"><span className="font-medium">Input:</span>- <span className="text-muted-foreground truncate text-ellipsis">{scanner.inputs.type}</span>+ <span className="text-muted-foreground truncate text-ellipsis">{transform.inputs.type}</span></div><div className="flex items-center gap-1"><span className="font-medium">Output:</span>- <span className="text-muted-foreground truncate text-ellipsis">{scanner.outputs.type}</span>+ <span className="text-muted-foreground truncate text-ellipsis">{transform.outputs.type}</span></div></div>}</div>@@ -100,7 +100,7 @@ const ScannerItem = memo(({ scanner, category }: ScannerItemProps) => {<DialogHeader><DialogTitle className="flex items-center gap-2"><div className="w-1 h-6 rounded-full" style={{ backgroundColor: borderInputColor }} />- {scanner.class_name}+ {transform.class_name}</DialogTitle></DialogHeader><div className="grid gap-4 py-4">@@ -112,16 +112,16 @@ const ScannerItem = memo(({ scanner, category }: ScannerItemProps) => {</div>}<div className="space-y-2"><h4 className="font-medium text-sm" style={{ color: borderInputColor }}>Description</h4>- <p className="text-sm text-muted-foreground">{scanner.description || "No description available"}</p>+ <p className="text-sm text-muted-foreground">{transform.description || "No description available"}</p></div><div className="space-y-2"><h4 className="font-medium text-sm" style={{ color: borderInputColor }}>Module</h4>- <p className="text-sm text-muted-foreground">{scanner.module}</p>+ <p className="text-sm text-muted-foreground">{transform.module}</p></div><div className="space-y-2"><h4 className="font-medium text-sm" style={{ color: borderInputColor }}>Input Properties</h4><div className="space-y-1">- {scanner?.inputs?.properties?.map((prop, index) => (+ {transform?.inputs?.properties?.map((prop, index) => (<div key={index} className="text-sm"><span className="font-medium">{prop.name}:</span>{" "}<span className="text-muted-foreground">{prop.type}</span>@@ -132,7 +132,7 @@ const ScannerItem = memo(({ scanner, category }: ScannerItemProps) => {<div className="space-y-2"><h4 className="font-medium text-sm" style={{ color: borderOutputColor }}>Output Properties</h4><div className="space-y-1">- {scanner?.outputs?.properties?.map((prop, index) => (+ {transform?.outputs?.properties?.map((prop, index) => (<div key={index} className="text-sm"><span className="font-medium">{prop.name}:</span>{" "}<span className="text-muted-foreground">{prop.type}</span>@@ -147,6 +147,6 @@ const ScannerItem = memo(({ scanner, category }: ScannerItemProps) => {)}, areEqual)-ScannerItem.displayName = "ScannerItem"+TransformItem.displayName = "TransformItem"-export default ScannerItem+export default TransformItemdiff --git a/flowsint-app/src/renderer/src/components/flows/scanner-node.tsx b/flowsint-app/src/renderer/src/components/flows/transform-node.tsxsimilarity index 95%rename from flowsint-app/src/renderer/src/components/flows/scanner-node.tsxrename to flowsint-app/src/renderer/src/components/flows/transform-node.tsxindex 9dd40193..c50a6378 100644--- a/flowsint-app/src/renderer/src/components/flows/scanner-node.tsx+++ b/flowsint-app/src/renderer/src/components/flows/transform-node.tsx@@ -6,7 +6,7 @@ import { NodeStatusIndicator } from "../xyflow/node-status-indicator"import { useNodesDisplaySettings } from "@/stores/node-display-settings"import { cn } from "@/lib/utils"import { Plus, TriangleAlert } from "lucide-react"-import { type ScannerNodeProps } from "@/types/transform"+import { type TransformNodeProps } from "@/types/transform"import { FlowNode, useFlowStore } from "@/stores/flow-store"import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"import { Button } from "../ui/button"@@ -14,7 +14,7 @@ import { ButtonHandle } from "../xyflow/button-handle"import { useIcon } from "@/hooks/use-icon"// Custom equality function to prevent unnecessary re-renders-function areEqual(prevProps: ScannerNodeProps, nextProps: ScannerNodeProps) {+function areEqual(prevProps: TransformNodeProps, nextProps: TransformNodeProps) {return (prevProps.data.class_name === nextProps.data.class_name &&prevProps.data.name === nextProps.data.name &&@@ -44,8 +44,8 @@ const getStateColor = (state?: string) => {}}-// Scanner node component for transform/scanner nodes only-const ScannerNode = memo(({ data, isConnectable }: ScannerNodeProps) => {+// Transform node component for transform/transform nodes only+const TransformNode = memo(({ data, isConnectable }: TransformNodeProps) => {const colors = useNodesDisplaySettings(s => s.colors)const inputColor = colors[data.inputs.type.toLowerCase()]const outputColor = colors[data.outputs.type.toLowerCase()]@@ -174,6 +174,6 @@ const ScannerNode = memo(({ data, isConnectable }: ScannerNodeProps) => {)}, areEqual)-ScannerNode.displayName = "ScannerNode"+TransformNode.displayName = "TransformNode"-export default ScannerNode+export default TransformNodediff --git a/flowsint-app/src/renderer/src/components/flows/type-node.tsx b/flowsint-app/src/renderer/src/components/flows/type-node.tsxindex 12c8c30b..4eb9ee1c 100644--- a/flowsint-app/src/renderer/src/components/flows/type-node.tsx+++ b/flowsint-app/src/renderer/src/components/flows/type-node.tsx@@ -4,11 +4,9 @@ import { useNodesDisplaySettings } from "@/stores/node-display-settings"import { useIcon } from "@/hooks/use-icon"import { cn } from "@/lib/utils"import { Plus } from "lucide-react"-import { type ScannerNodeProps } from "@/types/transform"+import { type TransformNodeProps } from "@/types/transform"import { ButtonHandle } from "@/components/xyflow/button-handle"import { Button } from "../ui/button"--import {NodeTooltip,NodeTooltipContent,@@ -18,7 +16,7 @@ import { BaseNode, BaseNodeContent } from "@/components/xyflow/base-node";import { FlowNode, useFlowStore } from "@/stores/flow-store"// Custom equality function to prevent unnecessary re-renders-function areEqual(prevProps: ScannerNodeProps, nextProps: ScannerNodeProps) {+function areEqual(prevProps: TransformNodeProps, nextProps: TransformNodeProps) {return (prevProps.data.class_name === nextProps.data.class_name &&prevProps.data.name === nextProps.data.name &&@@ -28,7 +26,7 @@ function areEqual(prevProps: ScannerNodeProps, nextProps: ScannerNodeProps) {}// Type node component for data types (domains, websites, IPs, etc.)-const TypeNode = memo(({ data }: ScannerNodeProps) => {+const TypeNode = memo(({ data }: TransformNodeProps) => {const colors = useNodesDisplaySettings(s => s.colors)const outputColor = colors[data.outputs.type.toLowerCase()]const Icon = useIcon(data.outputs.type.toLowerCase() as string, null)diff --git a/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx b/flowsint-app/src/renderer/src/components/graphs/context-menu.tsxindex 9a0f73b5..ffb3e9d7 100644--- a/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx+++ b/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx@@ -189,7 +189,7 @@ export default function ContextMenu({</div><div className='flex items-center gap-1'>{/* <FavoriteButton isFavorite={false} /> */}- <InfoButton description={transform.description} />+ <InfoButton description={transform.description ?? ""} /></div></button>))}diff --git a/flowsint-app/src/renderer/src/components/investigations/sketch-list.tsx b/flowsint-app/src/renderer/src/components/investigations/sketch-list.tsxindex a35af897..08304ea3 100644--- a/flowsint-app/src/renderer/src/components/investigations/sketch-list.tsx+++ b/flowsint-app/src/renderer/src/components/investigations/sketch-list.tsx@@ -24,7 +24,7 @@ const SketchList = () => {if (!searchQuery.trim()) return investigation.sketchesconst query = searchQuery.toLowerCase().trim()- return investigation.sketches.filter((sketch) =>+ return investigation.sketches.filter((sketch) =>sketch.title.toLowerCase().includes(query))}, [investigation?.sketches, searchQuery])@@ -33,15 +33,15 @@ const SketchList = () => {return (<div className="w-full h-full bg-card flex flex-col overflow-hidden"><div className="p-2 flex items-center gap-2 border-b shrink-0">- <NewSketch noDropDown>+ <NewSketch><Button variant="ghost" size="icon" className="h-7 w-7"><PlusIcon className="h-4 w-4" /></Button></NewSketch>- <Input- type="search"- className="!border border-border h-7"- placeholder="Search..."+ <Input+ type="search"+ className="!border border-border h-7"+ placeholder="Search..."value={searchQuery}onChange={(e) => setSearchQuery(e.target.value)}/>@@ -71,13 +71,13 @@ const SketchList = () => {{searchQuery ? "No matching sketches" : "No sketches yet"}</h3><p className="text-xs opacity-70 max-w-xs">- {searchQuery+ {searchQuery? "Try adjusting your search query to find what you're looking for.": "This investigation doesn't contain any sketch. Sketches let you organize and visualize your data as graphs."}</p></div>- <NewSketch noDropDown>+ <NewSketch><Button size="sm" variant="outline"><PlusIcon className="w-4 h-4 mr-2" />Create your first sketchdiff --git a/flowsint-app/src/renderer/src/components/layout/info.tsx b/flowsint-app/src/renderer/src/components/layout/info.tsxindex f1815e27..4bc799cd 100644--- a/flowsint-app/src/renderer/src/components/layout/info.tsx+++ b/flowsint-app/src/renderer/src/components/layout/info.tsx@@ -41,7 +41,7 @@ const InfoDialog = () => {<strong>Tracks and saves investigation states</strong> over time, letting you explore multiple hypotheses or revisit older threads without losing context.</li><li>- Supports <strong>live data enrichment</strong> from custom or built-in scanners, giving you <strong>actionable insights</strong> as you explore.+ Supports <strong>live data enrichment</strong> from custom or built-in transforms, giving you <strong>actionable insights</strong> as you explore.</li></ul>diff --git a/flowsint-app/src/renderer/src/components/table/columns.tsx b/flowsint-app/src/renderer/src/components/table/columns.tsxindex 8a88d312..d01815f6 100644--- a/flowsint-app/src/renderer/src/components/table/columns.tsx+++ b/flowsint-app/src/renderer/src/components/table/columns.tsx@@ -13,7 +13,8 @@ import {} from "@/components/ui/dropdown-menu"import { Checkbox } from "../ui/checkbox"import { useCallback, memo } from "react"-import { GraphNode, useGraphStore } from "@/stores/graph-store"+import { useGraphStore } from "@/stores/graph-store"+import { GraphNode } from "@/types"// Memoized icon component to prevent unnecessary re-rendersconst MemoizedIcon = memo(({ type, size = 16 }: { type: string; size?: number }) => {@@ -65,7 +66,7 @@ const getConfidenceColor = (confidence: number) => {export const columns: ColumnDef<GraphNode>[] = [{size: 50,- minSize:50,+ minSize: 50,maxSize: 50,enableResizing: false,accessorKey: "icon",@@ -100,7 +101,7 @@ export const columns: ColumnDef<GraphNode>[] = [{enableResizing: true,size: 200,- minSize:120,+ minSize: 120,maxSize: 500,accessorKey: "data.label",header: ({ column }) => {@@ -156,7 +157,7 @@ export const columns: ColumnDef<GraphNode>[] = [)},size: 120,- minSize:80,+ minSize: 80,maxSize: 200,accessorKey: "data.type",cell: ({ row }) => {@@ -171,7 +172,7 @@ export const columns: ColumnDef<GraphNode>[] = [{enableResizing: true,size: 200,- minSize:120,+ minSize: 120,maxSize: 500,header: ({ column }) => {return (@@ -202,7 +203,7 @@ export const columns: ColumnDef<GraphNode>[] = [{enableResizing: true,size: 200,- minSize:120,+ minSize: 120,maxSize: 500,header: ({ column }) => {return (@@ -218,7 +219,7 @@ export const columns: ColumnDef<GraphNode>[] = [},accessorKey: "data.source",cell: ({ row }) => {- const source = row.original.data.source || 'Scanner'+ const source = row.original.data.source || 'Transform'return (<div className="text-left font-medium text-gray-700 dark:text-gray-200">{source}@@ -229,7 +230,7 @@ export const columns: ColumnDef<GraphNode>[] = [{enableResizing: true,size: 200,- minSize:120,+ minSize: 120,maxSize: 500,header: ({ column }) => {return (@@ -266,7 +267,7 @@ export const columns: ColumnDef<GraphNode>[] = [{enableResizing: true,size: 200,- minSize:120,+ minSize: 120,maxSize: 500,header: ({ column }) => {return (@@ -295,7 +296,7 @@ export const columns: ColumnDef<GraphNode>[] = [{enableResizing: true,size: 200,- minSize:120,+ minSize: 120,maxSize: 500,header: ({ column }) => {return (@@ -322,7 +323,7 @@ export const columns: ColumnDef<GraphNode>[] = [{id: "actions",size: 50,- minSize:50,+ minSize: 50,maxSize: 50,enableResizing: false,header: () => {diff --git a/flowsint-app/src/renderer/src/routes/_auth.dashboard.tsx b/flowsint-app/src/renderer/src/routes/_auth.dashboard.tsxindex 76994cf5..6daaffc8 100644--- a/flowsint-app/src/renderer/src/routes/_auth.dashboard.tsx+++ b/flowsint-app/src/renderer/src/routes/_auth.dashboard.tsx@@ -12,7 +12,7 @@ function DashboardPage() {return (<RootLayout><Outlet />- {Boolean(settings.general.showFlow.value) && <FloatingChat />}+ {Boolean(settings?.general?.showFlow?.value) && <FloatingChat />}</RootLayout>)}diff --git a/flowsint-app/src/renderer/src/stores/flow-store.ts b/flowsint-app/src/renderer/src/stores/flow-store.tsindex d039086d..1e88a1ec 100644--- a/flowsint-app/src/renderer/src/stores/flow-store.ts+++ b/flowsint-app/src/renderer/src/stores/flow-store.ts@@ -12,9 +12,9 @@ import {applyEdgeChanges} from "@xyflow/react"import { toast } from "sonner"-import { type ScannerNodeData } from "@/types/transform"+import { type TransformNodeData } from "@/types/transform"-export type NodeData = ScannerNodeData+export type NodeData = TransformNodeDataexport type FlowNode = Node<NodeData>export type FlowEdge = Edgediff --git a/flowsint-app/src/renderer/src/types/event.ts b/flowsint-app/src/renderer/src/types/event.tsindex c2198af1..2d4895ca 100644--- a/flowsint-app/src/renderer/src/types/event.ts+++ b/flowsint-app/src/renderer/src/types/event.ts@@ -1,4 +1,4 @@-import type { GraphEdge, GraphNode } from "@/stores/graph-store";+import { GraphEdge, GraphNode } from "./graph"export enum EventLevel {// Standard log levels@@ -7,7 +7,7 @@ export enum EventLevel {FAILED = "FAILED",SUCCESS = "SUCCESS",DEBUG = "DEBUG",- // Scanner-specific statuses+ // Transform-specific statusesPENDING = "PENDING",RUNNING = "RUNNING",COMPLETED = "COMPLETED",diff --git a/flowsint-app/src/renderer/src/types/transform.ts b/flowsint-app/src/renderer/src/types/transform.tsindex 1e2470fd..db5fc932 100644--- a/flowsint-app/src/renderer/src/types/transform.ts+++ b/flowsint-app/src/renderer/src/types/transform.ts@@ -1,14 +1,14 @@-export interface ScannerProperty {+export interface TransformProperty {name: stringtype: string}-export interface ScannerIO {+export interface TransformIO {type: string- properties: ScannerProperty[]+ properties: TransformProperty[]}-export interface ScannerParamSchemaItem {+export interface TransformParamSchemaItem {name: stringtype: stringdescription: string@@ -16,7 +16,7 @@ export interface ScannerParamSchemaItem {required: boolean}-export interface Scanner {+export interface Transform {id: stringclass_name: stringcategory: string@@ -24,12 +24,12 @@ export interface Scanner {module: stringdocumentation: string | nulldescription: string | null- inputs: ScannerIO- outputs: ScannerIO+ inputs: TransformIO+ outputs: TransformIOtype: stringrequired_params: booleanparams: Record<string, string>- params_schema: ScannerParamSchemaItem[]+ params_schema: TransformParamSchemaItem[]settings?: Record<string, string>icon: string | null}@@ -38,7 +38,7 @@ export interface Scanner {// NODE DATA TYPE FOR TRANSFORM STORE// ================================-export interface ScannerNodeData extends Scanner, Record<string, unknown> {+export interface TransformNodeData extends Transform, Record<string, unknown> {color?: stringcomputationState?: "pending" | "processing" | "completed" | "error"key: string@@ -49,10 +49,10 @@ export interface ScannerNodeData extends Scanner, Record<string, unknown> {// ================================export interface ScansData {- [category: string]: Scanner[]+ [category: string]: Transform[]}-export interface ScannerData {+export interface TransformData {items: ScansData}@@ -60,31 +60,17 @@ export interface ScannerData {// COMPONENT PROPS INTERFACES// ================================-export interface ScannerItemProps {- scanner: Scanner+export interface TransformItemProps {+ transform: Transformcategory: string}-export interface ScannerNodeProps {- data: ScannerNodeData+export interface TransformNodeProps {+ data: TransformNodeDataisConnectable?: booleanselected?: boolean}-// ================================-// TRANSFORM TYPE DEFINITIONS-// ================================--export interface Transform {- id: string;- class_name: string;- name: string;- module: string;- description: string;- documentation: string;- category: string;-}-// ================================// TRANSFORM DATA STRUCTURES// ================================diff --git a/flowsint-core/poetry.lock b/flowsint-core/poetry.lockindex 0cb2523e..d253d797 100644--- a/flowsint-core/poetry.lock+++ b/flowsint-core/poetry.lock@@ -1781,7 +1781,7 @@ version = "2.1.0"description = "brain-dead simple config-ini parsing"optional = falsepython-versions = ">=3.8"-groups = ["dev"]+groups = ["main", "dev"]files = [{file = "iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760"},{file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"},@@ -3194,7 +3194,7 @@ version = "1.6.0"description = "plugin and hook calling mechanisms for python"optional = falsepython-versions = ">=3.9"-groups = ["dev"]+groups = ["main", "dev"]files = [{file = "pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"},{file = "pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3"},@@ -3778,7 +3778,7 @@ version = "2.19.2"description = "Pygments is a syntax highlighting package written in Python."optional = falsepython-versions = ">=3.8"-groups = ["main"]+groups = ["main", "dev"]files = [{file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"},{file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"},@@ -3925,24 +3925,25 @@ files = [[[package]]name = "pytest"-version = "7.4.4"+version = "8.4.2"description = "pytest: simple powerful testing with Python"optional = false-python-versions = ">=3.7"-groups = ["dev"]+python-versions = ">=3.9"+groups = ["main", "dev"]files = [- {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"},- {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"},+ {file = "pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79"},+ {file = "pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01"},][package.dependencies]-colorama = {version = "*", markers = "sys_platform == \"win32\""}-iniconfig = "*"-packaging = "*"-pluggy = ">=0.12,<2.0"+colorama = {version = ">=0.4", markers = "sys_platform == \"win32\""}+iniconfig = ">=1"+packaging = ">=20"+pluggy = ">=1.5,<2"+pygments = ">=2.7.2"[package.extras]-testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"]+dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "requests", "setuptools", "xmlschema"][[package]]name = "pytest-asyncio"@@ -5690,4 +5691,4 @@ propcache = ">=0.2.1"[metadata]lock-version = "2.1"python-versions = ">=3.11,<3.13"-content-hash = "481c8b5a48b2d2b3a0e94c4078b7998597f787294de986eca9bed0ece0778e9e"+content-hash = "ad8083b0650095db07fe8df95331cbf55aa672c6d72657f6bfcf51d084fcffda"diff --git a/flowsint-core/pyproject.toml b/flowsint-core/pyproject.tomlindex 10662a42..07b726ca 100644--- a/flowsint-core/pyproject.toml+++ b/flowsint-core/pyproject.toml@@ -25,9 +25,9 @@ alembic = "1.13.0"phonenumbers = "^9.0.8"python-multipart = "^0.0.20"docker = "^7.1.0"+pytest = "^8.4.2"[tool.poetry.group.dev.dependencies]-pytest = "^7.4"pytest-asyncio = "^0.21"black = "^23.0"isort = "^5.12"diff --git a/flowsint-core/src/flowsint_core/core/enums.py b/flowsint-core/src/flowsint_core/core/enums.pyindex 49471c73..a5c395e7 100644--- a/flowsint-core/src/flowsint_core/core/enums.py+++ b/flowsint-core/src/flowsint_core/core/enums.py@@ -8,7 +8,7 @@ class EventLevel(str, Enum):FAILED = "FAILED"SUCCESS = "SUCCESS"DEBUG = "DEBUG"- # Scanner-specific statuses+ # Transform-specific statusesPENDING = "PENDING"RUNNING = "RUNNING"COMPLETED = "COMPLETED"diff --git a/flowsint-core/src/flowsint_core/core/orchestrator.py b/flowsint-core/src/flowsint_core/core/orchestrator.pyindex 46081833..4797e0ac 100644--- a/flowsint-core/src/flowsint_core/core/orchestrator.py+++ b/flowsint-core/src/flowsint_core/core/orchestrator.py@@ -2,7 +2,7 @@ from typing import List, Dict, Anyfrom datetime import datetimeimport timefrom pydantic import ValidationError-from .scanner_base import Scanner+from .transform_base import Transformfrom .registry import TransformRegistryfrom .types import FlowBranch, FlowStepfrom .logger import Logger@@ -12,9 +12,9 @@ import jsonimport os-class FlowOrchestrator(Scanner):+class FlowOrchestrator(Transform):"""- Orchestrator for running a list of scanners.+ Orchestrator for running a list of transforms."""def __init__(@@ -27,10 +27,10 @@ class FlowOrchestrator(Scanner):):super().__init__(sketch_id, scan_id, neo4j_conn=neo4j_conn, vault=vault)self.transform_branches = transform_branches- self.scanners = {} # Map of nodeId -> scanner instance+ self.transforms = {} # Map of nodeId -> transform instanceself.execution_log_file = None # Path to the execution log fileself._create_execution_log()- self._load_scanners()+ self._load_transforms()def _create_execution_log(self) -> None:"""@@ -210,44 +210,44 @@ class FlowOrchestrator(Scanner):{"message": f"Failed to save transform branches: {str(e)}"},)- def _load_scanners(self) -> None:+ def _load_transforms(self) -> None:if not self.transform_branches:raise ValueError("No transform branches provided")- # Collect all scanner nodes across all branches- scanner_nodes = []+ # Collect all transform nodes across all branches+ transform_nodes = []for branch in self.transform_branches:for step in branch.steps:if step.type == "type":continue- scanner_nodes.append(step)+ transform_nodes.append(step)- if not scanner_nodes:- raise ValueError("No scanner nodes found in transform branches")+ if not transform_nodes:+ raise ValueError("No transform nodes found in transform branches")- # Create scanner instances for each node- for node in scanner_nodes:+ # Create transform instances for each node+ for node in transform_nodes:node_id = node.nodeId- # Extract scanner name from nodeId (assuming format like "transform_name-1234567890")+ # Extract transform name from nodeId (assuming format like "transform_name-1234567890")transform_name = node_id.split("-")[0]if not TransformRegistry.transform_exists(transform_name):- raise ValueError(f"Scanner '{transform_name}' not found in registry")+ raise ValueError(f"Transform '{transform_name}' not found in registry")- # Pass the step params to the scanner instance- scanner_params = (+ # Pass the step params to the transform instance+ transform_params = (node.params if hasattr(node, "params") and node.params else {})- scanner = TransformRegistry.get_scanner(+ transform = TransformRegistry.get_transform(transform_name,self.sketch_id,self.scan_id,neo4j_conn=self.neo4j_conn,vault=self.vault,- params=scanner_params,+ params=transform_params,)- self.scanners[node_id] = scanner+ self.transforms[node_id] = transformdef resolve_reference(self, ref_value: str, results_mapping: Dict[str, Any]) -> Any:"""@@ -258,11 +258,11 @@ class FlowOrchestrator(Scanner):return results_mapping[ref_value]return None- def prepare_scanner_inputs(+ def prepare_transform_inputs(self, step: FlowStep, results_mapping: Dict[str, Any], initial_values: List[str]) -> List[Any]:"""- Prepare the inputs for a scanner based on the references and previous results.+ Prepare the inputs for a transform based on the references and previous results.Handles single references, lists, and direct values."""inputs = {}@@ -290,12 +290,12 @@ class FlowOrchestrator(Scanner):# Si aucun input n'a été résolu, utiliser les valeurs initialesif not inputs:- scanner = self.scanners.get(step.nodeId)- if scanner:- primary_key = scanner.key()+ transform = self.transforms.get(step.nodeId)+ if transform:+ primary_key = transform.key()return {primary_key: initial_values}else:- scanner = self.scanners.get(step.nodeId)+ transform = self.transforms.get(step.nodeId)return inputs[input_key]def update_results_mapping(@@ -305,7 +305,7 @@ class FlowOrchestrator(Scanner):results_mapping: Dict[str, Any],) -> None:"""- Update the results mapping with new outputs from a scanner.+ Update the results mapping with new outputs from a transform."""for output_key, output_ref in step_outputs.items():if output_key in outputs:@@ -355,8 +355,8 @@ class FlowOrchestrator(Scanner):# Global mapping of output references to actual valuesresults_mapping = {}- # Cache for scanner results to avoid recomputation- scanner_results_cache = {}+ # Cache for transform results to avoid recomputation+ transform_results_cache = {}total_steps = sum(len(branch.steps) for branch in self.transform_branches)completed_steps = 0@@ -368,27 +368,27 @@ class FlowOrchestrator(Scanner):branch_results = {"id": branch_id, "name": branch_name, "steps": []}# Process each step in the branch- scanner_inputs = values+ transform_inputs = valuesfor step in branch.steps:if step.type == "type":continuenode_id = step.nodeId- scanner = self.scanners.get(node_id)+ transform = self.transforms.get(node_id)- if not scanner:+ if not transform:Logger.error(self.sketch_id,- {"message": f"Scanner not found for node {node_id}"},+ {"message": f"Transform not found for node {node_id}"},)continue- transform_name = scanner.name()+ transform_name = transform.name()step_start_time = time.time()step_result = {"nodeId": node_id,- "scanner": transform_name,+ "transform": transform_name,"status": "error", # Default to error, will update on success}@@ -399,7 +399,7 @@ class FlowOrchestrator(Scanner):"branch_name": branch_name,"node_id": node_id,"transform_name": transform_name,- "inputs": to_json_serializable(scanner_inputs),+ "inputs": to_json_serializable(transform_inputs),"outputs": None,"status": "running","error": None,@@ -410,7 +410,7 @@ class FlowOrchestrator(Scanner):try:# Update status for current stepcompleted_steps += 1- if not scanner_inputs:+ if not transform_inputs:error_msg = "No inputs available"step_result["error"] = error_msglog_entry["status"] = "error"@@ -422,20 +422,20 @@ class FlowOrchestrator(Scanner):branch_results["steps"].append(step_result)continue- # Check if we already have results for this scanner with these inputs- cache_key = f"{node_id}:{str(scanner_inputs)}"- if cache_key in scanner_results_cache:- outputs = scanner_results_cache[cache_key]+ # Check if we already have results for this transform with these inputs+ cache_key = f"{node_id}:{str(transform_inputs)}"+ if cache_key in transform_results_cache:+ outputs = transform_results_cache[cache_key]log_entry["cache_hit"] = Trueelse:- # Execute the scanner- outputs = await scanner.execute(scanner_inputs)+ # Execute the transform+ outputs = await transform.execute(transform_inputs)if not isinstance(outputs, (dict, list)):raise ValueError(- f"Scanner '{transform_name}' returned unsupported output format"+ f"Transform '{transform_name}' returned unsupported output format")# Cache the results- scanner_results_cache[cache_key] = outputs+ transform_results_cache[cache_key] = outputslog_entry["cache_hit"] = False# Store the outputs in the step result (serialize to avoid JSON issues)@@ -453,7 +453,7 @@ class FlowOrchestrator(Scanner):self.update_results_mapping(outputs, step.outputs, results_mapping)# Also store the raw outputs in the main resultsresults["results"][node_id] = outputs- scanner_inputs = outputs+ transform_inputs = outputsexcept ValidationError as e:error_msg = f"Validation error: {str(e)}"diff --git a/flowsint-core/src/flowsint_core/core/registry.py b/flowsint-core/src/flowsint_core/core/registry.pyindex 1d92f87a..bb09a4a8 100644--- a/flowsint-core/src/flowsint_core/core/registry.py+++ b/flowsint-core/src/flowsint_core/core/registry.py@@ -1,117 +1,117 @@import inspectfrom typing import Dict, Optional, Type, List-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transform-# Domain-related scanners-from flowsint_transforms.domain.to_subdomains import SubdomainScanner-from flowsint_transforms.domain.to_whois import WhoisScanner-from flowsint_transforms.domain.to_ip import ResolveScanner-from flowsint_transforms.domain.to_website import DomainToWebsiteScanner+# Domain-related transforms+from flowsint_transforms.domain.to_subdomains import SubdomainTransform+from flowsint_transforms.domain.to_whois import WhoisTransform+from flowsint_transforms.domain.to_ip import ResolveTransform+from flowsint_transforms.domain.to_website import DomainToWebsiteTransformfrom flowsint_transforms.domain.to_root_domain import DomainToRootDomain-from flowsint_transforms.domain.to_asn import DomainToAsnScanner-from flowsint_transforms.domain.to_history import DomainToHistoryScanner+from flowsint_transforms.domain.to_asn import DomainToAsnTransform+from flowsint_transforms.domain.to_history import DomainToHistoryTransform-# IP-related scanners-from flowsint_transforms.email.to_domains import EmailToDomainsScanner-from flowsint_transforms.individual.to_domains import IndividualToDomainsScanner-from flowsint_transforms.ip.to_domain import ReverseResolveScanner-from flowsint_transforms.ip.to_infos import IpToInfosScanner-from flowsint_transforms.ip.to_asn import IpToAsnScanner+# IP-related transforms+from flowsint_transforms.email.to_domains import EmailToDomainsTransform+from flowsint_transforms.individual.to_domains import IndividualToDomainsTransform+from flowsint_transforms.ip.to_domain import ReverseResolveTransform+from flowsint_transforms.ip.to_infos import IpToInfosTransform+from flowsint_transforms.ip.to_asn import IpToAsnTransform-# ASN-related scanners-from flowsint_transforms.asn.to_cidrs import AsnToCidrsScanner+# ASN-related transforms+from flowsint_transforms.asn.to_cidrs import AsnToCidrsTransform-# CIDR-related scanners-from flowsint_transforms.cidr.to_ips import CidrToIpsScanner+# CIDR-related transforms+from flowsint_transforms.cidr.to_ips import CidrToIpsTransform-# Social media scanners-from flowsint_transforms.organization.to_domains import OrgToDomainsScanner-from flowsint_transforms.social.to_maigret import MaigretScanner+# Social media transforms+from flowsint_transforms.organization.to_domains import OrgToDomainsTransform+from flowsint_transforms.social.to_maigret import MaigretTransform-# Organization-related scanners-from flowsint_transforms.organization.to_asn import OrgToAsnScanner-from flowsint_transforms.organization.to_infos import OrgToInfosScanner+# Organization-related transforms+from flowsint_transforms.organization.to_asn import OrgToAsnTransform+from flowsint_transforms.organization.to_infos import OrgToInfosTransform-# Cryptocurrency scanners+# Cryptocurrency transformsfrom flowsint_transforms.crypto.to_transactions import (CryptoWalletAddressToTransactions,)from flowsint_transforms.crypto.to_nfts import CryptoWalletAddressToNFTs-# Website-related scanners+# Website-related transformsfrom flowsint_transforms.website.to_crawler import WebsiteToCrawlerfrom flowsint_transforms.website.to_links import WebsiteToLinks-from flowsint_transforms.website.to_domain import WebsiteToDomainScanner+from flowsint_transforms.website.to_domain import WebsiteToDomainTransformfrom flowsint_transforms.website.to_text import WebsiteToText-from flowsint_transforms.website.to_webtrackers import WebsiteToWebtrackersScanner+from flowsint_transforms.website.to_webtrackers import WebsiteToWebtrackersTransform-# Email-related scanners-from flowsint_transforms.email.to_gravatar import EmailToGravatarScanner-from flowsint_transforms.email.to_leaks import EmailToBreachesScanner+# Email-related transforms+from flowsint_transforms.email.to_gravatar import EmailToGravatarTransform+from flowsint_transforms.email.to_leaks import EmailToBreachesTransform-# Phone-related scanners-from flowsint_transforms.phone.to_leaks import PhoneToBreachesScanner+# Phone-related transforms+from flowsint_transforms.phone.to_leaks import PhoneToBreachesTransform-# Individual-related scanners-from flowsint_transforms.individual.to_org import IndividualToOrgScanner+# Individual-related transforms+from flowsint_transforms.individual.to_org import IndividualToOrgTransform-# Integration scanners+# Integration transformsfrom flowsint_transforms.n8n.connector import N8nConnectorclass TransformRegistry:- _scanners: Dict[str, Type[Scanner]] = {}+ _transforms: Dict[str, Type[Transform]] = {}@classmethod- def register(cls, scanner_class: Type[Scanner]) -> None:- cls._scanners[scanner_class.name()] = scanner_class+ def register(cls, transform_class: Type[Transform]) -> None:+ cls._transforms[transform_class.name()] = transform_class@classmethoddef transform_exists(cls, name: str) -> bool:- return name in cls._scanners+ return name in cls._transforms@classmethod- def get_scanner(cls, name: str, sketch_id: str, scan_id: str, **kwargs) -> Scanner:- if name not in cls._scanners:- raise Exception(f"Scanner '{name}' not found")- return cls._scanners[name](sketch_id=sketch_id, scan_id=scan_id, **kwargs)+ def get_transform(cls, name: str, sketch_id: str, scan_id: str, **kwargs) -> Transform:+ if name not in cls._transforms:+ raise Exception(f"Transform '{name}' not found")+ return cls._transforms[name](sketch_id=sketch_id, scan_id=scan_id, **kwargs)@classmethod- def _create_scanner_metadata(cls, scanner: Type[Scanner]) -> Dict[str, str]:- """Helper method to create scanner metadata dictionary."""+ def _create_transform_metadata(cls, transform: Type[Transform]) -> Dict[str, str]:+ """Helper method to create transform metadata dictionary."""return {- "class_name": scanner.__name__,- "name": scanner.name(),- "module": scanner.__module__,- "description": scanner.__doc__,- "documentation": inspect.cleandoc(scanner.documentation()),- "category": scanner.category(),- "inputs": scanner.input_schema(),- "outputs": scanner.output_schema(),+ "class_name": transform.__name__,+ "name": transform.name(),+ "module": transform.__module__,+ "description": transform.__doc__,+ "documentation": inspect.cleandoc(transform.documentation()),+ "category": transform.category(),+ "inputs": transform.input_schema(),+ "outputs": transform.output_schema(),"params": {},- "params_schema": scanner.get_params_schema(),- "required_params": scanner.required_params(),- "icon": scanner.icon(),+ "params_schema": transform.get_params_schema(),+ "required_params": transform.required_params(),+ "icon": transform.icon(),}@classmethoddef list(cls, exclude: Optional[List[str]] = []) -> Dict[str, Dict[str, str]]:return {- name: cls._create_scanner_metadata(scanner)- for name, scanner in cls._scanners.items()+ name: cls._create_transform_metadata(transform)+ for name, transform in cls._transforms.items()if name not in exclude}@classmethoddef list_by_categories(cls) -> Dict[str, List[Dict[str, str]]]:- scanners_by_category = {}- for _, scanner in cls._scanners.items():- category = scanner.category()- if category not in scanners_by_category:- scanners_by_category[category] = []- scanners_by_category[category].append(cls._create_scanner_metadata(scanner))- return scanners_by_category+ transforms_by_category = {}+ for _, transform in cls._transforms.items():+ category = transform.category()+ if category not in transforms_by_category:+ transforms_by_category[category] = []+ transforms_by_category[category].append(cls._create_transform_metadata(transform))+ return transforms_by_category@classmethoddef list_by_input_type(@@ -121,70 +121,70 @@ class TransformRegistry:if input_type_lower == "any":return [- cls._create_scanner_metadata(scanner)- for scanner in cls._scanners.values()- if scanner.name() not in exclude+ cls._create_transform_metadata(transform)+ for transform in cls._transforms.values()+ if transform.name() not in exclude]return [- cls._create_scanner_metadata(scanner)- for scanner in cls._scanners.values()- if scanner.input_schema()["type"].lower() in ["any", input_type_lower]- and scanner.name() not in exclude+ cls._create_transform_metadata(transform)+ for transform in cls._transforms.values()+ if transform.input_schema()["type"].lower() in ["any", input_type_lower]+ and transform.name() not in exclude]-# Register all scanners+# Register all transforms-# Domain-related scanners-TransformRegistry.register(ReverseResolveScanner)-TransformRegistry.register(ResolveScanner)-TransformRegistry.register(SubdomainScanner)-TransformRegistry.register(WhoisScanner)-TransformRegistry.register(DomainToWebsiteScanner)+# Domain-related transforms+TransformRegistry.register(ReverseResolveTransform)+TransformRegistry.register(ResolveTransform)+TransformRegistry.register(SubdomainTransform)+TransformRegistry.register(WhoisTransform)+TransformRegistry.register(DomainToWebsiteTransform)TransformRegistry.register(DomainToRootDomain)-TransformRegistry.register(DomainToAsnScanner)-TransformRegistry.register(DomainToHistoryScanner)+TransformRegistry.register(DomainToAsnTransform)+TransformRegistry.register(DomainToHistoryTransform)-# IP-related scanners-TransformRegistry.register(IpToInfosScanner)-TransformRegistry.register(IpToAsnScanner)+# IP-related transforms+TransformRegistry.register(IpToInfosTransform)+TransformRegistry.register(IpToAsnTransform)-# ASN-related scanners-TransformRegistry.register(AsnToCidrsScanner)+# ASN-related transforms+TransformRegistry.register(AsnToCidrsTransform)-# CIDR-related scanners-TransformRegistry.register(CidrToIpsScanner)+# CIDR-related transforms+TransformRegistry.register(CidrToIpsTransform)-# Social media scanners-TransformRegistry.register(MaigretScanner)+# Social media transforms+TransformRegistry.register(MaigretTransform)-# Organization-related scanners-TransformRegistry.register(OrgToAsnScanner)-TransformRegistry.register(OrgToInfosScanner)-TransformRegistry.register(OrgToDomainsScanner)-# Cryptocurrency scanners+# Organization-related transforms+TransformRegistry.register(OrgToAsnTransform)+TransformRegistry.register(OrgToInfosTransform)+TransformRegistry.register(OrgToDomainsTransform)+# Cryptocurrency transformsTransformRegistry.register(CryptoWalletAddressToTransactions)TransformRegistry.register(CryptoWalletAddressToNFTs)-# Website-related scanners+# Website-related transformsTransformRegistry.register(WebsiteToCrawler)TransformRegistry.register(WebsiteToLinks)-TransformRegistry.register(WebsiteToDomainScanner)-TransformRegistry.register(WebsiteToWebtrackersScanner)+TransformRegistry.register(WebsiteToDomainTransform)+TransformRegistry.register(WebsiteToWebtrackersTransform)TransformRegistry.register(WebsiteToText)-# Email-related scanners-TransformRegistry.register(EmailToGravatarScanner)-TransformRegistry.register(EmailToBreachesScanner)-TransformRegistry.register(EmailToDomainsScanner)+# Email-related transforms+TransformRegistry.register(EmailToGravatarTransform)+TransformRegistry.register(EmailToBreachesTransform)+TransformRegistry.register(EmailToDomainsTransform)-# Phone-related scanners-TransformRegistry.register(PhoneToBreachesScanner)+# Phone-related transforms+TransformRegistry.register(PhoneToBreachesTransform)-# Individual-related scanners-TransformRegistry.register(IndividualToOrgScanner)-TransformRegistry.register(IndividualToDomainsScanner)+# Individual-related transforms+TransformRegistry.register(IndividualToOrgTransform)+TransformRegistry.register(IndividualToDomainsTransform)-# Integration scanners+# Integration transformsTransformRegistry.register(N8nConnector)diff --git a/flowsint-core/src/flowsint_core/core/scanner_base.py b/flowsint-core/src/flowsint_core/core/transform_base.pysimilarity index 94%rename from flowsint-core/src/flowsint_core/core/scanner_base.pyrename to flowsint-core/src/flowsint_core/core/transform_base.pyindex c4bbf186..71f99237 100644--- a/flowsint-core/src/flowsint_core/core/scanner_base.py+++ b/flowsint-core/src/flowsint_core/core/transform_base.py@@ -8,7 +8,7 @@ from .vault import VaultProtocolfrom ..utils import resolve_type-class InvalidScannerParams(Exception):+class InvalidTransformParams(Exception):pass@@ -34,13 +34,13 @@ def build_params_model(params_schema: list) -> BaseModel:return model-class Scanner(ABC):+class Transform(ABC):"""- Abstract base class for all scanners.+ Abstract base class for all transforms.## InputType and OutputType Pattern- Scanners only need to define InputType and OutputType as class attributes.+ Transforms only need to define InputType and OutputType as class attributes.The base class automatically handles schema generation:```python@@ -48,14 +48,14 @@ class Scanner(ABC):from flowsint_types import Domainfrom flowsint_types import Ip- class MyScanner(Scanner):+ class MyTransform(Transform):# Define types as class attributesInputType = List[Domain]OutputType = List[Ip]@classmethoddef name(cls):- return "my_scanner"+ return "my_transform"@classmethoddef category(cls):@@ -76,15 +76,15 @@ class Scanner(ABC):return results# Make types available at module level for easy access- InputType = MyScanner.InputType- OutputType = MyScanner.OutputType+ InputType = MyTransform.InputType+ OutputType = MyTransform.OutputType```The base class automatically provides:- input_schema() method using InputType- output_schema() method using OutputType- Error handling for missing type definitions- - Consistent schema generation across all scanners+ - Consistent schema generation across all transformsSubclasses can override input_schema() or output_schema() if needed for special cases."""@@ -128,8 +128,8 @@ class Scanner(ABC):validated = self.ParamsModel(**resolved_params)self.params = validated.model_dump()except ValidationError as e:- raise InvalidScannerParams(- f"Scanner '{self.name()}' received invalid parameters: {e}"+ raise InvalidTransformParams(+ f"Transform '{self.name()}' received invalid parameters: {e}")def resolve_params(self) -> Dict[str, Any]:@@ -192,13 +192,13 @@ class Scanner(ABC):@classmethod@abstractmethoddef key(cls) -> str:- """Primary key on which the scanner operates (e.g. domain, IP, etc.)"""+ """Primary key on which the transform operates (e.g. domain, IP, etc.)"""pass@classmethoddef documentation(cls) -> str:"""- Return formatted markdown documentation for this scanner.+ Return formatted markdown documentation for this transform.Override this method to provide custom documentation.Falls back to cleaned docstring if not overridden."""@@ -327,7 +327,7 @@ class Scanner(ABC):async def execute(self, values: List[str]) -> List[Dict[str, Any]]:if self.name() != "transform_orchestrator":- Logger.info(self.sketch_id, {"message": f"Scanner {self.name()} started."})+ Logger.info(self.sketch_id, {"message": f"Transform {self.name()} started."})try:await self.async_init()preprocessed = self.preprocess(values)@@ -336,7 +336,7 @@ class Scanner(ABC):if self.name() != "transform_orchestrator":Logger.completed(- self.sketch_id, {"message": f"Scanner {self.name()} finished."}+ self.sketch_id, {"message": f"Transform {self.name()} finished."})return processed@@ -345,7 +345,7 @@ class Scanner(ABC):if self.name() != "transform_orchestrator":Logger.error(self.sketch_id,- {"message": f"Scanner {self.name()} errored: '{str(e)}'."},+ {"message": f"Transform {self.name()} errored: '{str(e)}'."},)return []@@ -373,7 +373,6 @@ class Scanner(ABC):final_properties["type"] = node_type.lower()final_properties["sketch_id"] = self.sketch_idfinal_properties["label"] = final_properties.get("label", key_value)-set_clauses = [f"n.{prop} = ${prop}" for prop in final_properties.keys()]params = {key_prop: key_value, **final_properties}diff --git a/flowsint-core/src/flowsint_core/core/types.py b/flowsint-core/src/flowsint_core/core/types.pyindex c32c0fe7..1f2d0edd 100644--- a/flowsint-core/src/flowsint_core/core/types.py+++ b/flowsint-core/src/flowsint_core/core/types.py@@ -56,9 +56,9 @@ class FlowStep(BaseModel):params: Optional[Dict[str, Any]] = Field(None, description="Parameters for the step", title="Parameters")- type: Literal["type", "scanner"] = Field(+ type: Literal["type", "transform"] = Field(...,- description="Type of step - either type transformation or scanner",+ description="Type of step - either type transformation or transform",title="Step Type",)inputs: Dict[str, Any] = Field(diff --git a/flowsint-core/src/flowsint_core/core/vault.py b/flowsint-core/src/flowsint_core/core/vault.pyindex 10f51e91..2d96f6ce 100644--- a/flowsint-core/src/flowsint_core/core/vault.py+++ b/flowsint-core/src/flowsint_core/core/vault.py@@ -15,6 +15,17 @@ class Vault(VaultProtocol):raise ValueError("owner_id is required to use the vault.")self.db = dbself.owner_id = owner_id++ def set_secret(self, vault_ref, plain_key) -> None:+ try:+ ref_uuid = uuid.UUID(vault_ref)+ stmt = select(Key).where(Key.id == ref_uuid)+ except ValueError:+ stmt = select(Key).where(Key.name == vault_ref)+ stmt = stmt.where(Key.owner_id == self.owner_id)+ result = self.db.execute(stmt)+ row = result.scalars().first()+ return row.encrypted_key if row else Nonedef get_secret(self, vault_ref: str) -> Optional[str]:try:diff --git a/flowsint-core/src/flowsint_core/tasks/flow.py b/flowsint-core/src/flowsint_core/tasks/flow.pyindex f4bd6d76..0b718f22 100644--- a/flowsint-core/src/flowsint_core/tasks/flow.py+++ b/flowsint-core/src/flowsint_core/tasks/flow.py@@ -62,7 +62,7 @@ def run_flow()transform_branches = [FlowBranch(**branch) for branch in transform_branches]- scanner = FlowOrchestrator(+ transform = FlowOrchestrator(sketch_id=sketch_id,scan_id=str(scan_id),transform_branches=transform_branches,@@ -71,7 +71,7 @@ def run_flow()# Use the synchronous scan method which internally handles the async operations- results = scanner.scan(values=values)+ results = transform.scan(values=values)scan.status = EventLevel.COMPLETEDscan.results = to_json_serializable(results)diff --git a/flowsint-core/src/flowsint_core/tasks/transform.py b/flowsint-core/src/flowsint_core/tasks/transform.pyindex ade1241f..4a714bb6 100644--- a/flowsint-core/src/flowsint_core/tasks/transform.py+++ b/flowsint-core/src/flowsint_core/tasks/transform.py@@ -60,9 +60,9 @@ def run_transform()if not TransformRegistry.transform_exists(transform_name):- raise ValueError(f"Scanner '{transform_name}' not found in registry")+ raise ValueError(f"Transform '{transform_name}' not found in registry")- scanner = TransformRegistry.get_scanner(+ transform = TransformRegistry.get_transform(name=transform_name,sketch_id=sketch_id,scan_id=scan_id,@@ -70,7 +70,7 @@ def run_transform(vault=vault,)- results = asyncio.run(scanner.execute(values=values))+ results = asyncio.run(transform.execute(values=values))scan.status = EventLevel.COMPLETEDscan.results = to_json_serializable(results)diff --git a/flowsint-core/src/flowsint_core/tests/fixtures/database.py b/flowsint-core/src/flowsint_core/tests/fixtures/database.pydeleted file mode 100644index dac66630..00000000--- a/flowsint-core/src/flowsint_core/tests/fixtures/database.py+++ /dev/null@@ -1,132 +0,0 @@-import pytest-import uuid-from unittest.mock import Mock-from sqlalchemy import create_engine-from sqlalchemy.orm import sessionmaker-from app.models.models import Profile, Key-from flowsint_core.core.vault import Vault--[email protected](scope="function")-def test_db_session():- """Create an in-memory SQLite database for testing"""- # Create in-memory SQLite database- engine = create_engine("sqlite:///:memory:", echo=False)-- # Only create specific tables we need for testing (to avoid ARRAY type issues)- Profile.__table__.create(engine)- Key.__table__.create(engine)-- # Create session- TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)- session = TestingSessionLocal()-- yield session-- # Clean up- session.close()- engine.dispose()--[email protected](scope="function")-def test_owner_id():- """Generate a test owner ID"""- return uuid.uuid4()--[email protected](scope="function")-def test_profile(test_db_session, test_owner_id):- """Create a test profile in the database"""- profile = Profile(- id=test_owner_id,- email=f"test_{uuid.uuid4().hex[:8]}@example.com",- hashed_password="hashed_test_password",- is_active=True,- first_name="Test",- last_name="User",- )-- test_db_session.add(profile)- test_db_session.commit()- test_db_session.refresh(profile)-- return profile--[email protected](scope="function")-def test_api_key(test_db_session, test_owner_id):- """Create a test API key in the database"""- key = Key(- id=uuid.uuid4(),- name="test_etherscan_key",- owner_id=test_owner_id,- encrypted_key="test_api_key_12345",- )-- test_db_session.add(key)- test_db_session.commit()- test_db_session.refresh(key)-- return key--[email protected](scope="function")-def test_vault(test_db_session, test_owner_id, test_profile, test_api_key):- """Create a test vault instance with test data"""- return Vault(db=test_db_session, owner_id=test_owner_id)--[email protected](scope="function")-def mock_logger():- """Create a mock logger for testing"""- logger = Mock()- logger.info = Mock()- logger.error = Mock()- logger.warn = Mock()- logger.debug = Mock()- logger.success = Mock()- return logger--[email protected](scope="function")-def test_sketch_scan_ids():- """Generate test sketch and scan IDs"""- return {"sketch_id": str(uuid.uuid4()), "scan_id": str(uuid.uuid4())}--[email protected](scope="function")-def multiple_test_keys(test_db_session, test_owner_id):- """Create multiple test API keys for comprehensive testing"""- keys = []-- # Create keys with different names- key_configs = [- {"name": "etherscan_api_key", "value": "test_etherscan_key_123"},- {"name": "another_service_key", "value": "test_another_key_456"},- {"name": "vaultKey1", "value": "test_vault_key_789"},- ]-- for config in key_configs:- key = Key(- id=uuid.uuid4(),- name=config["name"],- owner_id=test_owner_id,- encrypted_key=config["value"],- )- test_db_session.add(key)- keys.append(key)-- test_db_session.commit()-- for key in keys:- test_db_session.refresh(key)-- return keys--[email protected](scope="function")-def test_vault_with_multiple_keys(- test_db_session, test_owner_id, test_profile, multiple_test_keys-):- """Create a test vault with multiple keys for comprehensive testing"""- return Vault(db=test_db_session, owner_id=test_owner_id)diff --git a/flowsint-core/src/flowsint_core/tests/orchestrator.py b/flowsint-core/src/flowsint_core/tests/orchestrator.pydeleted file mode 100644index 3c851684..00000000--- a/flowsint-core/src/flowsint_core/tests/orchestrator.py+++ /dev/null@@ -1,111 +0,0 @@-from flowsint_core.core.orchestrator import FlowOrchestrator-from flowsint_types.domain import Domain, Domain-from flowsint_types.whois import Whois--import pytest---def test_preprocess_valid_domains():- scanner = FlowOrchestrator(- "123", transform_names=["domain_resolve_scanner", "domain_whois_scanner"]- )- assert isinstance(scanner.scanners, list)- assert len(scanner.scanners) == 2---def test_preprocess_valid_domains():- with pytest.raises(- ValueError, match="Scanner 'this_scan_is_wrong' not found in registry"- ):- FlowOrchestrator(- "123", transform_names=["domain_resolve_scanner", "this_scan_is_wrong"]- )---def test_execute_domain_subdomains_scanner():- domains = ["example.com"]- scanner = FlowOrchestrator("123", transform_names=["domain_subdomains_scanner"])- results = scanner.execute(values=domains)- assert results["initial_values"] == domains- assert results["scanners"] == ["domain_subdomains_scanner"]- assert isinstance(results["results"]["domain_subdomains_scanner"], list)- assert isinstance(results["results"]["domain_subdomains_scanner"][0], Domain)---def test_execute_domain_whois_scanner():- domains = ["example.com"]- scanner = FlowOrchestrator("123", transform_names=["domain_whois_scanner"])- results = scanner.execute(values=domains)- assert results["initial_values"] == domains- assert results["scanners"] == ["domain_whois_scanner"]- assert isinstance(results["results"]["domain_whois_scanner"], list)- assert isinstance(results["results"]["domain_whois_scanner"][0], Whois)---def test_execute_ip_resolve():- ips = ["91.199.212.73"]- scanner = FlowOrchestrator("123", transform_names=["ip_reverse_resolve_scanner"])- results = scanner.execute(values=ips)- assert results["initial_values"] == ips- assert results["scanners"] == ["ip_reverse_resolve_scanner"]- assert isinstance(results["results"]["ip_reverse_resolve_scanner"], list)- if len(results["results"]["ip_reverse_resolve_scanner"]) != 0:- assert isinstance(results["results"]["ip_reverse_resolve_scanner"][0], Domain)---def test_execute_ip_resolve_and_whois():- ips = ["91.199.212.73"]- scanner = FlowOrchestrator(- "123", transform_names=["ip_reverse_resolve_scanner", "domain_whois_scanner"]- )- results = scanner.execute(values=ips)- assert results["initial_values"] == ips- assert results["scanners"] == ["ip_reverse_resolve_scanner", "domain_whois_scanner"]---def test_execute_ip_resolve_and_whois_multiple():- ips = ["91.199.212.73", "76.76.21.21"]- scanner = FlowOrchestrator(- "123", transform_names=["ip_reverse_resolve_scanner", "domain_whois_scanner"]- )- results = scanner.execute(values=ips)- assert results["initial_values"] == ips- assert results["scanners"] == ["ip_reverse_resolve_scanner", "domain_whois_scanner"]- print(results)---def test_execute_ip_resolve_and_whois_multiple():- ips = ["162.19.81.222"]- scanner = FlowOrchestrator(- "123",- transform_names=[- "ip_reverse_resolve_scanner",- "domain_whois_scanner",- "domain_subdomains_scanner",- ],- )- results = scanner.execute(values=ips)- assert results["initial_values"] == ips- assert results["scanners"] == [- "ip_reverse_resolve_scanner",- "domain_whois_scanner",- "domain_subdomains_scanner",- ]---def test_execute_domain_whois_and_subdomains():- domains = ["alliage.io"]- scanner = FlowOrchestrator(- "123", transform_names=["domain_whois_scanner", "domain_subdomains_scanner"]- )- results = scanner.execute(values=domains)- assert results["initial_values"] == domains- assert results["scanners"] == ["domain_whois_scanner", "domain_subdomains_scanner"]---def test_execute_domain_subdomains():- domains = ["alliage.io"]- scanner = FlowOrchestrator("123", transform_names=["domain_subdomains_scanner"])- results = scanner.execute(values=domains)- assert results["initial_values"] == domains- assert results["scanners"] == ["domain_subdomains_scanner"]diff --git a/flowsint-core/src/flowsint_core/tests/test_base_scanner.py b/flowsint-core/src/flowsint_core/tests/test_base_scanner.pydeleted file mode 100644index 7d2078c3..00000000--- a/flowsint-core/src/flowsint_core/tests/test_base_scanner.py+++ /dev/null@@ -1,203 +0,0 @@-import pytest-from typing import List, Dict, Any-from ..core.scanner_base import Scanner-from flowsint_types.domain import Domain-from flowsint_types.ip import Ip---class MockScanner(Scanner):- """Mock scanner for testing base functionality"""-- # Define InputType and OutputType- InputType = List[Domain]- OutputType = List[Ip]-- @classmethod- def name(cls) -> str:- return "mock_scanner"-- @classmethod- def category(cls) -> str:- return "Test"-- @classmethod- def key(cls) -> str:- return "domain"-- @classmethod- def input_schema(cls) -> Dict[str, Any]:- return cls.generate_input_schema()-- @classmethod- def output_schema(cls) -> Dict[str, Any]:- return cls.generate_output_schema()-- async def scan(self, values: List[str]) -> List[Dict[str, Any]]:- # Mock implementation- return [{"address": "1.2.3.4"}]---class IncompleteScanner(Scanner):- """Scanner without InputType/OutputType for testing error cases"""-- @classmethod- def name(cls) -> str:- return "incomplete_scanner"-- @classmethod- def category(cls) -> str:- return "Test"-- @classmethod- def key(cls) -> str:- return "test"-- @classmethod- def input_schema(cls) -> Dict[str, Any]:- return cls.generate_input_schema()-- @classmethod- def output_schema(cls) -> Dict[str, Any]:- return cls.generate_output_schema()-- async def scan(self, values: List[str]) -> List[Dict[str, Any]]:- return []---class TestBaseScannerInputOutputTypes:- """Test suite for Scanner InputType/OutputType functionality"""-- def test_input_type_output_type_class_attributes(self):- """Test that InputType and OutputType are properly set as class attributes"""- assert hasattr(MockScanner, "InputType")- assert hasattr(MockScanner, "OutputType")- assert MockScanner.InputType == List[Domain]- assert MockScanner.OutputType == List[Ip]-- def test_generate_input_schema_success(self):- """Test that generate_input_schema works correctly with valid InputType"""- schema = MockScanner.generate_input_schema()-- assert isinstance(schema, dict)- assert "type" in schema- assert "properties" in schema- assert schema["type"] == "Domain"-- # Check that properties are correctly extracted- properties = schema["properties"]- assert isinstance(properties, list)-- # Should have domain property- domain_prop = next((p for p in properties if p["name"] == "domain"), None)- assert domain_prop is not None- assert domain_prop["type"] == "string"-- def test_generate_output_schema_success(self):- """Test that generate_output_schema works correctly with valid OutputType"""- schema = MockScanner.generate_output_schema()-- assert isinstance(schema, dict)- assert "type" in schema- assert "properties" in schema- assert schema["type"] == "Ip"-- # Check that properties are correctly extracted- properties = schema["properties"]- assert isinstance(properties, list)-- # Should have address property- address_prop = next((p for p in properties if p["name"] == "address"), None)- assert address_prop is not None- assert address_prop["type"] == "string"-- def test_generate_input_schema_not_implemented_error(self):- """Test that generate_input_schema raises error when InputType is not defined"""- with pytest.raises(NotImplementedError) as exc_info:- IncompleteScanner.generate_input_schema()-- assert "InputType must be defined" in str(exc_info.value)- assert "IncompleteScanner" in str(exc_info.value)-- def test_generate_output_schema_not_implemented_error(self):- """Test that generate_output_schema raises error when OutputType is not defined"""- with pytest.raises(NotImplementedError) as exc_info:- IncompleteScanner.generate_output_schema()-- assert "OutputType must be defined" in str(exc_info.value)- assert "IncompleteScanner" in str(exc_info.value)-- def test_input_output_schema_methods_use_generate_methods(self):- """Test that the schema methods properly use the generate methods"""- input_schema = MockScanner.input_schema()- output_schema = MockScanner.output_schema()-- # These should be identical to calling generate methods directly- assert input_schema == MockScanner.generate_input_schema()- assert output_schema == MockScanner.generate_output_schema()-- def test_base_scanner_has_not_implemented_defaults(self):- """Test that base Scanner class has NotImplemented defaults"""- assert Scanner.InputType is NotImplemented- assert Scanner.OutputType is NotImplemented-- def test_inheritance_preserves_input_output_types(self):- """Test that InputType and OutputType are properly inherited"""-- class ChildScanner(MockScanner):- pass-- # Child should inherit the types from MockScanner- assert ChildScanner.InputType == List[Domain]- assert ChildScanner.OutputType == List[Ip]-- # And schema generation should work- input_schema = ChildScanner.generate_input_schema()- output_schema = ChildScanner.generate_output_schema()-- assert input_schema["type"] == "Domain"- assert output_schema["type"] == "Ip"-- def test_scanner_instance_can_access_class_types(self):- """Test that scanner instances can access InputType and OutputType"""- scanner = MockScanner("test_sketch", "test_scan")-- assert scanner.InputType == List[Domain]- assert scanner.OutputType == List[Ip]-- # Instance should be able to call generate methods- input_schema = scanner.generate_input_schema()- output_schema = scanner.generate_output_schema()-- assert input_schema["type"] == "Domain"- assert output_schema["type"] == "Ip"---class TestScannerFunctionality:- """Test other Scanner base functionality"""-- def test_scanner_initialization(self):- """Test that Scanner initializes correctly"""- scanner = MockScanner("test_sketch", "test_scan")-- assert scanner.sketch_id == "test_sketch"- assert scanner.scan_id == "test_scan"- assert scanner.params == {}- assert scanner.params_schema == []-- def test_scanner_initialization_with_defaults(self):- """Test Scanner initialization with default values"""- scanner = MockScanner()-- assert scanner.sketch_id == "system"- assert scanner.scan_id == "default"-- @pytest.mark.asyncio- async def test_scanner_execute_method(self):- """Test the execute method workflow"""- scanner = MockScanner("test_sketch", "test_scan")-- result = await scanner.execute(["test.com"])-- assert isinstance(result, list)- assert len(result) == 1- assert result[0]["address"] == "1.2.3.4"diff --git a/flowsint-core/src/flowsint_core/tests/test_crypto_with_vault.py b/flowsint-core/src/flowsint_core/tests/test_crypto_with_vault.pydeleted file mode 100644index 03b5551d..00000000--- a/flowsint-core/src/flowsint_core/tests/test_crypto_with_vault.py+++ /dev/null@@ -1,325 +0,0 @@-import pytest-import uuid-from flowsint_transforms.crypto.wallet_to_transactions import (- CryptoWalletAddressToTransactions,-)-from flowsint_transforms.crypto.wallet_to_nfts import CryptoWalletAddressToNFTs-from ..core.vault import Vault-from .fixtures.database import (- test_db_session,- test_owner_id,- test_profile,- test_api_key,- test_vault,- test_vault_with_multiple_keys,- multiple_test_keys,- test_sketch_scan_ids,-)---class TestVaultIntegration:- """Test suite for Vault integration"""-- def test_vault_creation(self, test_vault, test_owner_id):- """Test vault creation with valid owner ID"""- assert test_vault.owner_id == test_owner_id- assert test_vault.db is not None-- def test_vault_creation_without_owner_id(self, test_db_session):- """Test vault creation fails without owner ID"""- with pytest.raises(ValueError, match="owner_id is required to use the vault"):- Vault(db=test_db_session, owner_id=None)-- def test_vault_get_secret_by_name(self, test_vault_with_multiple_keys):- """Test retrieving secret by key name"""- secret = test_vault_with_multiple_keys.get_secret("vaultKey1")- assert secret == "test_vault_key_789"-- def test_vault_get_secret_by_uuid(- self, test_vault_with_multiple_keys, multiple_test_keys- ):- """Test retrieving secret by key UUID"""- # Get the UUID of the first key- key_uuid = str(multiple_test_keys[0].id)- secret = test_vault_with_multiple_keys.get_secret(key_uuid)- assert secret == "test_etherscan_key_123"-- def test_vault_get_secret_nonexistent(self, test_vault):- """Test retrieving non-existent secret returns None"""- secret = test_vault.get_secret("nonexistent_key")- assert secret is None-- def test_vault_get_secret_wrong_owner(self, test_db_session, test_api_key):- """Test retrieving secret with wrong owner ID returns None"""- wrong_owner_id = uuid.uuid4()- vault = Vault(db=test_db_session, owner_id=wrong_owner_id)- secret = vault.get_secret("test_etherscan_key")- assert secret is None---class TestCryptoWalletAddressToTransactions:- """Test suite for CryptoWalletAddressToTransactions scanner with vault integration"""-- def test_scanner_creation_with_vault(self, test_vault, test_sketch_scan_ids):- """Test scanner creation with vault"""- scanner = CryptoWalletAddressToTransactions(- sketch_id=test_sketch_scan_ids["sketch_id"],- scan_id=test_sketch_scan_ids["scan_id"],- vault=test_vault,- params={- "ETHERSCAN_API_KEY": "test_etherscan_key",- "ETHERSCAN_API_URL": "https://api.etherscan.io/api",- },- )-- assert scanner.sketch_id == test_sketch_scan_ids["sketch_id"]- assert scanner.scan_id == test_sketch_scan_ids["scan_id"]- assert scanner.vault == test_vault-- def test_scanner_creation_without_vault(self, test_sketch_scan_ids):- """Test scanner creation without vault"""- scanner = CryptoWalletAddressToTransactions(- sketch_id=test_sketch_scan_ids["sketch_id"],- scan_id=test_sketch_scan_ids["scan_id"],- params={- "ETHERSCAN_API_KEY": "direct_api_key",- "ETHERSCAN_API_URL": "https://api.etherscan.io/api",- },- )-- assert scanner.vault is None-- @pytest.mark.asyncio- async def test_async_init_with_vault_key_exists(- self, test_vault_with_multiple_keys, test_sketch_scan_ids- ):- """Test async_init when vault key exists"""- scanner = CryptoWalletAddressToTransactions(- sketch_id=test_sketch_scan_ids["sketch_id"],- scan_id=test_sketch_scan_ids["scan_id"],- vault=test_vault_with_multiple_keys,- params={- "ETHERSCAN_API_KEY": "vaultKey1", # This key exists in vault- "ETHERSCAN_API_URL": "https://api.etherscan.io/api",- },- )-- # Get params before async_init- params_before = scanner.get_params()- assert params_before["ETHERSCAN_API_KEY"] == "vaultKey1"-- # Initialize scanner- await scanner.async_init()-- # Get params after async_init - should be resolved from vault- params_after = scanner.get_params()- assert params_after["ETHERSCAN_API_KEY"] == "test_vault_key_789"-- @pytest.mark.asyncio- async def test_async_init_with_vault_key_not_exists(- self, test_vault, test_sketch_scan_ids- ):- """Test async_init when vault key doesn't exist"""- scanner = CryptoWalletAddressToTransactions(- sketch_id=test_sketch_scan_ids["sketch_id"],- scan_id=test_sketch_scan_ids["scan_id"],- vault=test_vault,- params={- "ETHERSCAN_API_KEY": "nonexistent_key", # This key doesn't exist- "ETHERSCAN_API_URL": "https://api.etherscan.io/api",- },- )-- # Get params before async_init- params_before = scanner.get_params()- assert params_before["ETHERSCAN_API_KEY"] == "nonexistent_key"-- # Initialize scanner- await scanner.async_init()-- # Get params after async_init - should remain unchanged since key not found- params_after = scanner.get_params()- assert params_after["ETHERSCAN_API_KEY"] == "nonexistent_key"-- @pytest.mark.asyncio- async def test_async_init_without_vault(self, test_sketch_scan_ids):- """Test async_init without vault"""- scanner = CryptoWalletAddressToTransactions(- sketch_id=test_sketch_scan_ids["sketch_id"],- scan_id=test_sketch_scan_ids["scan_id"],- params={- "ETHERSCAN_API_KEY": "direct_api_key",- "ETHERSCAN_API_URL": "https://api.etherscan.io/api",- },- )-- # Get params before async_init- params_before = scanner.get_params()- assert params_before["ETHERSCAN_API_KEY"] == "direct_api_key"-- # Initialize scanner- await scanner.async_init()-- # Get params after async_init - should remain unchanged- params_after = scanner.get_params()- assert params_after["ETHERSCAN_API_KEY"] == "direct_api_key"-- def test_scanner_properties(self, test_vault, test_sketch_scan_ids):- """Test scanner static properties"""- assert CryptoWalletAddressToTransactions.name() == "wallet_to_transactions"- assert CryptoWalletAddressToTransactions.category() == "CryptoWallet"- assert isinstance(CryptoWalletAddressToTransactions.required_params(), bool)---class TestCryptoWalletAddressToNFTs:- """Test suite for CryptoWalletAddressToNFTs scanner with vault integration"""-- def test_scanner_creation_with_vault(self, test_vault, test_sketch_scan_ids):- """Test scanner creation with vault"""- scanner = CryptoWalletAddressToNFTs(- sketch_id=test_sketch_scan_ids["sketch_id"],- scan_id=test_sketch_scan_ids["scan_id"],- vault=test_vault,- params={- "ETHERSCAN_API_KEY": "test_etherscan_key",- "ETHERSCAN_API_URL": "https://api.etherscan.io/api",- },- )-- assert scanner.sketch_id == test_sketch_scan_ids["sketch_id"]- assert scanner.scan_id == test_sketch_scan_ids["scan_id"]- assert scanner.vault == test_vault-- @pytest.mark.asyncio- async def test_async_init_with_vault_multiple_keys(- self, test_vault_with_multiple_keys, test_sketch_scan_ids- ):- """Test async_init with multiple vault keys"""- scanner = CryptoWalletAddressToNFTs(- sketch_id=test_sketch_scan_ids["sketch_id"],- scan_id=test_sketch_scan_ids["scan_id"],- vault=test_vault_with_multiple_keys,- params={- "ETHERSCAN_API_KEY": "etherscan_api_key", # This key exists in vault- "ETHERSCAN_API_URL": "https://api.etherscan.io/api",- },- )-- # Initialize scanner- await scanner.async_init()-- # Get params after async_init - should be resolved from vault- params_after = scanner.get_params()- assert params_after["ETHERSCAN_API_KEY"] == "test_etherscan_key_123"-- def test_scanner_properties(self):- """Test scanner static properties"""- assert CryptoWalletAddressToNFTs.name() == "wallet_to_nfts"- assert CryptoWalletAddressToNFTs.category() == "CryptoWallet"- assert isinstance(CryptoWalletAddressToNFTs.required_params(), bool)---class TestBothCryptoScannersIntegration:- """Test suite for testing both crypto scanners together"""-- @pytest.mark.asyncio- async def test_multiple_scanners_same_vault(- self, test_vault_with_multiple_keys, test_sketch_scan_ids- ):- """Test multiple scanners using the same vault"""- scanner1 = CryptoWalletAddressToTransactions(- sketch_id=test_sketch_scan_ids["sketch_id"],- scan_id=test_sketch_scan_ids["scan_id"],- vault=test_vault_with_multiple_keys,- params={- "ETHERSCAN_API_KEY": "vaultKey1",- "ETHERSCAN_API_URL": "https://api.etherscan.io/api",- },- )-- scanner2 = CryptoWalletAddressToNFTs(- sketch_id=test_sketch_scan_ids["sketch_id"],- scan_id=test_sketch_scan_ids["scan_id"],- vault=test_vault_with_multiple_keys,- params={- "ETHERSCAN_API_KEY": "vaultKey1",- "ETHERSCAN_API_URL": "https://api.etherscan.io/api",- },- )-- # Initialize both scanners- await scanner1.async_init()- await scanner2.async_init()-- # Both should have resolved the same key- params1 = scanner1.get_params()- params2 = scanner2.get_params()-- assert params1["ETHERSCAN_API_KEY"] == "test_vault_key_789"- assert params2["ETHERSCAN_API_KEY"] == "test_vault_key_789"-- @pytest.mark.asyncio- async def test_multiple_scanners_different_keys(- self, test_vault_with_multiple_keys, test_sketch_scan_ids- ):- """Test multiple scanners using different vault keys"""- scanner1 = CryptoWalletAddressToTransactions(- sketch_id=test_sketch_scan_ids["sketch_id"],- scan_id=test_sketch_scan_ids["scan_id"],- vault=test_vault_with_multiple_keys,- params={- "ETHERSCAN_API_KEY": "vaultKey1",- "ETHERSCAN_API_URL": "https://api.etherscan.io/api",- },- )-- scanner2 = CryptoWalletAddressToNFTs(- sketch_id=test_sketch_scan_ids["sketch_id"],- scan_id=test_sketch_scan_ids["scan_id"],- vault=test_vault_with_multiple_keys,- params={- "ETHERSCAN_API_KEY": "etherscan_api_key",- "ETHERSCAN_API_URL": "https://api.etherscan.io/api",- },- )-- # Initialize both scanners- await scanner1.async_init()- await scanner2.async_init()-- # Should have resolved different keys- params1 = scanner1.get_params()- params2 = scanner2.get_params()-- assert params1["ETHERSCAN_API_KEY"] == "test_vault_key_789"- assert params2["ETHERSCAN_API_KEY"] == "test_etherscan_key_123"-- def test_scanner_schemas_consistency(self):- """Test that both crypto scanners have consistent schemas"""- transactions_schema = CryptoWalletAddressToTransactions.input_schema()- nfts_schema = CryptoWalletAddressToNFTs.input_schema()-- # Both should accept the same input type (crypto wallet)- assert transactions_schema["type"] == nfts_schema["type"]-- # Both should have param schemas with ETHERSCAN_API_KEY- transactions_params = CryptoWalletAddressToTransactions.get_params_schema()- nfts_params = CryptoWalletAddressToNFTs.get_params_schema()-- # Check that both have ETHERSCAN_API_KEY parameter- transactions_api_key = next(- (- param- for param in transactions_params- if param["name"] == "ETHERSCAN_API_KEY"- ),- None,- )- nfts_api_key = next(- (param for param in nfts_params if param["name"] == "ETHERSCAN_API_KEY"),- None,- )-- assert transactions_api_key is not None- assert nfts_api_key is not None- assert transactions_api_key["required"] == nfts_api_key["required"]diff --git a/flowsint-core/src/flowsint_core/tests/test_input_output_migration.py b/flowsint-core/src/flowsint_core/tests/test_input_output_migration.pydeleted file mode 100644index 62d223fd..00000000--- a/flowsint-core/src/flowsint_core/tests/test_input_output_migration.py+++ /dev/null@@ -1,369 +0,0 @@-"""-Test migration from old TypeAlias pattern to new InputType/OutputType class attributes.--This test demonstrates the benefits and proper usage of the new pattern.-"""--import pytest-from typing import List, Dict, Any, TypeAlias-from pydantic import TypeAdapter-from ..core.scanner_base import Scanner-from flowsint_types.domain import Domain-from flowsint_types.ip import Ip-from flowsint_types.email import Email-from app.utils import resolve_type---class OldPatternScanner(Scanner):- """Example of old pattern using module-level TypeAlias"""-- @classmethod- def name(cls) -> str:- return "old_pattern_scanner"-- @classmethod- def category(cls) -> str:- return "Test"-- @classmethod- def key(cls) -> str:- return "domain"-- @classmethod- def input_schema(cls) -> Dict[str, Any]:- # Old pattern: manually defining TypeAlias and building schema- InputType: TypeAlias = List[Domain]- adapter = TypeAdapter(InputType)- schema = adapter.json_schema()- type_name, details = list(schema["$defs"].items())[0]- return {- "type": type_name,- "properties": [- {"name": prop, "type": resolve_type(info, schema)}- for prop, info in details["properties"].items()- ],- }-- @classmethod- def output_schema(cls) -> Dict[str, Any]:- # Old pattern: manually defining TypeAlias and building schema- OutputType: TypeAlias = List[Ip]- adapter = TypeAdapter(OutputType)- schema = adapter.json_schema()- type_name, details = list(schema["$defs"].items())[0]- return {- "type": type_name,- "properties": [- {"name": prop, "type": resolve_type(info, schema)}- for prop, info in details["properties"].items()- ],- }-- async def scan(self, values: List[str]) -> List[Dict[str, Any]]:- return [{"address": "1.2.3.4"}]---class NewPatternScanner(Scanner):- """Example of new pattern using class attributes with automatic schema generation"""-- # New pattern: just define class attributes - base class handles the rest!- InputType = List[Domain]- OutputType = List[Ip]-- @classmethod- def name(cls) -> str:- return "new_pattern_scanner"-- @classmethod- def category(cls) -> str:- return "Test"-- @classmethod- def key(cls) -> str:- return "domain"-- # No need to implement input_schema() or output_schema() - base class does it automatically!-- # Methods can use InputType/OutputType directly (once made available at module level)- def preprocess(self, data: List[Domain]) -> List[Domain]:- # Using concrete type for clarity in test, but would use InputType in real implementation- cleaned: List[Domain] = []- for item in data:- if isinstance(item, Domain):- cleaned.append(item)- return cleaned-- async def scan(self, values: List[Domain]) -> List[Ip]:- # Using concrete type for clarity in test, but would use InputType/OutputType in real implementation- results: List[Ip] = []- for domain in values:- results.append(Ip(address="1.2.3.4"))- return results---# Make types available at module level (this is what enables clean usage)-NewPatternInputType = NewPatternScanner.InputType-NewPatternOutputType = NewPatternScanner.OutputType---class AdvancedNewPatternScanner(Scanner):- """Example showing advanced usage with different types"""-- InputType = List[Email]- OutputType = List[Domain]-- @classmethod- def name(cls) -> str:- return "advanced_pattern_scanner"-- @classmethod- def category(cls) -> str:- return "Test"-- @classmethod- def key(cls) -> str:- return "email"-- # Schema methods automatic!-- async def scan(self, values: List[str]) -> List[Dict[str, Any]]:- return [{"domain": "example.com"}]---class TestInputOutputMigrationPattern:- """Test migration from old to new pattern"""-- def test_both_patterns_produce_same_schema(self):- """Test that old and new patterns produce identical schemas"""- old_input_schema = OldPatternScanner.input_schema()- new_input_schema = NewPatternScanner.input_schema()-- old_output_schema = OldPatternScanner.output_schema()- new_output_schema = NewPatternScanner.output_schema()-- # Schemas should be identical- assert old_input_schema == new_input_schema- assert old_output_schema == new_output_schema-- def test_new_pattern_benefits_code_reuse(self):- """Test that new pattern reduces code duplication"""- # With the new pattern, multiple scanners can easily reuse the same logic-- class Scanner1(Scanner):- InputType = List[Domain]- OutputType = List[Ip]-- @classmethod- def name(cls):- return "scanner1"-- @classmethod- def category(cls):- return "Test"-- @classmethod- def key(cls):- return "domain"-- # No need for input_schema() or output_schema() - automatic!- async def scan(self, values):- return []-- class Scanner2(Scanner):- InputType = List[Domain]- OutputType = List[Ip]-- @classmethod- def name(cls):- return "scanner2"-- @classmethod- def category(cls):- return "Test"-- @classmethod- def key(cls):- return "domain"-- # No need for input_schema() or output_schema() - automatic!- async def scan(self, values):- return []-- # Both should produce identical schemas with minimal code- assert Scanner1.input_schema() == Scanner2.input_schema()- assert Scanner1.output_schema() == Scanner2.output_schema()-- def test_new_pattern_type_introspection(self):- """Test that new pattern allows for better type introspection"""- # Can easily check what types a scanner uses- assert NewPatternScanner.InputType == List[Domain]- assert NewPatternScanner.OutputType == List[Ip]-- assert AdvancedNewPatternScanner.InputType == List[Email]- assert AdvancedNewPatternScanner.OutputType == List[Domain]-- # This wasn't easily possible with the old pattern-- def test_new_pattern_inheritance_works(self):- """Test that new pattern works well with inheritance"""-- class BaseDomainScanner(Scanner):- InputType = List[Domain]- OutputType = List[Ip]-- @classmethod- def name(cls):- return "base_domain"-- @classmethod- def category(cls):- return "Test"-- @classmethod- def key(cls):- return "domain"-- # Schema methods automatic!- async def scan(self, values):- return []-- class SpecializedDomainScanner(BaseDomainScanner):- @classmethod- def name(cls):- return "specialized_domain"-- # Inherits InputType and OutputType-- # Child should inherit the types and schemas- assert SpecializedDomainScanner.InputType == List[Domain]- assert SpecializedDomainScanner.OutputType == List[Ip]-- specialized_input = SpecializedDomainScanner.input_schema()- specialized_output = SpecializedDomainScanner.output_schema()-- assert specialized_input["type"] == "Domain"- assert specialized_output["type"] == "Ip"-- def test_new_pattern_error_handling(self):- """Test that new pattern provides better error handling"""-- class IncompleteDomainScanner(Scanner):- # Forgot to define InputType and OutputType- @classmethod- def name(cls):- return "incomplete"-- @classmethod- def category(cls):- return "Test"-- @classmethod- def key(cls):- return "domain"-- # Base class will try to generate schemas automatically and fail appropriately- async def scan(self, values):- return []-- # Should get clear error messages- with pytest.raises(NotImplementedError) as exc_info:- IncompleteDomainScanner.input_schema()- assert "InputType must be defined" in str(exc_info.value)- assert "IncompleteDomainScanner" in str(exc_info.value)-- with pytest.raises(NotImplementedError) as exc_info:- IncompleteDomainScanner.output_schema()- assert "OutputType must be defined" in str(exc_info.value)- assert "IncompleteDomainScanner" in str(exc_info.value)-- def test_new_pattern_runtime_accessibility(self):- """Test that types are accessible at runtime for dynamic operations"""-- # Can build registries or perform operations based on types- scanners = [NewPatternScanner, AdvancedNewPatternScanner]-- domain_input_scanners = [- scanner- for scanner in scanners- if hasattr(scanner, "InputType") and scanner.InputType == List[Domain]- ]-- email_input_scanners = [- scanner- for scanner in scanners- if hasattr(scanner, "InputType") and scanner.InputType == List[Email]- ]-- assert len(domain_input_scanners) == 1- assert domain_input_scanners[0] == NewPatternScanner-- assert len(email_input_scanners) == 1- assert email_input_scanners[0] == AdvancedNewPatternScanner-- def test_new_pattern_with_clean_type_usage(self):- """Test that the new pattern allows clean type usage without quotes"""- scanner = NewPatternScanner("test", "test")-- # Test that we can create data of the expected types- test_domains = [Domain(domain="example.com"), Domain(domain="test.com")]-- # Preprocess should work with clean type annotations- result = scanner.preprocess(test_domains)- assert len(result) == 2- assert all(isinstance(d, Domain) for d in result)-- @pytest.mark.asyncio- async def test_new_pattern_async_methods(self):- """Test that async methods work correctly with clean type annotations"""- scanner = NewPatternScanner("test", "test")-- test_domains = [Domain(domain="example.com")]- result = await scanner.scan(test_domains)-- assert len(result) == 1- assert isinstance(result[0], Ip)- assert result[0].address == "1.2.3.4"-- def test_module_level_type_access(self):- """Test that types are properly accessible at module level"""- # These should be available after the class definition- assert NewPatternInputType == List[Domain]- assert NewPatternOutputType == List[Ip]-- # And they should match the class attributes- assert NewPatternInputType == NewPatternScanner.InputType- assert NewPatternOutputType == NewPatternScanner.OutputType-- def test_migration_checklist(self):- """Test that demonstrates a complete migration checklist"""-- # Migration steps:- # 1. Define InputType and OutputType as class attributes- # 2. Remove input_schema() and output_schema() method implementations (base class handles automatically)- # 3. Add module-level assignments: InputType = MyScanner.InputType (optional, for clean usage)-- # Verify the new pattern is simpler and more maintainable- new_scanner_benefits = [- "Just define InputType and OutputType class attributes",- "Automatic schema generation by base class",- "No boilerplate schema methods needed",- "Consistent schema generation across all scanners",- "Clean type usage throughout class methods",- ]-- assert len(new_scanner_benefits) == 5-- # Verify functionality is preserved and automatic- schema = NewPatternScanner.input_schema()- assert schema["type"] == "Domain"- assert any(prop["name"] == "domain" for prop in schema["properties"])-- # Verify schemas are generated automatically without manual implementation- assert hasattr(NewPatternScanner, "input_schema")- assert hasattr(NewPatternScanner, "output_schema")-- # The base class should be handling the schema generation- input_schema = NewPatternScanner.input_schema()- output_schema = NewPatternScanner.output_schema()- assert input_schema is not None- assert output_schema is not Nonediff --git a/flowsint-core/src/flowsint_core/tests/test_registry.py b/flowsint-core/src/flowsint_core/tests/test_registry.pydeleted file mode 100644index 66f0d529..00000000--- a/flowsint-core/src/flowsint_core/tests/test_registry.py+++ /dev/null@@ -1,120 +0,0 @@-import pytest-from flowsint_core.core.registry import TransformRegistry-from flowsint_core.core.scanner_base import Scanner-from flowsint_types.domain import Domain-from flowsint_types.ip import Ip-from typing import List---class MockScanner(Scanner):- """Mock scanner for testing registry functionality"""-- InputType = List[Domain]- OutputType = List[Ip]-- @classmethod- def name(cls) -> str:- return "mock_scanner"-- @classmethod- def category(cls) -> str:- return "Test"-- @classmethod- def key(cls) -> str:- return "domain"-- @classmethod- def documentation(cls) -> str:- return "Mock scanner for testing"-- @classmethod- def get_params_schema(cls) -> List[dict]:- return []-- @classmethod- def required_params(cls) -> bool:- return False-- @classmethod- def icon(cls) -> str:- return "test-icon"-- async def scan(self, data: InputType) -> OutputType:- return []---class TestTransformRegistry:- """Test suite for TransformRegistry functionality"""-- def setup_method(self):- """Clear registry before each test"""- TransformRegistry.clear()-- def test_register_scanner(self):- """Test registering a scanner"""- TransformRegistry.register(MockScanner)- assert TransformRegistry.transform_exists("mock_scanner")-- scanners = TransformRegistry.list()- assert "mock_scanner" in scanners- assert scanners["mock_scanner"]["class_name"] == "MockScanner"-- def test_register_invalid_scanner(self):- """Test that registering a non-scanner class raises error"""-- class NotAScanner:- pass-- with pytest.raises(ValueError, match="must inherit from Scanner"):- TransformRegistry.register(NotAScanner)-- def test_get_scanner_instance(self):- """Test getting a scanner instance"""- TransformRegistry.register(MockScanner)-- scanner = TransformRegistry.get_scanner(- "mock_scanner", sketch_id="test_sketch", scan_id="test_scan"- )-- assert isinstance(scanner, MockScanner)- assert scanner.sketch_id == "test_sketch"- assert scanner.scan_id == "test_scan"-- def test_get_nonexistent_scanner(self):- """Test that getting a non-existent scanner raises exception"""- with pytest.raises(Exception, match="Scanner 'nonexistent' not found"):- TransformRegistry.get_scanner(- "nonexistent", sketch_id="test_sketch", scan_id="test_scan"- )-- def test_list_by_categories(self):- """Test listing scanners by category"""- TransformRegistry.register(MockScanner)-- by_category = TransformRegistry.list_by_categories()- assert "Test" in by_category- assert len(by_category["Test"]) == 1- assert by_category["Test"][0]["name"] == "mock_scanner"-- def test_list_by_input_type(self):- """Test listing scanners by input type"""- TransformRegistry.register(MockScanner)-- domain_scanners = TransformRegistry.list_by_input_type("Domain")- assert len(domain_scanners) == 1- assert domain_scanners[0]["name"] == "mock_scanner"-- # Test with "any" input type- any_scanners = TransformRegistry.list_by_input_type("any")- assert len(any_scanners) == 1- assert any_scanners[0]["name"] == "mock_scanner"-- def test_register_module(self):- """Test registering a module path"""- TransformRegistry.register_module("test.module.path")- assert "test.module.path" in TransformRegistry._scanner_modules-- # Registering the same module again should not duplicate- TransformRegistry.register_module("test.module.path")- assert TransformRegistry._scanner_modules.count("test.module.path") == 1diff --git a/flowsint-core/src/flowsint_core/tests/test_schema_generation.py b/flowsint-core/src/flowsint_core/tests/test_schema_generation.pydeleted file mode 100644index bb09fd8b..00000000--- a/flowsint-core/src/flowsint_core/tests/test_schema_generation.py+++ /dev/null@@ -1,165 +0,0 @@-"""-Test schema generation for scanners with various InputType and OutputType combinations.-"""--from typing import List-from flowsint_types.website import Website-from flowsint_types.domain import Domain-from flowsint_transforms.websites.to_domain import WebsiteToDomainScanner-from flowsint_transforms.websites.to_webtrackers import WebsiteToWebtrackersScanner-from flowsint_transforms.websites.to_crawler import WebsiteToCrawler-from flowsint_transforms.domains.to_website import DomainToWebsiteScanner---class TestSchemaGeneration:- """Test that schema generation correctly identifies InputType and OutputType."""-- def test_website_to_domain_scanner_schemas(self):- """Test that WebsiteToDomainScanner correctly shows Website as input type."""- scanner = WebsiteToDomainScanner-- # Test InputType attribute- assert scanner.InputType == List[Website]-- # Test input schema generation- input_schema = scanner.input_schema()- assert (- input_schema["type"] == "Website"- ), f"Expected 'Website', got '{input_schema['type']}'"-- # Test output schema generation- output_schema = scanner.output_schema()- assert (- output_schema["type"] == "Domain"- ), f"Expected 'Domain', got '{output_schema['type']}'"-- def test_website_to_webtrackers_scanner_schemas(self):- """Test that WebsiteToWebtrackersScanner correctly shows Website as input type."""- scanner = WebsiteToWebtrackersScanner-- # Test InputType attribute- assert scanner.InputType == List[Website]-- # Test input schema generation- input_schema = scanner.input_schema()- assert (- input_schema["type"] == "Website"- ), f"Expected 'Website', got '{input_schema['type']}'"-- # Test output schema generation- output_schema = scanner.output_schema()- assert (- output_schema["type"] == "WebTracker"- ), f"Expected 'WebTracker', got '{output_schema['type']}'"-- def test_website_to_crawler_scanner_schemas(self):- """Test that WebsiteToCrawler correctly shows Website as input type."""- scanner = WebsiteToCrawler-- # Test InputType attribute- assert scanner.InputType == List[Website]-- # Test input schema generation- input_schema = scanner.input_schema()- assert (- input_schema["type"] == "Website"- ), f"Expected 'Website', got '{input_schema['type']}'"-- def test_domain_to_website_scanner_schemas(self):- """Test that DomainToWebsiteScanner correctly shows Domain as input and Website as output."""- scanner = DomainToWebsiteScanner-- # Test InputType attribute- assert scanner.InputType == List[Domain]-- # Test input schema generation- input_schema = scanner.input_schema()- assert (- input_schema["type"] == "Domain"- ), f"Expected 'Domain', got '{input_schema['type']}'"-- # Test output schema generation- output_schema = scanner.output_schema()- assert (- output_schema["type"] == "Website"- ), f"Expected 'Website', got '{output_schema['type']}'"-- def test_all_website_scanners_have_correct_input_types(self):- """Test that all scanners taking Website input show Website in schema."""- website_input_scanners = [- (WebsiteToDomainScanner, "Website", "Domain"),- (WebsiteToWebtrackersScanner, "Website", "WebTracker"),- (WebsiteToCrawler, "Website", None), # Unknown output type- ]-- for scanner_class, expected_input, expected_output in website_input_scanners:- input_schema = scanner_class.input_schema()- assert (- input_schema["type"] == expected_input- ), f"{scanner_class.__name__}: Expected input '{expected_input}', got '{input_schema['type']}'"-- if expected_output:- output_schema = scanner_class.output_schema()- assert (- output_schema["type"] == expected_output- ), f"{scanner_class.__name__}: Expected output '{expected_output}', got '{output_schema['type']}'"-- def test_schema_generation_debug_info(self):- """Debug test to see what's actually in the schemas."""- scanner = WebsiteToDomainScanner-- # Get the raw TypeAdapter schema- from pydantic import TypeAdapter-- adapter = TypeAdapter(scanner.InputType)- raw_schema = adapter.json_schema()-- print(f"\n=== Debug Info for {scanner.__name__} ===")- print(f"InputType: {scanner.InputType}")- print(f"Raw schema keys: {list(raw_schema.keys())}")- if "$defs" in raw_schema:- print(f"$defs keys: {list(raw_schema['$defs'].keys())}")- print(f"Schema items: {raw_schema.get('items', 'No items')}")- print(f"Generated input schema: {scanner.input_schema()}")-- # This test always passes, it's just for debugging- assert True-- def test_schema_generation_follows_ref_not_first_def(self):- """- Regression test for the schema generation bug.-- Before the fix: generate_input_schema() picked the first definition in $defs- (alphabetically "Domain" came before "Website"), so Website scanners incorrectly- showed "Domain" as their input type.-- After the fix: generate_input_schema() follows the $ref in items to get the- correct type name.- """- scanner = WebsiteToDomainScanner-- # Get the raw schema to understand the structure- from pydantic import TypeAdapter-- adapter = TypeAdapter(scanner.InputType)- raw_schema = adapter.json_schema()-- # Verify the raw schema structure that caused the bug- assert "$defs" in raw_schema- assert "Domain" in raw_schema["$defs"]- assert "Website" in raw_schema["$defs"]- assert raw_schema["items"]["$ref"] == "#/$defs/Website"-- # Before fix: list(raw_schema["$defs"].items())[0][0] would be "Domain" (first alphabetically)- first_def_name = list(raw_schema["$defs"].items())[0][0]- assert first_def_name == "Domain" # This would have been the bug-- # After fix: We follow the $ref to get "Website"- ref_type = raw_schema["items"]["$ref"].split("/")[-1]- assert ref_type == "Website" # This is what we should use-- # Verify our fix works correctly- input_schema = scanner.input_schema()- assert (- input_schema["type"] == "Website"- ), f"Schema generation should follow $ref, not pick first def. Got '{input_schema['type']}'"diff --git a/flowsint-core/src/flowsint_core/tests/vault.py b/flowsint-core/src/flowsint_core/tests/vault.pynew file mode 100644index 00000000..a105d90a--- /dev/null+++ b/flowsint-core/src/flowsint_core/tests/vault.py@@ -0,0 +1,5 @@+from flowsint_core.core.vault import Vault+++def test_description():+ vault = Vault()\ No newline at end of filediff --git a/flowsint-core/src/flowsint_core/utils.py b/flowsint-core/src/flowsint_core/utils.pyindex 5a6441ac..1a0bd16d 100644--- a/flowsint-core/src/flowsint_core/utils.py+++ b/flowsint-core/src/flowsint_core/utils.py@@ -215,18 +215,18 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:input_output = input_node["data"]["outputs"]node_lookup = {node["id"]: node for node in nodes}- scanners = []+ transforms = []for edge in edges:target_id = edge["target"]source_handle = edge["sourceHandle"]target_handle = edge["targetHandle"]- scanner_node = node_lookup.get(target_id)- if scanner_node and scanner_node["data"]["type"] == "scanner":- scanners.append(+ transform_node = node_lookup.get(target_id)+ if transform_node and transform_node["data"]["type"] == "transform":+ transforms.append({- "transform_name": scanner_node["data"]["name"],- "module": scanner_node["data"]["module"],+ "transform_name": transform_node["data"]["name"],+ "module": transform_node["data"]["module"],"input": source_handle,"output": target_handle,}@@ -237,8 +237,8 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:"name": input_node["data"]["name"],"outputs": input_output,},- "scanners": scanners,- "transform_names": [scanner["transform_name"] for scanner in scanners],+ "transforms": transforms,+ "transform_names": [transform["transform_name"] for transform in transforms],}diff --git a/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.py b/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.pyindex b4fb0c21..724e174d 100644--- a/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.py+++ b/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.py@@ -1,14 +1,14 @@import jsonimport subprocessfrom typing import List, Dict, Any, Union-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.cidr import CIDRfrom flowsint_types.asn import ASNfrom flowsint_core.utils import is_valid_asn, parse_asnfrom flowsint_core.core.logger import Logger-class AsnToCidrsScanner(Scanner):+class AsnToCidrsTransform(Transform):"""Takes an ASN and returns its corresponding CIDRs."""# Define types as class attributes - base class handles schema generation automatically@@ -202,5 +202,5 @@ class AsnToCidrsScanner(Scanner):# Make types available at module level for easy access-InputType = AsnToCidrsScanner.InputType-OutputType = AsnToCidrsScanner.OutputType+InputType = AsnToCidrsTransform.InputType+OutputType = AsnToCidrsTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.py b/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.pyindex 1192479f..1666b135 100644--- a/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.py+++ b/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.py@@ -1,12 +1,12 @@import subprocessfrom typing import List, Union-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.cidr import CIDRfrom flowsint_types.ip import Ipfrom flowsint_core.core.logger import Logger-class CidrToIpsScanner(Scanner):+class CidrToIpsTransform(Transform):"""Takes a CIDR and returns its corresponding IP addresses."""# Define types as class attributes - base class handles schema generation automatically@@ -120,5 +120,5 @@ class CidrToIpsScanner(Scanner):# Make types available at module level for easy access-InputType = CidrToIpsScanner.InputType-OutputType = CidrToIpsScanner.OutputType+InputType = CidrToIpsTransform.InputType+OutputType = CidrToIpsTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.pyindex a3985ef1..de0d4c16 100644--- a/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py@@ -1,7 +1,7 @@import osfrom typing import List, Dict, Any, Optional, Unionimport requests-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.wallet import CryptoWallet, CryptoNFTfrom flowsint_core.core.logger import Loggerfrom flowsint_core.core.graph_db import Neo4jConnection@@ -12,7 +12,7 @@ load_dotenv()ETHERSCAN_API_URL = os.getenv("ETHERSCAN_API_URL")-class CryptoWalletAddressToNFTs(Scanner):+class CryptoWalletAddressToNFTs(Transform):"""Resolve NFTs for a wallet address (ETH)."""# Define types as class attributes - base class handles schema generation automatically@@ -46,7 +46,7 @@ class CryptoWalletAddressToNFTs(Scanner):@classmethoddef get_params_schema(cls) -> List[Dict[str, Any]]:- """Declare required parameters for this scanner"""+ """Declare required parameters for this transform"""return [{"name": "ETHERSCAN_API_KEY",diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.pyindex fc7b99e4..0b10dc6a 100644--- a/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py@@ -4,7 +4,7 @@ import requestsimport requests.exceptionsfrom datetime import datetimefrom dotenv import load_dotenv-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.wallet import CryptoWallet, CryptoWalletTransactionfrom flowsint_core.core.graph_db import Neo4jConnectionfrom flowsint_core.core.logger import Logger@@ -18,7 +18,7 @@ def wei_to_eth(wei_str):return int(wei_str) / 10**18-class CryptoWalletAddressToTransactions(Scanner):+class CryptoWalletAddressToTransactions(Transform):# Define types as class attributes - base class handles schema generation automaticallyInputType = List[CryptoWallet]@@ -51,7 +51,7 @@ class CryptoWalletAddressToTransactions(Scanner):@classmethoddef get_params_schema(cls) -> List[Dict[str, Any]]:- """Declare required parameters for this scanner"""+ """Declare required parameters for this transform"""return [{"name": "ETHERSCAN_API_KEY",diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_asn.py b/flowsint-transforms/src/flowsint_transforms/domain/to_asn.pyindex 635ced2a..ec8684d4 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_asn.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_asn.py@@ -2,14 +2,14 @@ import jsonimport socketimport subprocessfrom typing import List, Union-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.domain import Domainfrom flowsint_types.asn import ASNfrom flowsint_core.utils import is_valid_domainfrom flowsint_core.core.logger import Logger-class DomainToAsnScanner(Scanner):+class DomainToAsnTransform(Transform):"""Takes a domain and returns its corresponding ASN."""# Define types as class attributes - base class handles schema generation automatically@@ -88,5 +88,5 @@ class DomainToAsnScanner(Scanner):# Make types available at module level for easy access-InputType = DomainToAsnScanner.InputType-OutputType = DomainToAsnScanner.OutputType+InputType = DomainToAsnTransform.InputType+OutputType = DomainToAsnTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_history.py b/flowsint-transforms/src/flowsint_transforms/domain/to_history.pyindex 49bacfc6..545421d3 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_history.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_history.py@@ -2,7 +2,7 @@ import osimport reimport jsonfrom typing import Any, List, Union, Dict, Set-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.domain import Domainfrom flowsint_types.individual import Individualfrom flowsint_types.organization import Organization@@ -17,7 +17,7 @@ load_dotenv()WHOXY_API_KEY = os.getenv("WHOXY_API_KEY")-class DomainToHistoryScanner(Scanner):+class DomainToHistoryTransform(Transform):"""[WHOXY] Takes a domain and returns history infos about it (history, organization, owners, emails, etc.)."""# Define types as class attributes - base class handles schema generation automatically@@ -649,5 +649,5 @@ class DomainToHistoryScanner(Scanner):return results-InputType = DomainToHistoryScanner.InputType-OutputType = DomainToHistoryScanner.OutputType+InputType = DomainToHistoryTransform.InputType+OutputType = DomainToHistoryTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py b/flowsint-transforms/src/flowsint_transforms/domain/to_ip.pyindex e5b5590a..8fd21493 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py@@ -1,12 +1,12 @@import socketfrom typing import List, Union-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.domain import Domainfrom flowsint_types.ip import Ipfrom flowsint_core.utils import is_valid_domain, is_root_domain-class ResolveScanner(Scanner):+class ResolveTransform(Transform):"""Resolve domain names to IP addresses."""# Define the input and output types as class attributes@@ -27,15 +27,15 @@ class ResolveScanner(Scanner):@classmethoddef documentation(cls) -> str:- """Return formatted markdown documentation for the domain resolver scanner."""+ """Return formatted markdown documentation for the domain resolver transform."""return """-# Domain Resolver Scanner+# Domain Resolver Transform-Resolve domain names to their corresponding IP addresses using DNS queries. This scanner performs forward DNS resolution to discover the IP addresses associated with domain names and subdomains.+Resolve domain names to their corresponding IP addresses using DNS queries. This transform performs forward DNS resolution to discover the IP addresses associated with domain names and subdomains.## Overview-The Domain Resolver Scanner takes domain names as input and returns their resolved IP addresses. It automatically handles different input formats and validates domains before attempting resolution.+The Domain Resolver Transform takes domain names as input and returns their resolved IP addresses. It automatically handles different input formats and validates domains before attempting resolution.## Input/Output Types@@ -44,7 +44,7 @@ The Domain Resolver Scanner takes domain names as input and returns their resolv## Input Format Support-The scanner accepts multiple input formats and automatically converts them:+The transform accepts multiple input formats and automatically converts them:### String Format@@ -151,7 +151,7 @@ MERGE (d)-[:RESOLVES_TO {sketch_id: "sketch-uuid"}]->(ip)## Domain Type Classification-The scanner automatically classifies domains:+The transform automatically classifies domains:- **Root Domain**: `example.com` → `type: "domain"`- **Subdomain**: `api.example.com` → `type: "subdomain"`@@ -160,7 +160,7 @@ The scanner automatically classifies domains:### Resolution Failures-When DNS resolution fails, the scanner:+When DNS resolution fails, the transform:- Logs the error with domain name- Continues processing remaining domains@@ -174,7 +174,7 @@ Common resolution failures:### Invalid Input Handling-The scanner filters out:+The transform filters out:- Malformed domain names- Empty strings@@ -207,7 +207,7 @@ The scanner filters out:```[Domain Discovery] → [Domain Resolver] → [IP Geolocation]- → [Port Scanner]+ → [Port Transform]→ [ASN Lookup]```@@ -234,7 +234,7 @@ The scanner filters out:### Debug Information-The scanner provides logging for:+The transform provides logging for:- Input validation results- DNS resolution attempts- Graph database operations@@ -311,5 +311,5 @@ Check FlowSint logs for detailed resolution information.# Make types available at module level for easy access-InputType = ResolveScanner.InputType-OutputType = ResolveScanner.OutputType+InputType = ResolveTransform.InputType+OutputType = ResolveTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py b/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.pyindex 30a0a98e..c5e99eee 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py@@ -1,11 +1,11 @@from typing import List, Unionfrom flowsint_transforms.utils import is_valid_domain, get_root_domain-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.domain import Domainfrom flowsint_core.core.logger import Logger-class DomainToRootDomain(Scanner):+class DomainToRootDomain(Transform):"""Subdomain to root domain."""InputType = List[Domain]diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py b/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.pyindex 55cd497b..3514e441 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py@@ -1,7 +1,7 @@import shutilimport requestsfrom typing import List, Union-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.domain import Domainfrom flowsint_core.utils import is_valid_domainfrom flowsint_core.core.logger import Logger@@ -9,8 +9,8 @@ from tools.network.subfinder import SubfinderToolfrom flowsint_core.core.logger import Logger-class SubdomainScanner(Scanner):- """Scanner to find subdomains associated with a domain."""+class SubdomainTransform(Transform):+ """Transform to find subdomains associated with a domain."""# Define types as class attributes - base class handles schema generation automaticallyInputType = List[Domain | str]@@ -144,5 +144,5 @@ class SubdomainScanner(Scanner):return output-InputType = SubdomainScanner.InputType-OutputType = SubdomainScanner.OutputType+InputType = SubdomainTransform.InputType+OutputType = SubdomainTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_website.py b/flowsint-transforms/src/flowsint_transforms/domain/to_website.pyindex 3194d4a6..d2c3a7fe 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_website.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_website.py@@ -1,13 +1,13 @@from typing import List, Unionimport requestsfrom flowsint_core.utils import is_valid_domain-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.domain import Domainfrom flowsint_types.website import Websitefrom flowsint_core.core.logger import Logger-class DomainToWebsiteScanner(Scanner):+class DomainToWebsiteTransform(Transform):"""From domain to website."""# Define types as class attributes - base class handles schema generation automatically@@ -141,5 +141,5 @@ class DomainToWebsiteScanner(Scanner):# Make types available at module level for easy access-InputType = DomainToWebsiteScanner.InputType-OutputType = DomainToWebsiteScanner.OutputType+InputType = DomainToWebsiteTransform.InputType+OutputType = DomainToWebsiteTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py b/flowsint-transforms/src/flowsint_transforms/domain/to_whois.pyindex 8671a101..77cfe3a0 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py@@ -1,7 +1,7 @@from typing import List, Unionimport whoisfrom flowsint_core.utils import is_valid_domain-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.domain import Domain, Domainfrom flowsint_types.whois import Whoisfrom flowsint_types.email import Email@@ -9,7 +9,7 @@ from flowsint_core.core.logger import Loggerfrom datetime import datetime-class WhoisScanner(Scanner):+class WhoisTransform(Transform):"""Scan for WHOIS information of a domain."""# Define types as class attributes - base class handles schema generation automatically@@ -203,5 +203,5 @@ class WhoisScanner(Scanner):# Make types available at module level for easy access-InputType = WhoisScanner.InputType-OutputType = WhoisScanner.OutputType+InputType = WhoisTransform.InputType+OutputType = WhoisTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/email/to_domains.py b/flowsint-transforms/src/flowsint_transforms/email/to_domains.pyindex ff2d2883..a741a6dc 100644--- a/flowsint-transforms/src/flowsint_transforms/email/to_domains.py+++ b/flowsint-transforms/src/flowsint_transforms/email/to_domains.py@@ -1,7 +1,7 @@import osimport refrom typing import Any, List, Union, Dict, Set-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.domain import Domainfrom flowsint_types.individual import Individualfrom flowsint_types.email import Email@@ -15,7 +15,7 @@ load_dotenv()WHOXY_API_KEY = os.getenv("WHOXY_API_KEY")-class EmailToDomainsScanner(Scanner):+class EmailToDomainsTransform(Transform):"""[WHOXY] Takes an email and returns the domains it registered."""# Define types as class attributes - base class handles schema generation automatically@@ -405,5 +405,5 @@ class EmailToDomainsScanner(Scanner):)-InputType = EmailToDomainsScanner.InputType-OutputType = EmailToDomainsScanner.OutputType+InputType = EmailToDomainsTransform.InputType+OutputType = EmailToDomainsTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py b/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.pyindex 318f9918..b6864fd7 100644--- a/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py+++ b/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py@@ -1,13 +1,13 @@import hashlibfrom typing import List, Unionimport requests-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_core.core.logger import Loggerfrom flowsint_types.email import Emailfrom flowsint_types.gravatar import Gravatar-class EmailToGravatarScanner(Scanner):+class EmailToGravatarTransform(Transform):"""From md5 hash of email to gravatar."""InputType = List[Email]@@ -129,5 +129,5 @@ class EmailToGravatarScanner(Scanner):# Make types available at module level for easy access-InputType = EmailToGravatarScanner.InputType-OutputType = EmailToGravatarScanner.OutputType+InputType = EmailToGravatarTransform.InputType+OutputType = EmailToGravatarTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.pyindex 70149614..34b78faa 100644--- a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py+++ b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py@@ -2,7 +2,7 @@ import osfrom typing import Any, Dict, List, Optional, Unionimport requestsfrom urllib.parse import urljoin-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_core.core.logger import Loggerfrom flowsint_types.email import Emailfrom flowsint_types.breach import Breach@@ -15,7 +15,7 @@ load_dotenv()HIBP_API_KEY = os.getenv("HIBP_API_KEY")-class EmailToBreachesScanner(Scanner):+class EmailToBreachesTransform(Transform):"""[HIBPWNED] Get the breaches the email might be invovled in."""InputType = List[Email]@@ -56,7 +56,7 @@ class EmailToBreachesScanner(Scanner):@classmethoddef get_params_schema(cls) -> List[Dict[str, Any]]:- """Declare required parameters for this scanner"""+ """Declare required parameters for this transform"""return [{"name": "HIBP_API_KEY",@@ -102,7 +102,7 @@ class EmailToBreachesScanner(Scanner):Logger.error(self.sketch_id, {"message": "Could not find HIBP_API_URL in params."})- headers = {"hibp-api-key": api_key, "User-Agent": "FlowsInt-Scanner"}+ headers = {"hibp-api-key": api_key, "User-Agent": "FlowsInt-Transform"}Logger.info(self.sketch_id, {"message": f"HIBP API URL: {api_url}"})for email in data:try:@@ -235,5 +235,5 @@ class EmailToBreachesScanner(Scanner):# Make types available at module level for easy access-InputType = EmailToBreachesScanner.InputType-OutputType = EmailToBreachesScanner.OutputType+InputType = EmailToBreachesTransform.InputType+OutputType = EmailToBreachesTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py b/flowsint-transforms/src/flowsint_transforms/individual/to_domains.pyindex 5f2a2d2d..6d61b3bf 100644--- a/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py+++ b/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py@@ -1,7 +1,7 @@import osimport refrom typing import Any, List, Union, Dict, Set-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.domain import Domainfrom flowsint_types.organization import Organizationfrom flowsint_types.individual import Individual@@ -15,7 +15,7 @@ load_dotenv()WHOXY_API_KEY = os.getenv("WHOXY_API_KEY")-class IndividualToDomainsScanner(Scanner):+class IndividualToDomainsTransform(Transform):"""[WHOXY] Takes an individual and returns the domains it registered."""# Define types as class attributes - base class handles schema generation automatically@@ -418,5 +418,5 @@ class IndividualToDomainsScanner(Scanner):# Make types available at module level for easy access-InputType = IndividualToDomainsScanner.InputType-OutputType = IndividualToDomainsScanner.OutputType+InputType = IndividualToDomainsTransform.InputType+OutputType = IndividualToDomainsTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/individual/to_org.py b/flowsint-transforms/src/flowsint_transforms/individual/to_org.pyindex 3ab74da9..9d376c3a 100644--- a/flowsint-transforms/src/flowsint_transforms/individual/to_org.py+++ b/flowsint-transforms/src/flowsint_transforms/individual/to_org.py@@ -1,13 +1,13 @@import requestsfrom typing import List, Dict, Union, Set-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.organization import Organizationfrom flowsint_types.individual import Individualfrom flowsint_core.core.logger import Loggerfrom tools.organizations.sirene import SireneTool-class IndividualToOrgScanner(Scanner):+class IndividualToOrgTransform(Transform):"""[SIRENE] Find organization from a person with data from SIRENE (France only)."""# Define types as class attributes - base class handles schema generation automatically@@ -340,5 +340,5 @@ class IndividualToOrgScanner(Scanner):# Make types available at module level for easy access-InputType = IndividualToOrgScanner.InputType-OutputType = IndividualToOrgScanner.OutputType+InputType = IndividualToOrgTransform.InputType+OutputType = IndividualToOrgTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/ip/to_asn.py b/flowsint-transforms/src/flowsint_transforms/ip/to_asn.pyindex 75364cfe..63281d96 100644--- a/flowsint-transforms/src/flowsint_transforms/ip/to_asn.py+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_asn.py@@ -2,14 +2,14 @@ import jsonimport socketimport subprocessfrom typing import List, Union-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.ip import Ipfrom flowsint_types.asn import ASNfrom flowsint_core.utils import is_valid_ipfrom flowsint_core.core.logger import Logger-class IpToAsnScanner(Scanner):+class IpToAsnTransform(Transform):"""[ASNMAP] Takes an IP address and returns its corresponding ASN."""# Define types as class attributes - base class handles schema generation automatically@@ -18,7 +18,7 @@ class IpToAsnScanner(Scanner):@classmethoddef name(cls) -> str:- return "ip_to_asn_scanner"+ return "ip_to_asn_transform"@classmethoddef category(cls) -> str:@@ -108,5 +108,5 @@ class IpToAsnScanner(Scanner):# Make types available at module level for easy access-InputType = IpToAsnScanner.InputType-OutputType = IpToAsnScanner.OutputType+InputType = IpToAsnTransform.InputType+OutputType = IpToAsnTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py b/flowsint-transforms/src/flowsint_transforms/ip/to_domain.pyindex fb4910d3..2cecd107 100644--- a/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py@@ -3,7 +3,7 @@ import socketimport requestsfrom typing import List, Unionfrom flowsint_core.core.logger import Logger-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.domain import Domainfrom flowsint_types.ip import Ipfrom flowsint_core.utils import is_valid_ip@@ -11,7 +11,7 @@ from flowsint_core.utils import is_valid_ipPTR_BLACKLIST = re.compile(r"^ip\d+\.ip-\d+-\d+-\d+-\d+\.")-class ReverseResolveScanner(Scanner):+class ReverseResolveTransform(Transform):"""Resolve IP addresses to domain names using PTR, Certificate Transparency and optional API calls."""# Define types as class attributes - base class handles schema generation automatically@@ -120,5 +120,5 @@ class ReverseResolveScanner(Scanner):# Make types available at module level for easy access-InputType = ReverseResolveScanner.InputType-OutputType = ReverseResolveScanner.OutputType+InputType = ReverseResolveTransform.InputType+OutputType = ReverseResolveTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/ip/to_infos.py b/flowsint-transforms/src/flowsint_transforms/ip/to_infos.pyindex 6f563cea..c76270e0 100644--- a/flowsint-transforms/src/flowsint_transforms/ip/to_infos.py+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_infos.py@@ -1,7 +1,7 @@import requestsfrom typing import List, Dict, Any, TypeAlias, Unionfrom pydantic import TypeAdapter-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.ip import Ip, Ipfrom flowsint_core.utils import resolve_type, is_valid_ip@@ -9,7 +9,7 @@ InputType: TypeAlias = List[Ip]OutputType: TypeAlias = List[Ip]-class IpToInfosScanner(Scanner):+class IpToInfosTransform(Transform):"""[ip-api.com] Get information data for IP addresses."""# Define types as class attributes - base class handles schema generation automatically@@ -131,5 +131,5 @@ class IpToInfosScanner(Scanner):return {}-InputType = IpToInfosScanner.InputType-OutputType = IpToInfosScanner.OutputType+InputType = IpToInfosTransform.InputType+OutputType = IpToInfosTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.pyindex c5b310e3..55b0dba8 100644--- a/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.py+++ b/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.py@@ -1,6 +1,6 @@from typing import Dict, Any, List, Unionimport hibpwned-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_core.core.logger import Loggerimport osfrom dotenv import load_dotenv@@ -11,7 +11,7 @@ load_dotenv()HIBP_API_KEY = os.getenv("HIBP_API_KEY")-class HibpScanner(Scanner):+class HibpTransform(Transform):"""Queries HaveIBeenPwned for potential leaks."""# Define types as class attributes - base class handles schema generation automatically@@ -116,5 +116,5 @@ class HibpScanner(Scanner):# Make types available at module level for easy access-InputType = HibpScanner.InputType-OutputType = HibpScanner.OutputType+InputType = HibpTransform.InputType+OutputType = HibpTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/n8n/connector.py b/flowsint-transforms/src/flowsint_transforms/n8n/connector.pyindex 36089513..46d0cb87 100644--- a/flowsint-transforms/src/flowsint_transforms/n8n/connector.py+++ b/flowsint-transforms/src/flowsint_transforms/n8n/connector.py@@ -1,12 +1,12 @@import jsonimport aiohttpfrom typing import List, Dict, Any, Optional-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_core.core.logger import Loggerfrom flowsint_core.core.graph_db import Neo4jConnection-class N8nConnector(Scanner):+class N8nConnector(Transform):"""Connect to your custom n8n workflows to process data through webhooks."""diff --git a/flowsint-transforms/src/flowsint_transforms/organization/to_asn.py b/flowsint-transforms/src/flowsint_transforms/organization/to_asn.pyindex c7ca4c62..6a368635 100644--- a/flowsint-transforms/src/flowsint_transforms/organization/to_asn.py+++ b/flowsint-transforms/src/flowsint_transforms/organization/to_asn.py@@ -1,13 +1,13 @@import jsonimport subprocessfrom typing import List, Dict, Any, Union-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.organization import Organizationfrom flowsint_types.asn import ASNfrom flowsint_core.core.logger import Logger-class OrgToAsnScanner(Scanner):+class OrgToAsnTransform(Transform):"""Takes an organization and returns its corresponding ASN."""# Define types as class attributes - base class handles schema generation automatically@@ -159,5 +159,5 @@ class OrgToAsnScanner(Scanner):# Make types available at module level for easy access-InputType = OrgToAsnScanner.InputType-OutputType = OrgToAsnScanner.OutputType+InputType = OrgToAsnTransform.InputType+OutputType = OrgToAsnTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py b/flowsint-transforms/src/flowsint_transforms/organization/to_domains.pyindex 609ee776..d4cd1b70 100644--- a/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py+++ b/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py@@ -1,7 +1,7 @@import osimport refrom typing import Any, List, Union, Dict, Set-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.domain import Domainfrom flowsint_types.organization import Organizationfrom flowsint_types.individual import Individual@@ -17,7 +17,7 @@ load_dotenv()WHOXY_API_KEY = os.getenv("WHOXY_API_KEY")-class OrgToDomainsScanner(Scanner):+class OrgToDomainsTransform(Transform):"""[WHOXY] Takes an organization and returns the domains it registered."""# Define types as class attributes - base class handles schema generation automatically@@ -406,5 +406,5 @@ class OrgToDomainsScanner(Scanner):)-InputType = OrgToDomainsScanner.InputType-OutputType = OrgToDomainsScanner.OutputType+InputType = OrgToDomainsTransform.InputType+OutputType = OrgToDomainsTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/organization/to_infos.py b/flowsint-transforms/src/flowsint_transforms/organization/to_infos.pyindex 24d13e9f..4d241a0a 100644--- a/flowsint-transforms/src/flowsint_transforms/organization/to_infos.py+++ b/flowsint-transforms/src/flowsint_transforms/organization/to_infos.py@@ -1,11 +1,11 @@from typing import List, Dict, Any, Union-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.organization import Organizationfrom flowsint_core.core.logger import Loggerfrom tools.organizations.sirene import SireneTool-class OrgToInfosScanner(Scanner):+class OrgToInfosTransform(Transform):"""Enrich Organization with data from SIRENE (France only)."""# Define types as class attributes - base class handles schema generation automatically@@ -412,5 +412,5 @@ class OrgToInfosScanner(Scanner):# Make types available at module level for easy access-InputType = OrgToInfosScanner.InputType-OutputType = OrgToInfosScanner.OutputType+InputType = OrgToInfosTransform.InputType+OutputType = OrgToInfosTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py b/flowsint-transforms/src/flowsint_transforms/phone/to_infos.pyindex cbcebeb5..a041ecbd 100644--- a/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py+++ b/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py@@ -1,13 +1,13 @@import asynciofrom typing import Dict, Any, List, Union-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_core.utils import is_valid_numberfrom flowsint_core.core.logger import Loggerfrom flowsint_types.phone import Phoneimport httpx-class IgnorantScanner(Scanner):+class IgnorantTransform(Transform):# Define types as class attributes - base class handles schema generation automaticallyInputType = List[str] # Phone numbers as strings@@ -120,5 +120,5 @@ class IgnorantScanner(Scanner):# Make types available at module level for easy access-InputType = IgnorantScanner.InputType-OutputType = IgnorantScanner.OutputType+InputType = IgnorantTransform.InputType+OutputType = IgnorantTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.pyindex a7dc6a89..e40cfe32 100644--- a/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.py+++ b/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.py@@ -2,7 +2,7 @@ import osfrom typing import Any, Dict, List, Optional, Unionimport requestsfrom urllib.parse import urljoin-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_core.core.logger import Loggerfrom flowsint_types.phone import Phonefrom flowsint_types.breach import Breach@@ -15,7 +15,7 @@ load_dotenv()HIBP_API_KEY = os.getenv("HIBP_API_KEY")-class PhoneToBreachesScanner(Scanner):+class PhoneToBreachesTransform(Transform):"""[HIBPWNED] Get the breaches the phone number might be invovled in."""InputType = List[Phone]@@ -56,7 +56,7 @@ class PhoneToBreachesScanner(Scanner):@classmethoddef get_params_schema(cls) -> List[Dict[str, Any]]:- """Declare required parameters for this scanner"""+ """Declare required parameters for this transform"""return [{"name": "HIBP_API_KEY",@@ -102,7 +102,7 @@ class PhoneToBreachesScanner(Scanner):Logger.error(self.sketch_id, {"message": "Could not find HIBP_API_URL in params."})- headers = {"hibp-api-key": api_key, "User-Agent": "FlowsInt-Scanner"}+ headers = {"hibp-api-key": api_key, "User-Agent": "FlowsInt-Transform"}Logger.info(self.sketch_id, {"message": f"HIBP API URL: {api_url}"})for phone in data:try:@@ -235,5 +235,5 @@ class PhoneToBreachesScanner(Scanner):# Make types available at module level for easy access-InputType = PhoneToBreachesScanner.InputType-OutputType = PhoneToBreachesScanner.OutputType+InputType = PhoneToBreachesTransform.InputType+OutputType = PhoneToBreachesTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py b/flowsint-transforms/src/flowsint_transforms/social/to_maigret.pyindex 53d2d206..29e53567 100644--- a/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py+++ b/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py@@ -3,14 +3,14 @@ import subprocessfrom pathlib import Pathfrom typing import List, Unionfrom flowsint_core.utils import is_valid_username-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.social import SocialProfilefrom flowsint_core.core.logger import Loggerfalse_positives = ["LeagueOfLegends"]-class MaigretScanner(Scanner):+class MaigretTransform(Transform):"""[MAIGRET] Scans usernames for associated social accounts using Maigret."""# Define types as class attributes - base class handles schema generation automatically@@ -182,5 +182,5 @@ class MaigretScanner(Scanner):# Make types available at module level for easy access-InputType = MaigretScanner.InputType-OutputType = MaigretScanner.OutputType+InputType = MaigretTransform.InputType+OutputType = MaigretTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py b/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.pyindex 2303b633..5081ca59 100644--- a/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py+++ b/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py@@ -3,11 +3,11 @@ from pathlib import Pathfrom typing import List, Unionfrom flowsint_core.utils import is_valid_usernamefrom flowsint_types.social import SocialProfile-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_core.core.logger import Logger-class SherlockScanner(Scanner):+class SherlockTransform(Transform):"""[SHERLOCK] Scans the usernames for associated social accounts using Sherlock."""# Define types as class attributes - base class handles schema generation automatically@@ -127,5 +127,5 @@ class SherlockScanner(Scanner):# Make types available at module level for easy access-InputType = SherlockScanner.InputType-OutputType = SherlockScanner.OutputType+InputType = SherlockTransform.InputType+OutputType = SherlockTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/utils.py b/flowsint-transforms/src/flowsint_transforms/utils.pyindex 6a752679..b912a880 100644--- a/flowsint-transforms/src/flowsint_transforms/utils.py+++ b/flowsint-transforms/src/flowsint_transforms/utils.py@@ -268,18 +268,18 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:input_output = input_node["data"]["outputs"]node_lookup = {node["id"]: node for node in nodes}- scanners = []+ transforms = []for edge in edges:target_id = edge["target"]source_handle = edge["sourceHandle"]target_handle = edge["targetHandle"]- scanner_node = node_lookup.get(target_id)- if scanner_node and scanner_node["data"]["type"] == "scanner":- scanners.append(+ transform_node = node_lookup.get(target_id)+ if transform_node and transform_node["data"]["type"] == "transform":+ transforms.append({- "transform_name": scanner_node["data"]["name"],- "module": scanner_node["data"]["module"],+ "transform_name": transform_node["data"]["name"],+ "module": transform_node["data"]["module"],"input": source_handle,"output": target_handle,}@@ -290,8 +290,8 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:"name": input_node["data"]["name"],"outputs": input_output,},- "scanners": scanners,- "transform_names": [scanner["transform_name"] for scanner in scanners],+ "transforms": transforms,+ "transform_names": [transform["transform_name"] for transform in transforms],}diff --git a/flowsint-transforms/src/flowsint_transforms/website/to_crawler.py b/flowsint-transforms/src/flowsint_transforms/website/to_crawler.pyindex ab2fb295..67e86044 100644--- a/flowsint-transforms/src/flowsint_transforms/website/to_crawler.py+++ b/flowsint-transforms/src/flowsint_transforms/website/to_crawler.py@@ -1,6 +1,6 @@from typing import List, Optional, Unionfrom urllib.parse import urlparse-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.website import Websitefrom flowsint_types.phone import Phonefrom flowsint_types.email import Email@@ -15,7 +15,7 @@ class ReturnType(BaseModel):phones: Optional[Phone]-class WebsiteToCrawler(Scanner):+class WebsiteToCrawler(Transform):"""From website to crawler."""# Define types as class attributes - base class handles schema generation automaticallydiff --git a/flowsint-transforms/src/flowsint_transforms/website/to_domain.py b/flowsint-transforms/src/flowsint_transforms/website/to_domain.pyindex a9e0e5a5..cb400323 100644--- a/flowsint-transforms/src/flowsint_transforms/website/to_domain.py+++ b/flowsint-transforms/src/flowsint_transforms/website/to_domain.py@@ -1,12 +1,12 @@from typing import List, Unionfrom urllib.parse import urlparse-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.website import Websitefrom flowsint_types.domain import Domainfrom flowsint_core.core.logger import Logger-class WebsiteToDomainScanner(Scanner):+class WebsiteToDomainTransform(Transform):"""From website to domain."""# Define types as class attributes - base class handles schema generation automatically@@ -110,5 +110,5 @@ class WebsiteToDomainScanner(Scanner):# Make types available at module level for easy access-InputType = WebsiteToDomainScanner.InputType-OutputType = WebsiteToDomainScanner.OutputType+InputType = WebsiteToDomainTransform.InputType+OutputType = WebsiteToDomainTransform.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/website/to_links.py b/flowsint-transforms/src/flowsint_transforms/website/to_links.pyindex bc90ee31..f31e3f37 100644--- a/flowsint-transforms/src/flowsint_transforms/website/to_links.py+++ b/flowsint-transforms/src/flowsint_transforms/website/to_links.py@@ -1,12 +1,12 @@from typing import List, Unionfrom urllib.parse import urlparse-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.website import Websitefrom flowsint_core.core.logger import Loggerfrom reconspread import Crawler-class WebsiteToLinks(Scanner):+class WebsiteToLinks(Transform):"""From website to spread crawler that extracts domains and internal/external links."""# Define types as class attributes - base class handles schema generation automaticallydiff --git a/flowsint-transforms/src/flowsint_transforms/website/to_text.py b/flowsint-transforms/src/flowsint_transforms/website/to_text.pyindex 825ef88d..a23757a1 100644--- a/flowsint-transforms/src/flowsint_transforms/website/to_text.py+++ b/flowsint-transforms/src/flowsint_transforms/website/to_text.py@@ -1,12 +1,12 @@from typing import List, Union-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.phrase import Phrasefrom flowsint_types.website import Websiteimport requestsfrom bs4 import BeautifulSoup-class WebsiteToText(Scanner):+class WebsiteToText(Transform):"""Extracts the texts in a webpage."""InputType = List[Website]diff --git a/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py b/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.pyindex cb5f77e8..4ddefaac 100644--- a/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py+++ b/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py@@ -1,5 +1,5 @@from typing import List, Dict, Any, Union, Optional-from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.transform_base import Transformfrom flowsint_types.website import Websitefrom flowsint_types.web_tracker import WebTrackerfrom flowsint_core.core.logger import Logger@@ -8,7 +8,7 @@ from flowsint_core.core.vault import VaultProtocolfrom recontrack import TrackingCodeExtractor-class WebsiteToWebtrackersScanner(Scanner):+class WebsiteToWebtrackersTransform(Transform):"""From website to webtrackers."""# Define types as class attributes - base class handles schema generation automatically@@ -128,5 +128,5 @@ class WebsiteToWebtrackersScanner(Scanner):# Make types available at module level for easy access-InputType = WebsiteToWebtrackersScanner.InputType-OutputType = WebsiteToWebtrackersScanner.OutputType+InputType = WebsiteToWebtrackersTransform.InputType+OutputType = WebsiteToWebtrackersTransform.OutputTypediff --git a/flowsint-transforms/tests/insert_command.py b/flowsint-transforms/tests/insert_command.pyindex 8e6cc2de..4df5a035 100644--- a/flowsint-transforms/tests/insert_command.py+++ b/flowsint-transforms/tests/insert_command.py@@ -7,7 +7,7 @@ if __name__ == "__main__":from flowsint_types.domain import Domainfrom flowsint_types.ip import Ip-from flowsint_transforms.domains.resolve import ResolveScanner+from flowsint_transforms.domains.resolve import ResolveTransformasync def main():@@ -15,11 +15,11 @@ async def main():domains = [Domain(domain="adaltas.com")]ips = [Ip(address="12.23.34.45"), Ip(address="56.67.78.89")]- # Test the scanner- scanner = ResolveScanner("sketch_123", "scan_123")+ # Test the transform+ transform = ResolveTransform("sketch_123", "scan_123")# Test the new KISS postprocess method- scanner.postprocess(ips[:1], domains) # Only use first IP to match domains length+ transform.postprocess(ips[:1], domains) # Only use first IP to match domains lengthprint("Postprocess test completed successfully!")diff --git a/flowsint-transforms/tests/scanners/__init__.py b/flowsint-transforms/tests/transforms/__init__.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/__init__.pyrename to flowsint-transforms/tests/transforms/__init__.pydiff --git a/flowsint-transforms/tests/scanners/crypto/__init__.py b/flowsint-transforms/tests/transforms/crypto/__init__.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/crypto/__init__.pyrename to flowsint-transforms/tests/transforms/crypto/__init__.pydiff --git a/flowsint-transforms/tests/scanners/crypto/wallet_to_nfts.py b/flowsint-transforms/tests/transforms/crypto/wallet_to_nfts.pysimilarity index 83%rename from flowsint-transforms/tests/scanners/crypto/wallet_to_nfts.pyrename to flowsint-transforms/tests/transforms/crypto/wallet_to_nfts.pyindex 9e7778e6..6ad61434 100644--- a/flowsint-transforms/tests/scanners/crypto/wallet_to_nfts.py+++ b/flowsint-transforms/tests/transforms/crypto/wallet_to_nfts.py@@ -2,24 +2,24 @@ from flowsint_transforms.crypto.wallet_to_nfts import CryptoWalletAddressToNFTsfrom flowsint_types.wallet import CryptoWallet, CryptoNFTfrom pydantic import HttpUrl-scanner = CryptoWalletAddressToNFTs("sketch_123", "scan_123")+transform = CryptoWalletAddressToNFTs("sketch_123", "scan_123")def test_wallet_address_to_transactions_name():- assert scanner.name() == "wallet_to_nfts"+ assert transform.name() == "wallet_to_nfts"def test_wallet_address_to_transactions_category():- assert scanner.category() == "crypto"+ assert transform.category() == "crypto"def test_wallet_address_to_transactions_key():- assert scanner.key() == "address"+ assert transform.key() == "address"def test_preprocess_with_string():input_data = ["0x742d35Cc6634C0532925a3b844Bc454e4438f44e"]- result = scanner.preprocess(input_data)+ result = transform.preprocess(input_data)assert len(result) == 1assert isinstance(result[0], CryptoWallet)assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"@@ -27,7 +27,7 @@ def test_preprocess_with_string():def test_preprocess_with_dict():input_data = [{"address": "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"}]- result = scanner.preprocess(input_data)+ result = transform.preprocess(input_data)assert len(result) == 1assert isinstance(result[0], CryptoWallet)assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"@@ -36,7 +36,7 @@ def test_preprocess_with_dict():def test_preprocess_with_wallet_object():wallet = CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")input_data = [wallet]- result = scanner.preprocess(input_data)+ result = transform.preprocess(input_data)assert len(result) == 1assert isinstance(result[0], CryptoWallet)assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"@@ -59,10 +59,10 @@ def test_scan_mocked_transactions(monkeypatch):)]- monkeypatch.setattr(scanner, "_get_nfts", mock_get_nfts)+ monkeypatch.setattr(transform, "_get_nfts", mock_get_nfts)input_data = [CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")]- result = scanner.scan(input_data)+ result = transform.scan(input_data)assert len(result) == 1assert len(result[0]) == 1diff --git a/flowsint-transforms/tests/scanners/crypto/wallet_to_transactions.py b/flowsint-transforms/tests/transforms/crypto/wallet_to_transactions.pysimilarity index 76%rename from flowsint-transforms/tests/scanners/crypto/wallet_to_transactions.pyrename to flowsint-transforms/tests/transforms/crypto/wallet_to_transactions.pyindex af0deb74..fd4c4090 100644--- a/flowsint-transforms/tests/scanners/crypto/wallet_to_transactions.py+++ b/flowsint-transforms/tests/transforms/crypto/wallet_to_transactions.py@@ -4,7 +4,7 @@ from flowsint_transforms.crypto.wallet_to_transactions import ()from flowsint_types.wallet import CryptoWallet, CryptoWalletTransaction-scanner = CryptoWalletAddressToTransactions(+transform = CryptoWalletAddressToTransactions("sketch_123","scan_123",params={"ETHERSCAN_API_KEY": "ta-clef-api"},@@ -12,20 +12,20 @@ scanner = CryptoWalletAddressToTransactions(def test_wallet_address_to_transactions_name():- assert scanner.name() == "wallet_to_transactions"+ assert transform.name() == "wallet_to_transactions"def test_wallet_address_to_transactions_category():- assert scanner.category() == "CryptoCryptoWallet"+ assert transform.category() == "CryptoCryptoWallet"def test_wallet_address_to_transactions_key():- assert scanner.key() == "address"+ assert transform.key() == "address"def test_preprocess_with_string():input_data = ["0x742d35Cc6634C0532925a3b844Bc454e4438f44e"]- result = scanner.preprocess(input_data)+ result = transform.preprocess(input_data)assert len(result) == 1assert isinstance(result[0], CryptoWallet)assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"@@ -33,7 +33,7 @@ def test_preprocess_with_string():def test_preprocess_with_dict():input_data = [{"address": "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"}]- result = scanner.preprocess(input_data)+ result = transform.preprocess(input_data)assert len(result) == 1assert isinstance(result[0], CryptoWallet)assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"@@ -42,7 +42,7 @@ def test_preprocess_with_dict():def test_preprocess_with_wallet_object():wallet = CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")input_data = [wallet]- result = scanner.preprocess(input_data)+ result = transform.preprocess(input_data)assert len(result) == 1assert isinstance(result[0], CryptoWallet)assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"@@ -74,10 +74,10 @@ async def test_scan_mocked_transactions(monkeypatch):)]- monkeypatch.setattr(scanner, "_get_transactions", mock_get_transactions)+ monkeypatch.setattr(transform, "_get_transactions", mock_get_transactions)input_data = [CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")]- result = await scanner.scan(input_data)+ result = await transform.scan(input_data)assert len(result) == 1assert len(result[0]) == 1@@ -88,18 +88,18 @@ async def test_scan_mocked_transactions(monkeypatch):assert result[0][0].timestamp == "1234567890"-def test_scanner_requires_api_key():- """Test that the scanner validates required ETHERSCAN_API_KEY parameter at construction"""+def test_transform_requires_api_key():+ """Test that the transform validates required ETHERSCAN_API_KEY parameter at construction"""with pytest.raises(- ValueError, match="Scanner wallet_to_transactions received invalid params"+ ValueError, match="Transform wallet_to_transactions received invalid params"):CryptoWalletAddressToTransactions("sketch_123", "scan_123", params={})-def test_scanner_with_invalid_api_key_type():- """Test that the scanner validates parameter types"""+def test_transform_with_invalid_api_key_type():+ """Test that the transform validates parameter types"""with pytest.raises(- ValueError, match="Scanner wallet_to_transactions received invalid params"+ ValueError, match="Transform wallet_to_transactions received invalid params"):CryptoWalletAddressToTransactions("sketch_123", "scan_123", params={"ETHERSCAN_API_KEY": 123}diff --git a/flowsint-transforms/tests/scanners/domain/__init__.py b/flowsint-transforms/tests/transforms/domain/__init__.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/domain/__init__.pyrename to flowsint-transforms/tests/transforms/domain/__init__.pydiff --git a/flowsint-transforms/tests/scanners/domain/resolve.py b/flowsint-transforms/tests/transforms/domain/resolve.pysimilarity index 71%rename from flowsint-transforms/tests/scanners/domain/resolve.pyrename to flowsint-transforms/tests/transforms/domain/resolve.pyindex de72d91a..bccf0101 100644--- a/flowsint-transforms/tests/scanners/domain/resolve.py+++ b/flowsint-transforms/tests/transforms/domain/resolve.py@@ -1,10 +1,10 @@-from flowsint_transforms.domains.resolve import ResolveScanner+from flowsint_transforms.domains.resolve import ResolveTransformfrom flowsint_types.domain import Domainfrom flowsint_types.ip import Ipfrom typing import Listimport pytest-scanner = ResolveScanner("sketch_123", "scan_123")+transform = ResolveTransform("sketch_123", "scan_123")def test_preprocess_valid_domains():@@ -12,7 +12,7 @@ def test_preprocess_valid_domains():Domain(domain="example.com"),Domain(domain="example2.com"),]- result = scanner.preprocess(domains)+ result = transform.preprocess(domains)result_domains = [d.domain for d in result]expected_domains = [d.domain for d in domains]@@ -25,7 +25,7 @@ def test_unprocessed_valid_domains():"example.com","example2.com",]- result = scanner.preprocess(domains)+ result = transform.preprocess(domains)result_domains = [d for d in result]expected_domains = [Domain(domain=d) for d in domains]assert result_domains == expected_domains@@ -37,7 +37,7 @@ def test_preprocess_invalid_domains():Domain(domain="invalid_domain"),Domain(domain="example.org"),]- result = scanner.preprocess(domains)+ result = transform.preprocess(domains)result_domains = [d.domain for d in result]assert "example.com" in result_domains@@ -52,7 +52,7 @@ def test_preprocess_multiple_formats():Domain(domain="example.org"),"example.org",]- result = scanner.preprocess(domains)+ result = transform.preprocess(domains)result_domains = [d.domain for d in result]assert "example.com" in result_domains@@ -70,15 +70,15 @@ async def test_scan_returns_ip(monkeypatch):monkeypatch.setattr("socket.gethostbyname", mock_gethostbyname)input_data = [Domain(domain="example.com")]- output = await scanner.execute(input_data)+ output = await transform.execute(input_data)print(output)assert isinstance(output, list)assert output[0].address == "12.23.34.45"def test_schemas():- input_schema = scanner.input_schema()- output_schema = scanner.output_schema()+ input_schema = transform.input_schema()+ output_schema = transform.output_schema()# Test the structure and key properties rather than exact matchassert input_schema["type"] == "Domain"@@ -93,20 +93,20 @@ def test_schemas():class TestResolveInputOutputTypes:- """Test the InputType/OutputType functionality for ResolveScanner"""+ """Test the InputType/OutputType functionality for ResolveTransform"""def test_input_output_types_are_defined(self):"""Test that InputType and OutputType are properly defined"""- assert hasattr(ResolveScanner, "InputType")- assert hasattr(ResolveScanner, "OutputType")- assert ResolveScanner.InputType == List[Domain]- assert ResolveScanner.OutputType == List[Ip]+ assert hasattr(ResolveTransform, "InputType")+ assert hasattr(ResolveTransform, "OutputType")+ assert ResolveTransform.InputType == List[Domain]+ assert ResolveTransform.OutputType == List[Ip]def test_schemas_use_generate_methods(self):"""Test that schema methods use the new generate methods"""# These should work without error- input_schema = ResolveScanner.generate_input_schema()- output_schema = ResolveScanner.generate_output_schema()+ input_schema = ResolveTransform.generate_input_schema()+ output_schema = ResolveTransform.generate_output_schema()assert isinstance(input_schema, dict)assert isinstance(output_schema, dict)@@ -115,12 +115,12 @@ class TestResolveInputOutputTypes:def test_schema_methods_return_same_as_generate_methods(self):"""Test that input_schema() and output_schema() return the same as generate methods"""- assert ResolveScanner.input_schema() == ResolveScanner.generate_input_schema()- assert ResolveScanner.output_schema() == ResolveScanner.generate_output_schema()+ assert ResolveTransform.input_schema() == ResolveTransform.generate_input_schema()+ assert ResolveTransform.output_schema() == ResolveTransform.generate_output_schema()def test_input_schema_properties(self):"""Test input schema has expected properties"""- schema = ResolveScanner.input_schema()+ schema = ResolveTransform.input_schema()properties = schema["properties"]property_names = [p["name"] for p in properties]@@ -130,7 +130,7 @@ class TestResolveInputOutputTypes:def test_output_schema_properties(self):"""Test output schema has expected properties"""- schema = ResolveScanner.output_schema()+ schema = ResolveTransform.output_schema()properties = schema["properties"]property_names = [p["name"] for p in properties]@@ -139,15 +139,15 @@ class TestResolveInputOutputTypes:assert "address" in property_namesdef test_type_accessibility_from_instance(self):- """Test that types are accessible from scanner instance"""- scanner_instance = ResolveScanner("test", "test")+ """Test that types are accessible from transform instance"""+ transform_instance = ResolveTransform("test", "test")- assert scanner_instance.InputType == List[Domain]- assert scanner_instance.OutputType == List[Ip]+ assert transform_instance.InputType == List[Domain]+ assert transform_instance.OutputType == List[Ip]# Should be able to generate schemas from instance- input_schema = scanner_instance.generate_input_schema()- output_schema = scanner_instance.generate_output_schema()+ input_schema = transform_instance.generate_input_schema()+ output_schema = transform_instance.generate_output_schema()assert input_schema["type"] == "Domain"assert output_schema["type"] == "Ip"diff --git a/flowsint-transforms/tests/scanners/domain/subdomains.py b/flowsint-transforms/tests/transforms/domain/subdomains.pysimilarity index 85%rename from flowsint-transforms/tests/scanners/domain/subdomains.pyrename to flowsint-transforms/tests/transforms/domain/subdomains.pyindex d323721c..16ee1835 100644--- a/flowsint-transforms/tests/scanners/domain/subdomains.py+++ b/flowsint-transforms/tests/transforms/domain/subdomains.py@@ -1,7 +1,7 @@-from flowsint_transforms.domains.subdomains import SubdomainScanner+from flowsint_transforms.domains.subdomains import SubdomainTransformfrom flowsint_types.domain import Domain, Domain-scanner = SubdomainScanner("sketch_123", "scan_123")+transform = SubdomainTransform("sketch_123", "scan_123")def test_preprocess_valid_domains():@@ -9,7 +9,7 @@ def test_preprocess_valid_domains():Domain(domain="example.com"),Domain(domain="example2.com"),]- result = scanner.preprocess(domains)+ result = transform.preprocess(domains)result_domains = [d.domain for d in result]expected_domains = [d.domain for d in domains]@@ -22,7 +22,7 @@ def test_unprocessed_valid_domains():"example.com","example2.com",]- result = scanner.preprocess(domains)+ result = transform.preprocess(domains)result_domains = [d for d in result]expected_domains = [Domain(domain=d) for d in domains]assert result_domains == expected_domains@@ -34,7 +34,7 @@ def test_preprocess_invalid_domains():Domain(domain="invalid_domain"),Domain(domain="example.org"),]- result = scanner.preprocess(domains)+ result = transform.preprocess(domains)result_domains = [d.domain for d in result]assert "example.com" in result_domains@@ -49,7 +49,7 @@ def test_preprocess_multiple_formats():Domain(domain="example.org"),"example.org",]- result = scanner.preprocess(domains)+ result = transform.preprocess(domains)result_domains = [d.domain for d in result]assert "example.com" in result_domains@@ -81,11 +81,11 @@ def test_scan_extracts_subdomains(monkeypatch):assert "example.com" in urlreturn MockRequestsResponse(mock_response)- # Patch la requête réseau dans le module scanner+ # Patch la requête réseau dans le module transformmonkeypatch.setattr("requests.get", mock_get)input_data = [Domain(domain="example.com")]- domains = scanner.execute(input_data)+ domains = transform.execute(input_data)assert isinstance(domains, list)for sub in domains:print(sub)diff --git a/flowsint-transforms/tests/scanners/domain/to_history.py b/flowsint-transforms/tests/transforms/domain/to_history.pysimilarity index 69%rename from flowsint-transforms/tests/scanners/domain/to_history.pyrename to flowsint-transforms/tests/transforms/domain/to_history.pyindex bdb62db5..73619dbd 100644--- a/flowsint-transforms/tests/scanners/domain/to_history.py+++ b/flowsint-transforms/tests/transforms/domain/to_history.py@@ -2,7 +2,7 @@ import pytestimport jsonimport osfrom unittest.mock import Mock-from flowsint_transforms.domain.to_history import DomainToHistoryScanner+from flowsint_transforms.domain.to_history import DomainToHistoryTransformfrom flowsint_types.domain import Domain@@ -37,7 +37,7 @@ class MockNeo4jConn:pass-class MockScanner(DomainToHistoryScanner):+class MockTransform(DomainToHistoryTransform):def __init__(self):self.sketch_id = "test_sketch_123"self.neo4j_conn = MockNeo4jConn()@@ -50,10 +50,10 @@ class MockScanner(DomainToHistoryScanner):@pytest.fixture-def scanner():- """Create a scanner instance for testing."""- scanner = MockScanner()- return scanner+def transform():+ """Create a transform instance for testing."""+ transform = MockTransform()+ return transform@pytest.fixture@@ -65,13 +65,13 @@ def test_data():return json.load(f)-def test_preprocess_valid_domains(scanner):+def test_preprocess_valid_domains(transform):"""Test preprocessing with valid domains."""domains = [Domain(domain="example.com"),Domain(domain="example2.com"),]- result = scanner.preprocess(domains)+ result = transform.preprocess(domains)result_domains = [d.domain for d in result]expected_domains = [d.domain for d in domains]@@ -79,10 +79,10 @@ def test_preprocess_valid_domains(scanner):assert result_domains == expected_domains-def test_preprocess_string_domains(scanner):+def test_preprocess_string_domains(transform):"""Test preprocessing with string domains."""domains = ["example.com", "example2.com"]- result = scanner.preprocess(domains)+ result = transform.preprocess(domains)assert len(result) == 2assert all(isinstance(d, Domain) for d in result)@@ -90,10 +90,10 @@ def test_preprocess_string_domains(scanner):assert result[1].domain == "example2.com"-def test_preprocess_dict_domains(scanner):+def test_preprocess_dict_domains(transform):"""Test preprocessing with dict domains."""domains = [{"domain": "example.com"}, {"domain": "example2.com"}]- result = scanner.preprocess(domains)+ result = transform.preprocess(domains)assert len(result) == 2assert all(isinstance(d, Domain) for d in result)@@ -101,14 +101,14 @@ def test_preprocess_dict_domains(scanner):assert result[1].domain == "example2.com"-def test_preprocess_invalid_domains(scanner):+def test_preprocess_invalid_domains(transform):"""Test preprocessing with invalid domains."""domains = [Domain(domain="example.com"),Domain(domain="invalid_domain"),Domain(domain="example.org"),]- result = scanner.preprocess(domains)+ result = transform.preprocess(domains)result_domains = [d.domain for d in result]assert "example.com" in result_domains@@ -116,21 +116,21 @@ def test_preprocess_invalid_domains(scanner):assert "invalid_domain" not in result_domains-def test_is_redacted(scanner):+def test_is_redacted(transform):"""Test the __is_redacted method."""# Should be redacted- assert scanner._DomainToHistoryScanner__is_redacted("REDACTED FOR PRIVACY")- assert scanner._DomainToHistoryScanner__is_redacted("redacted for privacy")- assert scanner._DomainToHistoryScanner__is_redacted("Some text with PRIVACY in it")+ assert transform._DomainToHistoryTransform__is_redacted("REDACTED FOR PRIVACY")+ assert transform._DomainToHistoryTransform__is_redacted("redacted for privacy")+ assert transform._DomainToHistoryTransform__is_redacted("Some text with PRIVACY in it")# Should NOT be redacted- assert not scanner._DomainToHistoryScanner__is_redacted("JOHN DOE")- assert not scanner._DomainToHistoryScanner__is_redacted("[email protected]")- assert not scanner._DomainToHistoryScanner__is_redacted("123 JOHN STREET")- assert not scanner._DomainToHistoryScanner__is_redacted("DOE CITY")+ assert not transform._DomainToHistoryTransform__is_redacted("JOHN DOE")+ assert not transform._DomainToHistoryTransform__is_redacted("[email protected]")+ assert not transform._DomainToHistoryTransform__is_redacted("123 JOHN STREET")+ assert not transform._DomainToHistoryTransform__is_redacted("DOE CITY")-def test_has_non_redacted_info(scanner):+def test_has_non_redacted_info(transform):"""Test the __has_non_redacted_info method."""# Contact with valid informationvalid_contact = {@@ -142,7 +142,7 @@ def test_has_non_redacted_info(scanner):"zip_code": "12345","country_name": "United States",}- assert scanner._DomainToHistoryScanner__has_non_redacted_info(valid_contact)+ assert transform._DomainToHistoryTransform__has_non_redacted_info(valid_contact)# Contact with all redacted informationredacted_contact = {@@ -154,13 +154,13 @@ def test_has_non_redacted_info(scanner):"zip_code": "REDACTED FOR PRIVACY","country_name": "REDACTED FOR PRIVACY",}- assert not scanner._DomainToHistoryScanner__has_non_redacted_info(redacted_contact)+ assert not transform._DomainToHistoryTransform__has_non_redacted_info(redacted_contact)# Empty contact- assert not scanner._DomainToHistoryScanner__has_non_redacted_info({})+ assert not transform._DomainToHistoryTransform__has_non_redacted_info({})-def test_extract_individual_from_contact(scanner):+def test_extract_individual_from_contact(transform):"""Test the __extract_individual_from_contact method."""# Valid contactvalid_contact = {@@ -173,7 +173,7 @@ def test_extract_individual_from_contact(scanner):"country_name": "United States",}- individual = scanner._DomainToHistoryScanner__extract_individual_from_contact(+ individual = transform._DomainToHistoryTransform__extract_individual_from_contact(valid_contact, "REGISTRANT")@@ -187,7 +187,7 @@ def test_extract_individual_from_contact(scanner):assert individual.phone_numbers == ["+123456789"]-def test_extract_individual_redacted_name(scanner):+def test_extract_individual_redacted_name(transform):"""Test that individuals with redacted names are skipped."""redacted_contact = {"full_name": "REDACTED FOR PRIVACY",@@ -195,27 +195,27 @@ def test_extract_individual_redacted_name(scanner):"phone_number": "+1234567890",}- individual = scanner._DomainToHistoryScanner__extract_individual_from_contact(+ individual = transform._DomainToHistoryTransform__extract_individual_from_contact(redacted_contact, "REGISTRANT")assert individual is None-def test_is_valid_email(scanner):+def test_is_valid_email(transform):"""Test the __is_valid_email method."""# Valid emails- assert scanner._DomainToHistoryScanner__is_valid_email("[email protected]")- assert scanner._DomainToHistoryScanner__is_valid_email("[email protected]")- assert scanner._DomainToHistoryScanner__is_valid_email("[email protected]")+ assert transform._DomainToHistoryTransform__is_valid_email("[email protected]")+ assert transform._DomainToHistoryTransform__is_valid_email("[email protected]")+ assert transform._DomainToHistoryTransform__is_valid_email("[email protected]")# Invalid emails- assert not scanner._DomainToHistoryScanner__is_valid_email("invalid-email")- assert not scanner._DomainToHistoryScanner__is_valid_email("@example.com")- assert not scanner._DomainToHistoryScanner__is_valid_email("test@")- assert not scanner._DomainToHistoryScanner__is_valid_email("")+ assert not transform._DomainToHistoryTransform__is_valid_email("invalid-email")+ assert not transform._DomainToHistoryTransform__is_valid_email("@example.com")+ assert not transform._DomainToHistoryTransform__is_valid_email("test@")+ assert not transform._DomainToHistoryTransform__is_valid_email("")-def test_extract_physical_address(scanner):+def test_extract_physical_address(transform):"""Test the __extract_physical_address method."""# Valid addressvalid_contact = {@@ -225,7 +225,7 @@ def test_extract_physical_address(scanner):"country_name": "United States",}- address = scanner._DomainToHistoryScanner__extract_physical_address(valid_contact)+ address = transform._DomainToHistoryTransform__extract_physical_address(valid_contact)assert address is not Noneassert address.address == "123 JOHN STREET"@@ -241,14 +241,14 @@ def test_extract_physical_address(scanner):"country_name": "United States",}- address = scanner._DomainToHistoryScanner__extract_physical_address(+ address = transform._DomainToHistoryTransform__extract_physical_address(redacted_contact)assert address is None@pytest.mark.asyncio-async def test_scan_with_test_data(scanner, test_data, monkeypatch):+async def test_scan_with_test_data(transform, test_data, monkeypatch):"""Test the scan method with test data."""# Mock the __get_infos_from_whoxy method to return test data@@ -258,30 +258,30 @@ async def test_scan_with_test_data(scanner, test_data, monkeypatch):return {}monkeypatch.setattr(- scanner, "_DomainToHistoryScanner__get_infos_from_whoxy", mock_get_infos+ transform, "_DomainToHistoryTransform__get_infos_from_whoxy", mock_get_infos)# Test with epios.com domaininput_domains = [Domain(domain="epios.com")]- results = await scanner.scan(input_domains)+ results = await transform.scan(input_domains)# Should find the domain (one for each WHOIS record)assert len(results) == 16 # 16 WHOIS records in the test dataassert all(r.domain == "epios.com" for r in results)# Should have extracted data- assert len(scanner._extracted_data) == 16+ assert len(transform._extracted_data) == 16# Should have extracted individualsassert (- len(scanner._extracted_individuals) > 0+ len(transform._extracted_individuals) > 0), "Should have extracted some individuals"# Check that JOHN DOE is in the extracted individualsmarc_found = Falsemarc_individuals = []- for individual_info in scanner._extracted_individuals:+ for individual_info in transform._extracted_individuals:individual = individual_info["individual"]if "JOHN DOE" in individual.full_name:marc_found = True@@ -299,7 +299,7 @@ async def test_scan_with_test_data(scanner, test_data, monkeypatch):# Print summary of all extracted individualsprint(f"\n=== Summary of extracted individuals ===")- for individual_info in scanner._extracted_individuals:+ for individual_info in transform._extracted_individuals:individual = individual_info["individual"]print(f"- {individual.full_name} ({individual_info['contact_type']}) for {individual_info['domain_name']}"@@ -310,7 +310,7 @@ async def test_scan_with_test_data(scanner, test_data, monkeypatch):print(f" Phones: {individual.phone_numbers}")-def test_postprocess_creates_nodes_and_relationships(scanner, test_data, monkeypatch):+def test_postprocess_creates_nodes_and_relationships(transform, test_data, monkeypatch):"""Test that postprocess creates the expected nodes and relationships."""# Mock the __get_infos_from_whoxy method@@ -320,7 +320,7 @@ def test_postprocess_creates_nodes_and_relationships(scanner, test_data, monkeypreturn {}monkeypatch.setattr(- scanner, "_DomainToHistoryScanner__get_infos_from_whoxy", mock_get_infos+ transform, "_DomainToHistoryTransform__get_infos_from_whoxy", mock_get_infos)# First run scan to populate _extracted_data and _extracted_individuals@@ -331,13 +331,13 @@ def test_postprocess_creates_nodes_and_relationships(scanner, test_data, monkeyptry:input_domains = [Domain(domain="epios.com")]- results = loop.run_until_complete(scanner.scan(input_domains))+ results = loop.run_until_complete(transform.scan(input_domains))# Debug: Check what individuals were extractedprint(- f"\n=== DEBUG: _extracted_individuals has {len(scanner._extracted_individuals)} individuals ==="+ f"\n=== DEBUG: _extracted_individuals has {len(transform._extracted_individuals)} individuals ===")- for i, individual_info in enumerate(scanner._extracted_individuals):+ for i, individual_info in enumerate(transform._extracted_individuals):individual = individual_info["individual"]print(f"Individual {i+1}: {individual.full_name} ({individual_info['contact_type']}) for {individual_info['domain_name']}"@@ -349,28 +349,28 @@ def test_postprocess_creates_nodes_and_relationships(scanner, test_data, monkeyp# Now run postprocessprint(f"\n=== Running postprocess ===")- scanner.postprocess(results, input_domains)+ transform.postprocess(results, input_domains)# Debug: Check what happened during postprocessprint(f"=== Postprocess completed ===")- print(f"Nodes created: {len(scanner.neo4j_conn.nodes_created)}")- print(f"Relationships created: {len(scanner.neo4j_conn.relationships_created)}")+ print(f"Nodes created: {len(transform.neo4j_conn.nodes_created)}")+ print(f"Relationships created: {len(transform.neo4j_conn.relationships_created)}")# Should have created some nodes- assert len(scanner.neo4j_conn.nodes_created) > 0+ assert len(transform.neo4j_conn.nodes_created) > 0# Should have created some relationships- assert len(scanner.neo4j_conn.relationships_created) > 0+ assert len(transform.neo4j_conn.relationships_created) > 0# Check for domain nodedomain_nodes = [- n for n in scanner.neo4j_conn.nodes_created if n["label"] == "domain"+ n for n in transform.neo4j_conn.nodes_created if n["label"] == "domain"]assert len(domain_nodes) > 0# Check for individual nodes (should include JOHN DOE)individual_nodes = [- n for n in scanner.neo4j_conn.nodes_created if n["label"] == "individual"+ n for n in transform.neo4j_conn.nodes_created if n["label"] == "individual"]assert len(individual_nodes) > 0@@ -384,10 +384,10 @@ def test_postprocess_creates_nodes_and_relationships(scanner, test_data, monkeyploop.close()-def test_schemas(scanner):- """Test that the scanner has the expected schemas."""- input_schema = scanner.input_schema()- output_schema = scanner.output_schema()+def test_schemas(transform):+ """Test that the transform has the expected schemas."""+ input_schema = transform.input_schema()+ output_schema = transform.output_schema()assert input_schema is not Noneassert output_schema is not Nonediff --git a/flowsint-transforms/tests/scanners/domain/whois.py b/flowsint-transforms/tests/transforms/domain/whois.pysimilarity index 86%rename from flowsint-transforms/tests/scanners/domain/whois.pyrename to flowsint-transforms/tests/transforms/domain/whois.pyindex c03eac81..333d90f5 100644--- a/flowsint-transforms/tests/scanners/domain/whois.py+++ b/flowsint-transforms/tests/transforms/domain/whois.py@@ -1,7 +1,7 @@-from flowsint_transforms.domains.whois import WhoisScanner+from flowsint_transforms.domains.whois import WhoisTransformfrom flowsint_types.domain import Domain-scanner = WhoisScanner("sketch_123", "scan_123")+transform = WhoisTransform("sketch_123", "scan_123")def test_preprocess_valid_domains():@@ -9,7 +9,7 @@ def test_preprocess_valid_domains():Domain(domain="example.com"),Domain(domain="example2.com"),]- result = scanner.preprocess(domains)+ result = transform.preprocess(domains)result_domains = [d.domain for d in result]expected_domains = [d.domain for d in domains]@@ -22,7 +22,7 @@ def test_unprocessed_valid_domains():"example.com","example2.com",]- result = scanner.preprocess(domains)+ result = transform.preprocess(domains)result_domains = [d for d in result]expected_domains = [Domain(domain=d) for d in domains]assert result_domains == expected_domains@@ -34,7 +34,7 @@ def test_preprocess_invalid_domains():Domain(domain="invalid_domain"),Domain(domain="example.org"),]- result = scanner.preprocess(domains)+ result = transform.preprocess(domains)result_domains = [d.domain for d in result]assert "example.com" in result_domains@@ -49,7 +49,7 @@ def test_preprocess_multiple_formats():Domain(domain="example.org"),"example.org",]- result = scanner.preprocess(domains)+ result = transform.preprocess(domains)result_domains = [d.domain for d in result]assert "example.com" in result_domains@@ -73,7 +73,7 @@ def test_scan_returns_whois_objects(monkeypatch):monkeypatch.setattr("whois.whois", mock_whois)input_data = [Domain(domain="example.com")]- output = scanner.execute(input_data)+ output = transform.execute(input_data)assert isinstance(output, list)assert isinstance(output[0], Domain)assert output[0].whois.org == "MockOrg"@@ -81,8 +81,8 @@ def test_scan_returns_whois_objects(monkeypatch):def test_schemas():- input_schema = scanner.input_schema()- output_schema = scanner.output_schema()+ input_schema = transform.input_schema()+ output_schema = transform.output_schema()assert input_schema == {"type": "Domain","properties": [diff --git a/flowsint-transforms/tests/scanners/email/__init__.py b/flowsint-transforms/tests/transforms/email/__init__.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/email/__init__.pyrename to flowsint-transforms/tests/transforms/email/__init__.pydiff --git a/flowsint-transforms/tests/scanners/email/to_gravatar.py b/flowsint-transforms/tests/transforms/email/to_gravatar.pysimilarity index 86%rename from flowsint-transforms/tests/scanners/email/to_gravatar.pyrename to flowsint-transforms/tests/transforms/email/to_gravatar.pyindex 871b5b4f..0e230039 100644--- a/flowsint-transforms/tests/scanners/email/to_gravatar.py+++ b/flowsint-transforms/tests/transforms/email/to_gravatar.py@@ -1,30 +1,30 @@import hashlibfrom unittest.mock import Mock, patch-from flowsint_transforms.emails.to_gravatar import EmailToGravatarScanner+from flowsint_transforms.emails.to_gravatar import EmailToGravatarTransformfrom flowsint_types.email import Emailfrom flowsint_types.gravatar import Gravatar-scanner = EmailToGravatarScanner("sketch_123", "scan_123")+transform = EmailToGravatarTransform("sketch_123", "scan_123")-class TestEmailToGravatarScanner:- """Test suite for EmailToGravatarScanner"""+class TestEmailToGravatarTransform:+ """Test suite for EmailToGravatarTransform"""def test_name(self):- """Test the scanner name"""- assert EmailToGravatarScanner.name() == "to_gravatar"+ """Test the transform name"""+ assert EmailToGravatarTransform.name() == "to_gravatar"def test_category(self):- """Test the scanner category"""- assert EmailToGravatarScanner.category() == "Email"+ """Test the transform category"""+ assert EmailToGravatarTransform.category() == "Email"def test_key(self):- """Test the scanner key"""- assert EmailToGravatarScanner.key() == "email"+ """Test the transform key"""+ assert EmailToGravatarTransform.key() == "email"def test_input_schema(self):"""Test the input schema generation"""- schema = EmailToGravatarScanner.input_schema()+ schema = EmailToGravatarTransform.input_schema()assert schema["type"] == "Email"assert "properties" in schema# Check that email property is present@@ -36,7 +36,7 @@ class TestEmailToGravatarScanner:def test_output_schema(self):"""Test the output schema generation"""- schema = EmailToGravatarScanner.output_schema()+ schema = EmailToGravatarTransform.output_schema()assert schema["type"] == "Gravatar"assert "properties" in schema# Check that required properties are present@@ -55,7 +55,7 @@ class TestEmailToGravatarScanner:]- result = scanner.preprocess(emails)+ result = transform.preprocess(emails)assert len(result) == 2assert all(isinstance(email, Email) for email in result)assert result[0].email == "[email protected]"@@ -67,7 +67,7 @@ class TestEmailToGravatarScanner:{"email": "[email protected]"},{"email": "[email protected]"},]- result = scanner.preprocess(emails)+ result = transform.preprocess(emails)assert len(result) == 2assert all(isinstance(email, Email) for email in result)assert result[0].email == "[email protected]"@@ -79,7 +79,7 @@ class TestEmailToGravatarScanner:Email(email="[email protected]"),Email(email="[email protected]"),]- result = scanner.preprocess(emails)+ result = transform.preprocess(emails)assert len(result) == 2assert all(isinstance(email, Email) for email in result)assert result[0].email == "[email protected]"@@ -92,7 +92,7 @@ class TestEmailToGravatarScanner:{"email": "[email protected]"},Email(email="[email protected]"),]- result = scanner.preprocess(emails)+ result = transform.preprocess(emails)assert len(result) == 3assert all(isinstance(email, Email) for email in result)assert result[0].email == "[email protected]"@@ -108,7 +108,7 @@ class TestEmailToGravatarScanner:None,123,]- result = scanner.preprocess(emails)+ result = transform.preprocess(emails)# The preprocess method doesn't validate email format, it just creates Email objects# for valid string inputs and dicts with email keyassert len(result) == 2 # "not-an-email" and "invalid-email" are processed@@ -117,7 +117,7 @@ class TestEmailToGravatarScanner:def test_preprocess_empty_list(self):"""Test preprocessing with empty list"""- result = scanner.preprocess([])+ result = transform.preprocess([])assert result == []@patch("requests.get")@@ -129,7 +129,7 @@ class TestEmailToGravatarScanner:mock_get.return_value = mock_responseemails = [Email(email="[email protected]")]- result = scanner.scan(emails)+ result = transform.scan(emails)assert len(result) == 1assert isinstance(result[0], Gravatar)@@ -145,7 +145,7 @@ class TestEmailToGravatarScanner:mock_get.return_value = mock_responseemails = [Email(email="[email protected]")]- result = scanner.scan(emails)+ result = transform.scan(emails)assert len(result) == 0@@ -156,7 +156,7 @@ class TestEmailToGravatarScanner:mock_get.side_effect = Exception("Network error")emails = [Email(email="[email protected]")]- result = scanner.scan(emails)+ result = transform.scan(emails)assert len(result) == 0@@ -173,7 +173,7 @@ class TestEmailToGravatarScanner:Email(email="[email protected]"),Email(email="[email protected]"),]- result = scanner.scan(emails)+ result = transform.scan(emails)assert len(result) == 3assert all(isinstance(gravatar, Gravatar) for gravatar in result)@@ -200,7 +200,7 @@ class TestEmailToGravatarScanner:Email(email="[email protected]"),Email(email="[email protected]"),]- result = scanner.scan(emails)+ result = transform.scan(emails)# Should get 1 result for the first email (success) and 0 for the second (failure)assert len(result) == 1@@ -210,7 +210,7 @@ class TestEmailToGravatarScanner:"""Test postprocessing with Neo4j connection"""# Mock Neo4j connectionmock_neo4j = Mock()- scanner_with_neo4j = EmailToGravatarScanner(+ transform_with_neo4j = EmailToGravatarTransform("sketch_123", "scan_123", neo4j_conn=mock_neo4j)@@ -223,7 +223,7 @@ class TestEmailToGravatarScanner:Email(email="[email protected]"),]- result = scanner_with_neo4j.postprocess(gravatars, original_input)+ result = transform_with_neo4j.postprocess(gravatars, original_input)# Verify Neo4j queries were executedassert mock_neo4j.query.call_count == 2@@ -238,7 +238,7 @@ class TestEmailToGravatarScanner:]original_input = [Email(email="[email protected]")]- result = scanner.postprocess(gravatars, original_input)+ result = transform.postprocess(gravatars, original_input)# Should return results unchangedassert result == gravatars@@ -250,7 +250,7 @@ class TestEmailToGravatarScanner:]original_input = [] # Empty list- result = scanner.postprocess(gravatars, original_input)+ result = transform.postprocess(gravatars, original_input)# Should handle gracefully and return resultsassert result == gravatars@@ -263,7 +263,7 @@ class TestEmailToGravatarScanner:# The postprocess method doesn't handle None input properly# Let's test with an empty list instead- result = scanner.postprocess(gravatars, [])+ result = transform.postprocess(gravatars, [])# Should handle gracefully and return resultsassert result == gravatars@@ -277,7 +277,7 @@ class TestEmailToGravatarScanner:mock_get.return_value = mock_responseemails = ["[email protected]"]- result = scanner.execute(emails)+ result = transform.execute(emails)assert len(result) == 1assert isinstance(result[0], Gravatar)@@ -295,9 +295,9 @@ class TestEmailToGravatarScanner:mock_response.status_code = 200mock_get.return_value = mock_response- result = scanner.execute(emails)+ result = transform.execute(emails)- # The scanner processes any string as an email, so it will create Email objects+ # The transform processes any string as an email, so it will create Email objects# and attempt to get gravatars for themassert len(result) == 2assert all(isinstance(gravatar, Gravatar) for gravatar in result)@@ -313,7 +313,7 @@ class TestEmailToGravatarScanner:mock_get.return_value = mock_responseemails = [Email(email=email)]- result = scanner.scan(emails)+ result = transform.scan(emails)assert len(result) == 1assert result[0].hash == expected_hash@@ -330,7 +330,7 @@ class TestEmailToGravatarScanner:mock_get.return_value = mock_responseemails = [Email(email=email)]- result = scanner.scan(emails)+ result = transform.scan(emails)assert len(result) == 1assert str(result[0].src) == expected_urldiff --git a/flowsint-transforms/tests/scanners/email/to_leaks.py b/flowsint-transforms/tests/transforms/email/to_leaks.pysimilarity index 82%rename from flowsint-transforms/tests/scanners/email/to_leaks.pyrename to flowsint-transforms/tests/transforms/email/to_leaks.pyindex 77d68e37..b05113d6 100644--- a/flowsint-transforms/tests/scanners/email/to_leaks.py+++ b/flowsint-transforms/tests/transforms/email/to_leaks.py@@ -1,22 +1,22 @@import pytestfrom unittest.mock import patch, MagicMock-from flowsint_transforms.emails.to_leaks import EmailToBreachesScanner+from flowsint_transforms.emails.to_leaks import EmailToBreachesTransformfrom flowsint_types.email import Emailfrom flowsint_types.breach import Breach-scanner = EmailToBreachesScanner("sketch_123", "scan_123")+transform = EmailToBreachesTransform("sketch_123", "scan_123")def test_transform_name():- assert EmailToBreachesScanner.name() == "to_leaks"+ assert EmailToBreachesTransform.name() == "to_leaks"-def test_scanner_category():- assert EmailToBreachesScanner.category() == "Email"+def test_transform_category():+ assert EmailToBreachesTransform.category() == "Email"-def test_scanner_key():- assert EmailToBreachesScanner.key() == "email"+def test_transform_key():+ assert EmailToBreachesTransform.key() == "email"def test_preprocess_string_emails():@@ -24,7 +24,7 @@ def test_preprocess_string_emails():]- result = scanner.preprocess(emails)+ result = transform.preprocess(emails)expected_emails = [Email(email=email) for email in emails]assert result == expected_emails@@ -34,7 +34,7 @@ def test_preprocess_dict_emails():{"email": "[email protected]"},{"email": "[email protected]"},]- result = scanner.preprocess(emails)+ result = transform.preprocess(emails)expected_emails = [Email(email=email["email"]) for email in emails]assert result == expected_emails@@ -44,7 +44,7 @@ def test_preprocess_email_objects():Email(email="[email protected]"),Email(email="[email protected]"),]- result = scanner.preprocess(emails)+ result = transform.preprocess(emails)assert result == emails@@ -55,7 +55,7 @@ def test_preprocess_mixed_formats():Email(email="[email protected]"),{"invalid_key": "[email protected]"},]- result = scanner.preprocess(emails)+ result = transform.preprocess(emails)result_emails = [email.email for email in result]assert "[email protected]" in result_emails@@ -76,7 +76,7 @@ def test_scan_successful_response(mock_get):mock_get.return_value = mock_responseemails = [Email(email="[email protected]")]- result = scanner.scan(emails)+ result = transform.scan(emails)assert len(result) == 2assert isinstance(result[0], Breach)@@ -95,7 +95,7 @@ def test_scan_no_breaches_found(mock_get):mock_get.return_value = mock_responseemails = [Email(email="[email protected]")]- result = scanner.scan(emails)+ result = transform.scan(emails)assert len(result) == 0@@ -106,7 +106,7 @@ def test_scan_api_error(mock_get):mock_get.side_effect = Exception("API Error")emails = [Email(email="[email protected]")]- result = scanner.scan(emails)+ result = transform.scan(emails)assert len(result) == 0@@ -123,7 +123,7 @@ def test_scan_missing_name_field(mock_get):mock_get.return_value = mock_responseemails = [Email(email="[email protected]")]- result = scanner.scan(emails)+ result = transform.scan(emails)assert len(result) == 2assert result[0].name == "unknown" # Should default to "unknown"@@ -134,16 +134,16 @@ def test_scan_missing_name_field(mock_get):@patch("src.transforms.emails.to_leaks.HIBP_API_KEY", None)def test_scan_no_api_key():- """Test that scanner raises ValueError when HIBP_API_KEY is not set."""+ """Test that transform raises ValueError when HIBP_API_KEY is not set."""emails = [Email(email="[email protected]")]with pytest.raises(ValueError, match="HIBP_API_KEY not set"):- scanner.scan(emails)+ transform.scan(emails)def test_postprocess():# Test postprocess method with mocked neo4j connection- scanner.neo4j_conn = MagicMock()+ transform.neo4j_conn = MagicMock()# Create breach objects with the new structurebreach1 = Breach(@@ -164,7 +164,7 @@ def test_postprocess():breaches = [breach1, breach2]original_input = [Email(email="[email protected]")]- result = scanner.postprocess(breaches, original_input)+ result = transform.postprocess(breaches, original_input)assert result == breaches# Verify that neo4j queries were called:@@ -172,4 +172,4 @@ def test_postprocess():# - 1 email node creation query# - 2 relationship creation queries# Total: 5 queries- assert scanner.neo4j_conn.query.call_count == 5+ assert transform.neo4j_conn.query.call_count == 5diff --git a/flowsint-transforms/tests/scanners/ip/__init__.py b/flowsint-transforms/tests/transforms/ip/__init__.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/ip/__init__.pyrename to flowsint-transforms/tests/transforms/ip/__init__.pydiff --git a/flowsint-transforms/tests/scanners/ip/asn_to_cidrs.py b/flowsint-transforms/tests/transforms/ip/asn_to_cidrs.pysimilarity index 82%rename from flowsint-transforms/tests/scanners/ip/asn_to_cidrs.pyrename to flowsint-transforms/tests/transforms/ip/asn_to_cidrs.pyindex fc40c860..4cc7a05e 100644--- a/flowsint-transforms/tests/scanners/ip/asn_to_cidrs.py+++ b/flowsint-transforms/tests/transforms/ip/asn_to_cidrs.py@@ -1,7 +1,7 @@-from flowsint_transforms.ips.asn_to_cidrs import AsnToCidrsScanner+from flowsint_transforms.ips.asn_to_cidrs import AsnToCidrsTransformfrom flowsint_types.asn import ASN-scanner = AsnToCidrsScanner("sketch_123", "scan_123")+transform = AsnToCidrsTransform("sketch_123", "scan_123")def test_preprocess_valid_asns():@@ -9,7 +9,7 @@ def test_preprocess_valid_asns():ASN(number=15169),ASN(number=13335),]- result = scanner.preprocess(asns)+ result = transform.preprocess(asns)result_numbers = [asn.number for asn in result]expected_numbers = [asn.number for asn in asns]@@ -22,7 +22,7 @@ def test_unprocessed_valid_asns():"15169","13335",]- result = scanner.preprocess(asns)+ result = transform.preprocess(asns)result_asns = [asn for asn in result]expected_asns = [ASN(number=int(asn)) for asn in asns]assert result_asns == expected_asns@@ -34,7 +34,7 @@ def test_preprocess_invalid_asns():ASN(number=999999999999), # Invalid ASN numberASN(number=13335),]- result = scanner.preprocess(asns)+ result = transform.preprocess(asns)result_numbers = [asn.number for asn in result]assert 15169 in result_numbers@@ -49,7 +49,7 @@ def test_preprocess_multiple_formats():ASN(number=13335),"15169",]- result = scanner.preprocess(asns)+ result = transform.preprocess(asns)result_numbers = [asn.number for asn in result]assert 15169 in result_numbers@@ -60,8 +60,8 @@ def test_preprocess_multiple_formats():def test_schemas():- input_schema = scanner.input_schema()- output_schema = scanner.output_schema()+ input_schema = transform.input_schema()+ output_schema = transform.output_schema()# Input schema should have number fieldassert "properties" in input_schemadiff --git a/flowsint-transforms/tests/scanners/ip/cidr_to_ips.py b/flowsint-transforms/tests/transforms/ip/cidr_to_ips.pysimilarity index 86%rename from flowsint-transforms/tests/scanners/ip/cidr_to_ips.pyrename to flowsint-transforms/tests/transforms/ip/cidr_to_ips.pyindex 0803c9da..1aac0493 100644--- a/flowsint-transforms/tests/scanners/ip/cidr_to_ips.py+++ b/flowsint-transforms/tests/transforms/ip/cidr_to_ips.py@@ -1,10 +1,10 @@-from flowsint_transforms.ips.cidr_to_ips import CidrToIpsScanner+from flowsint_transforms.ips.cidr_to_ips import CidrToIpsTransformfrom flowsint_types.cidr import CIDRfrom flowsint_types.ip import Ipfrom tests.logger import TestLoggerlogger = TestLogger()-scanner = CidrToIpsScanner("sketch_123", "scan_123", logger)+transform = CidrToIpsTransform("sketch_123", "scan_123", logger)def test_preprocess_valid_cidrs():@@ -12,7 +12,7 @@ def test_preprocess_valid_cidrs():CIDR(network="8.8.8.0/24"),CIDR(network="1.1.1.0/24"),]- result = scanner.preprocess(cidrs)+ result = transform.preprocess(cidrs)result_networks = [cidr.network for cidr in result]expected_networks = [cidr.network for cidr in cidrs]@@ -25,7 +25,7 @@ def test_preprocess_unprocessed_valid_cidrs():"8.8.8.0/24","1.1.1.0/24",]- result = scanner.preprocess(cidrs)+ result = transform.preprocess(cidrs)result_cidrs = [c for c in result]expected_cidrs = [CIDR(network=c) for c in cidrs]assert result_cidrs == expected_cidrs@@ -37,7 +37,7 @@ def test_preprocess_invalid_cidrs():"invalid-cidr","not-a-cidr",]- result = scanner.preprocess(cidrs)+ result = transform.preprocess(cidrs)result_networks = [str(cidr.network) for cidr in result]assert "8.8.8.0/24" in result_networksassert "invalid-cidr" not in result_networks@@ -51,7 +51,7 @@ def test_preprocess_multiple_formats():CIDR(network="9.9.9.0/24"),"InvalidCIDR",]- result = scanner.preprocess(cidrs)+ result = transform.preprocess(cidrs)result_networks = [str(cidr.network) for cidr in result]assert "8.8.8.0/24" in result_networksassert "9.9.9.0/24" in result_networks@@ -75,11 +75,11 @@ def test_scan_extracts_ips(monkeypatch):assert "-ptr" in cmdreturn MockSubprocessResult(mock_dnsx_output)- # Patch the subprocess call in the scanner+ # Patch the subprocess call in the transformmonkeypatch.setattr("subprocess.run", mock_subprocess_run)input_data = [CIDR(network="8.35.200.0/24")]- ips = scanner.scan(input_data)+ ips = transform.scan(input_data)assert isinstance(ips, list)assert len(ips) == 4@@ -103,7 +103,7 @@ def test_scan_handles_empty_output(monkeypatch):monkeypatch.setattr("subprocess.run", mock_subprocess_run)input_data = [CIDR(network="8.8.8.0/24")]- ips = scanner.scan(input_data)+ ips = transform.scan(input_data)assert isinstance(ips, list)assert len(ips) == 0@@ -116,7 +116,7 @@ def test_scan_handles_subprocess_exception(monkeypatch):monkeypatch.setattr("subprocess.run", mock_subprocess_run)input_data = [CIDR(network="8.8.8.0/24")]- ips = scanner.scan(input_data)+ ips = transform.scan(input_data)assert isinstance(ips, list)assert len(ips) == 0diff --git a/flowsint-transforms/tests/scanners/ip/ip_to_asn.py b/flowsint-transforms/tests/transforms/ip/ip_to_asn.pysimilarity index 89%rename from flowsint-transforms/tests/scanners/ip/ip_to_asn.pyrename to flowsint-transforms/tests/transforms/ip/ip_to_asn.pyindex 091017e7..4768dd23 100644--- a/flowsint-transforms/tests/scanners/ip/ip_to_asn.py+++ b/flowsint-transforms/tests/transforms/ip/ip_to_asn.py@@ -1,14 +1,14 @@import jsonfrom unittest.mock import Mock-from flowsint_transforms.ips.ip_to_asn import IpToAsnScanner+from flowsint_transforms.ips.ip_to_asn import IpToAsnTransformfrom flowsint_types.ip import Ipfrom flowsint_types.asn import ASNfrom flowsint_types.cidr import CIDRfrom tests.logger import TestLoggerlogger = TestLogger()-# The scanner will get a mock logger from conftest.py automatically-scanner = IpToAsnScanner("sketch_123", "scan_123", logger)+# The transform will get a mock logger from conftest.py automatically+transform = IpToAsnTransform("sketch_123", "scan_123", logger)def test_preprocess_valid_ips():@@ -16,7 +16,7 @@ def test_preprocess_valid_ips():Ip(address="8.8.8.8"),Ip(address="1.1.1.1"),]- result = scanner.preprocess(ips)+ result = transform.preprocess(ips)result_addresses = [ip.address for ip in result]expected_addresses = [ip.address for ip in ips]@@ -29,7 +29,7 @@ def test_unprocessed_valid_ips():"8.8.8.8","1.1.1.1",]- result = scanner.preprocess(ips)+ result = transform.preprocess(ips)result_ips = [ip for ip in result]expected_ips = [Ip(address=ip) for ip in ips]assert result_ips == expected_ips@@ -41,7 +41,7 @@ def test_preprocess_invalid_ips():Ip(address="invalid_ip"),Ip(address="192.168.1.1"),]- result = scanner.preprocess(ips)+ result = transform.preprocess(ips)result_addresses = [ip.address for ip in result]assert "8.8.8.8" in result_addresses@@ -56,7 +56,7 @@ def test_preprocess_multiple_formats():Ip(address="192.168.1.1"),"10.0.0.1",]- result = scanner.preprocess(ips)+ result = transform.preprocess(ips)result_addresses = [ip.address for ip in result]assert "8.8.8.8" in result_addresses@@ -86,11 +86,11 @@ def test_scan_extracts_asn_info(monkeypatch):assert input == "8.8.8.8"return MockSubprocessResult(json.dumps(mock_asnmap_output))- # Patch the subprocess call in the scanner+ # Patch the subprocess call in the transformmonkeypatch.setattr("subprocess.run", mock_subprocess_run)input_data = [Ip(address="8.8.8.8")]- asns = scanner.scan(input_data)+ asns = transform.scan(input_data)assert isinstance(asns, list)assert len(asns) == 1@@ -118,7 +118,7 @@ def test_scan_handles_no_asn_found(monkeypatch):monkeypatch.setattr("subprocess.run", mock_subprocess_run)input_data = [Ip(address="192.168.1.1")]- asns = scanner.scan(input_data)+ asns = transform.scan(input_data)assert isinstance(asns, list)assert len(asns) == 1@@ -138,7 +138,7 @@ def test_scan_handles_subprocess_exception(monkeypatch):monkeypatch.setattr("subprocess.run", mock_subprocess_run)input_data = [Ip(address="8.8.8.8")]- asns = scanner.scan(input_data)+ asns = transform.scan(input_data)assert isinstance(asns, list)assert len(asns) == 1@@ -181,7 +181,7 @@ def test_scan_multiple_ips(monkeypatch):monkeypatch.setattr("subprocess.run", mock_subprocess_run)input_data = [Ip(address="8.8.8.8"), Ip(address="1.1.1.1")]- asns = scanner.scan(input_data)+ asns = transform.scan(input_data)assert len(asns) == 2@@ -195,8 +195,8 @@ def test_scan_multiple_ips(monkeypatch):def test_schemas():- input_schema = scanner.input_schema()- output_schema = scanner.output_schema()+ input_schema = transform.input_schema()+ output_schema = transform.output_schema()# Input schema should have address fieldassert "properties" in input_schema@@ -218,7 +218,7 @@ def test_schemas():def test_postprocess_creates_neo4j_relationships(monkeypatch):# Mock Neo4j connectionmock_neo4j = Mock()- scanner.neo4j_conn = mock_neo4j+ transform.neo4j_conn = mock_neo4jinput_data = [Ip(address="8.8.8.8")]asn_results = [@@ -230,7 +230,7 @@ def test_postprocess_creates_neo4j_relationships(monkeypatch):)]- result = scanner.postprocess(asn_results, input_data)+ result = transform.postprocess(asn_results, input_data)# Verify Neo4j query was calledmock_neo4j.query.assert_called_once()@@ -251,14 +251,14 @@ def test_postprocess_creates_neo4j_relationships(monkeypatch):def test_postprocess_skips_unknown_asns(monkeypatch):# Mock Neo4j connectionmock_neo4j = Mock()- scanner.neo4j_conn = mock_neo4j+ transform.neo4j_conn = mock_neo4jinput_data = [Ip(address="192.168.1.1")]asn_results = [ASN(number=0, name="Unknown", country="Unknown", cidrs=[]) # Unknown ASN]- result = scanner.postprocess(asn_results, input_data)+ result = transform.postprocess(asn_results, input_data)# Verify Neo4j query was NOT called for unknown ASNmock_neo4j.query.assert_not_called()diff --git a/flowsint-transforms/tests/scanners/ip/ip_to_infos.py b/flowsint-transforms/tests/transforms/ip/ip_to_infos.pysimilarity index 84%rename from flowsint-transforms/tests/scanners/ip/ip_to_infos.pyrename to flowsint-transforms/tests/transforms/ip/ip_to_infos.pyindex cc44e2bd..8e137377 100644--- a/flowsint-transforms/tests/scanners/ip/ip_to_infos.py+++ b/flowsint-transforms/tests/transforms/ip/ip_to_infos.py@@ -1,7 +1,7 @@-from flowsint_transforms.ips.geolocation import IpToInfosScanner+from flowsint_transforms.ips.geolocation import IpToInfosTransformfrom flowsint_types.ip import Ip, Ip-scanner = IpToInfosScanner("sketch_123", "scan_123")+transform = IpToInfosTransform("sketch_123", "scan_123")def test_preprocess_valid_ips():@@ -9,7 +9,7 @@ def test_preprocess_valid_ips():Ip(address="8.8.8.8"),Ip(address="1.1.1.1"),]- result = scanner.preprocess(ips)+ result = transform.preprocess(ips)result_ips = [d.address for d in result]expected_ips = [d.address for d in ips]assert result_ips == expected_ips@@ -20,7 +20,7 @@ def test_preprocess_string_ips():"8.8.8.8","1.1.1.1",]- result = scanner.preprocess(ips)+ result = transform.preprocess(ips)result_ips = [d.address for d in result]expected_ips = [d for d in ips]assert [ip.address for ip in result] == expected_ips@@ -32,7 +32,7 @@ def test_preprocess_invalid_ips():Ip(address="invalid_ip"),Ip(address="1.1.1.1"),]- result = scanner.preprocess(ips)+ result = transform.preprocess(ips)result_ips = [d.address for d in result]assert "8.8.8.8" in result_ipsassert "1.1.1.1" in result_ips@@ -46,7 +46,7 @@ def test_preprocess_multiple_formats():Ip(address="1.1.1.1"),"1.1.1.1",]- result = scanner.preprocess(ips)+ result = transform.preprocess(ips)result_ips = [d.address for d in result]assert "8.8.8.8" in result_ipsassert "1.1.1.1" in result_ips@@ -64,10 +64,10 @@ def test_scan_returns_ip(monkeypatch):"isp": "Google LLC",}- monkeypatch.setattr(scanner, "get_location_data", mock_get_location_data)+ monkeypatch.setattr(transform, "get_location_data", mock_get_location_data)input_data = [Ip(address="8.8.8.8")]- output = scanner.execute(input_data)+ output = transform.execute(input_data)assert isinstance(output, list)assert isinstance(output[0], Ip)assert output[0].address == "8.8.8.8"@@ -77,8 +77,8 @@ def test_scan_returns_ip(monkeypatch):def test_schemas():- input_schema = scanner.input_schema()- output_schema = scanner.output_schema()+ input_schema = transform.input_schema()+ output_schema = transform.output_schema()assert input_schema == {"type": "Ip","properties": [diff --git a/flowsint-transforms/tests/scanners/organization/__init__.py b/flowsint-transforms/tests/transforms/organization/__init__.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/organization/__init__.pyrename to flowsint-transforms/tests/transforms/organization/__init__.pydiff --git a/flowsint-transforms/tests/scanners/organization/to_infos.py b/flowsint-transforms/tests/transforms/organization/to_infos.pysimilarity index 74%rename from flowsint-transforms/tests/scanners/organization/to_infos.pyrename to flowsint-transforms/tests/transforms/organization/to_infos.pyindex 26eb180f..9c3cca16 100644--- a/flowsint-transforms/tests/scanners/organization/to_infos.py+++ b/flowsint-transforms/tests/transforms/organization/to_infos.py@@ -1,12 +1,12 @@-from flowsint_transforms.organizations.to_infos import OrgToInfosScanner+from flowsint_transforms.organizations.to_infos import OrgToInfosTransformfrom flowsint_types.organization import Organization-scanner = OrgToInfosScanner("sketch_123", "scan_123")+transform = OrgToInfosTransform("sketch_123", "scan_123")def test_preprocess_valid_names():data = [Organization(name="OpenAI"), {"name": "Inria"}, "OVH"]- result = scanner.preprocess(data)+ result = transform.preprocess(data)result_names = [org.name for org in result]assert result_names == ["OpenAI", "Inria", "OVH"]@@ -20,10 +20,10 @@ def test_preprocess_valid_names():# "",# {"name": ""},# ]-# result = scanner.preprocess(data)+# result = transform.preprocess(data)# assert result == []def test_execute():- scanner.execute(["Karim Terrache"])+ transform.execute(["Karim Terrache"])assert Truediff --git a/flowsint-transforms/tests/scanners/social/__init__.py b/flowsint-transforms/tests/transforms/social/__init__.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/social/__init__.pyrename to flowsint-transforms/tests/transforms/social/__init__.pydiff --git a/flowsint-transforms/tests/scanners/social/maigret.py b/flowsint-transforms/tests/transforms/social/maigret.pysimilarity index 76%rename from flowsint-transforms/tests/scanners/social/maigret.pyrename to flowsint-transforms/tests/transforms/social/maigret.pyindex e0c368a7..3a47aef9 100644--- a/flowsint-transforms/tests/scanners/social/maigret.py+++ b/flowsint-transforms/tests/transforms/social/maigret.py@@ -1,8 +1,8 @@from pathlib import Path-from flowsint_transforms.socials.maigret import MaigretScanner+from flowsint_transforms.socials.maigret import MaigretTransformfrom flowsint_types.social import SocialProfile-scanner = MaigretScanner("sketch_123", "scan_123")+transform = MaigretTransform("sketch_123", "scan_123")def test_unprocessed_valid_usernames():@@ -10,7 +10,7 @@ def test_unprocessed_valid_usernames():"toto123","DorianXd78",]- result = scanner.preprocess(usernames)+ result = transform.preprocess(usernames)result_usernames = [d for d in result]expected_usernames = [SocialProfile(username=d) for d in usernames]assert result_usernames == expected_usernames@@ -22,7 +22,7 @@ def test_preprocess_invalid_usernames():SocialProfile(username="DorianXd78_Official"),SocialProfile(username="This is not a username"),]- result = scanner.preprocess(usernames)+ result = transform.preprocess(usernames)result_usernames = [d.username for d in result]assert "toto123" in result_usernames@@ -37,7 +37,7 @@ def test_preprocess_multiple_formats():SocialProfile(username="DorianXd78_Official"),"MySimpleUsername",]- result = scanner.preprocess(usernames)+ result = transform.preprocess(usernames)result_usernames = [d.username for d in result]assert "toto123" in result_usernames@@ -47,11 +47,11 @@ def test_preprocess_multiple_formats():def test_parsing_invalid_output_file():- results = scanner.parse_maigret_output("toto123", Path("/this/path/does/not/exist"))+ results = transform.parse_maigret_output("toto123", Path("/this/path/does/not/exist"))assert results == []def test_parsing():- results = scanner.parse_maigret_output("toto123", Path("/tmp/maigret_test.json"))+ results = transform.parse_maigret_output("toto123", Path("/tmp/maigret_test.json"))print(results)assert len(results) == 2diff --git a/flowsint-transforms/tests/scanners/website/__init__.py b/flowsint-transforms/tests/transforms/website/__init__.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/website/__init__.pyrename to flowsint-transforms/tests/transforms/website/__init__.pydiff --git a/flowsint-transforms/tests/scanners/website/to_links.py b/flowsint-transforms/tests/transforms/website/to_links.pysimilarity index 80%rename from flowsint-transforms/tests/scanners/website/to_links.pyrename to flowsint-transforms/tests/transforms/website/to_links.pyindex 02c668ab..a2fdb270 100644--- a/flowsint-transforms/tests/scanners/website/to_links.py+++ b/flowsint-transforms/tests/transforms/website/to_links.py@@ -39,34 +39,34 @@ class MockCrawler:@pytest.mark.asyncioasync def test_website_to_links_real_time_neo4j_creation():"""Test that Neo4j nodes are created in real-time during the callback."""- scanner = WebsiteToLinks(sketch_id="test", scan_id="test")+ transform = WebsiteToLinks(sketch_id="test", scan_id="test")# Mock neo4j connection and methods- scanner.neo4j_conn = Mock()- scanner.create_node = Mock()- scanner.create_relationship = Mock()- scanner.log_graph_message = Mock()+ transform.neo4j_conn = Mock()+ transform.create_node = Mock()+ transform.create_relationship = Mock()+ transform.log_graph_message = Mock()# Test inputwebsites = [Website(url="https://example.com")]with patch("src.transforms.websites.to_links.Crawler", MockCrawler):- results = await scanner.scan(websites)+ results = await transform.scan(websites)# Verify main website and domain nodes were created upfront- scanner.create_node.assert_any_call(+ transform.create_node.assert_any_call("website","url","https://example.com",caption="https://example.com",type="website",)- scanner.create_node.assert_any_call(+ transform.create_node.assert_any_call("domain", "name", "example.com", caption="example.com", type="domain")# Verify main website to domain relationship- scanner.create_relationship.assert_any_call(+ transform.create_relationship.assert_any_call("website","url","https://example.com",@@ -77,14 +77,14 @@ async def test_website_to_links_real_time_neo4j_creation():)# Verify internal website nodes were created in callback- scanner.create_node.assert_any_call(+ transform.create_node.assert_any_call("website","url","https://example.com/page1",caption="https://example.com/page1",type="website",)- scanner.create_node.assert_any_call(+ transform.create_node.assert_any_call("website","url","https://example.com/page2",@@ -93,7 +93,7 @@ async def test_website_to_links_real_time_neo4j_creation():)# Verify internal website relationships- scanner.create_relationship.assert_any_call(+ transform.create_relationship.assert_any_call("website","url","https://example.com",@@ -102,7 +102,7 @@ async def test_website_to_links_real_time_neo4j_creation():"https://example.com/page1","LINKS_TO",)- scanner.create_relationship.assert_any_call(+ transform.create_relationship.assert_any_call("website","url","https://example.com",@@ -113,14 +113,14 @@ async def test_website_to_links_real_time_neo4j_creation():)# Verify external website nodes were created in callback- scanner.create_node.assert_any_call(+ transform.create_node.assert_any_call("website","url","https://external.com/page",caption="https://external.com/page",type="website",)- scanner.create_node.assert_any_call(+ transform.create_node.assert_any_call("website","url","https://another-external.org/resource",@@ -129,10 +129,10 @@ async def test_website_to_links_real_time_neo4j_creation():)# Verify external domain nodes were created in callback- scanner.create_node.assert_any_call(+ transform.create_node.assert_any_call("domain", "name", "external.com", caption="external.com", type="domain")- scanner.create_node.assert_any_call(+ transform.create_node.assert_any_call("domain","name","another-external.org",@@ -141,7 +141,7 @@ async def test_website_to_links_real_time_neo4j_creation():)# Verify main website to external website relationships- scanner.create_relationship.assert_any_call(+ transform.create_relationship.assert_any_call("website","url","https://example.com",@@ -150,7 +150,7 @@ async def test_website_to_links_real_time_neo4j_creation():"https://external.com/page","LINKS_TO",)- scanner.create_relationship.assert_any_call(+ transform.create_relationship.assert_any_call("website","url","https://example.com",@@ -161,7 +161,7 @@ async def test_website_to_links_real_time_neo4j_creation():)# Verify external website to domain relationships- scanner.create_relationship.assert_any_call(+ transform.create_relationship.assert_any_call("website","url","https://external.com/page",@@ -170,7 +170,7 @@ async def test_website_to_links_real_time_neo4j_creation():"external.com","BELONGS_TO_DOMAIN",)- scanner.create_relationship.assert_any_call(+ transform.create_relationship.assert_any_call("website","url","https://another-external.org/resource",@@ -181,7 +181,7 @@ async def test_website_to_links_real_time_neo4j_creation():)# Verify main website to external domain relationships- scanner.create_relationship.assert_any_call(+ transform.create_relationship.assert_any_call("website","url","https://example.com",@@ -190,7 +190,7 @@ async def test_website_to_links_real_time_neo4j_creation():"external.com","LINKS_TO_DOMAIN",)- scanner.create_relationship.assert_any_call(+ transform.create_relationship.assert_any_call("website","url","https://example.com",@@ -204,13 +204,13 @@ async def test_website_to_links_real_time_neo4j_creation():@pytest.mark.asyncioasync def test_website_to_links_error_handling_with_neo4j():"""Test that main nodes are still created even when crawling fails."""- scanner = WebsiteToLinks(sketch_id="test", scan_id="test")+ transform = WebsiteToLinks(sketch_id="test", scan_id="test")# Mock neo4j connection and methods- scanner.neo4j_conn = Mock()- scanner.create_node = Mock()- scanner.create_relationship = Mock()- scanner.log_graph_message = Mock()+ transform.neo4j_conn = Mock()+ transform.create_node = Mock()+ transform.create_relationship = Mock()+ transform.log_graph_message = Mock()# Mock crawler that raises an exceptiondef mock_crawler_error(*args, **kwargs):@@ -219,22 +219,22 @@ async def test_website_to_links_error_handling_with_neo4j():websites = [Website(url="https://example.com")]with patch("src.transforms.websites.to_links.Crawler", mock_crawler_error):- results = await scanner.scan(websites)+ results = await transform.scan(websites)# Verify main website and domain nodes were still created despite error- scanner.create_node.assert_any_call(+ transform.create_node.assert_any_call("website","url","https://example.com",caption="https://example.com",type="website",)- scanner.create_node.assert_any_call(+ transform.create_node.assert_any_call("domain", "name", "example.com", caption="example.com", type="domain")# Verify main website to domain relationship was created- scanner.create_relationship.assert_any_call(+ transform.create_relationship.assert_any_call("website","url","https://example.com",@@ -256,7 +256,7 @@ async def test_website_to_links_error_handling_with_neo4j():def test_postprocess_simplified():"""Test that postprocess now just returns results as-is."""- scanner = WebsiteToLinks(sketch_id="test", scan_id="test")+ transform = WebsiteToLinks(sketch_id="test", scan_id="test")original_input = [Website(url="https://example.com")]results = [@@ -269,7 +269,7 @@ def test_postprocess_simplified():}]- processed_results = scanner.postprocess(results, original_input)+ processed_results = transform.postprocess(results, original_input)# Should just return the same results since Neo4j work is done in real-timeassert processed_results == resultsdiff --git a/flowsint-transforms/tests/utils.py b/flowsint-transforms/tests/utils.pydeleted file mode 100644index d9affb08..00000000--- a/flowsint-transforms/tests/utils.py+++ /dev/null@@ -1,40 +0,0 @@-from flowsint_core.core.scanner_base import build_params_model---def test_build_params_model_valid():- param_schema = [- {- "name": "ETHERSCAN_API_KEY",- "type": "string",- "description": "The Etherscan API key to use for the transaction lookup.",- "required": True,- },- {- "name": "url",- "type": "string",- "description": "Base URL for API",- "required": False,- "default": "https://api.etherscan.io/api",- },- ]- ParamsModel = build_params_model(param_schema)- validated_params = ParamsModel(ETHERSCAN_API_KEY="clef-123")- assert validated_params.ETHERSCAN_API_KEY == "clef-123"- assert validated_params.url == "https://api.etherscan.io/api"---def test_build_params_model_invalid():- param_schema = [- {},- {- "name": "url",- "type": "string",- "description": "Base URL for API",- "required": False,- "default": "https://api.etherscan.io/api",- },- ]- ParamsModel = build_params_model(param_schema)- validated_params = ParamsModel(ETHERSCAN_API_KEY="clef-123")- assert validated_params.ETHERSCAN_API_KEY == "clef-123"- assert validated_params.url == "https://api.etherscan.io/api"