Skip to content

A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.

refactor: remove occurences of scanner

dextmorgn980bb85af476
diff --git a/README.md b/README.md
index 73aec6c3..ad696571 100644
--- a/README.md
+++ b/README.md
@@ -64,7 +64,7 @@ Core utilities and base classes used by all other modules:
- Authentication and authorization
- Logging and event handling
- Configuration management
-- Base classes for scanners and tools
+- Base classes for transforms and tools
- Utility functions
### flowsint-types
@@ -81,11 +81,11 @@ Pydantic models for all data types:
Transform modules that process data:
-- Domain scanners (subdomains, WHOIS, resolution)
-- IP scanners (geolocation, ASN lookup)
-- Social media scanners (Maigret, Sherlock)
-- Email scanners (breaches, Gravatar)
-- Crypto scanners (transactions, NFTs)
+- Domain transforms (subdomains, WHOIS, resolution)
+- IP transforms (geolocation, ASN lookup)
+- Social media transforms (Maigret, Sherlock)
+- Email transforms (breaches, Gravatar)
+- Crypto transforms (transactions, NFTs)
- And many more...
### flowsint-api
diff --git a/flowsint-api/README.md b/flowsint-api/README.md
index 7e233ddc..2b553006 100644
--- a/flowsint-api/README.md
+++ b/flowsint-api/README.md
@@ -50,4 +50,4 @@ The crawler can be configured with:
## Usage
-The API provides REST endpoints for various scanning operations. See the individual scanner modules for specific usage examples.
+The API provides REST endpoints for various scanning operations. See the individual transform modules for specific usage examples.
diff --git a/flowsint-api/alembic/versions/2da47dbd4a52_add_cascade_delete_to_scans_and_logs.py b/flowsint-api/alembic/versions/2da47dbd4a52_add_cascade_delete_to_scans_and_logs.py
index ca0ee0b0..dc4590d7 100644
--- a/flowsint-api/alembic/versions/2da47dbd4a52_add_cascade_delete_to_scans_and_logs.py
+++ b/flowsint-api/alembic/versions/2da47dbd4a52_add_cascade_delete_to_scans_and_logs.py
@@ -22,7 +22,7 @@ def upgrade() -> None:
"""Upgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column('scans', 'status',
- existing_type=postgresql.ENUM('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', name='scannerstatus'),
+ existing_type=postgresql.ENUM('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', name='transformstatus'),
nullable=True)
op.drop_constraint('scans_sketch_id_fkey', 'scans', type_='foreignkey')
op.create_foreign_key(None, 'scans', 'sketches', ['sketch_id'], ['id'], onupdate='CASCADE', ondelete='CASCADE')
@@ -35,6 +35,6 @@ def downgrade() -> None:
op.drop_constraint(None, 'scans', type_='foreignkey')
op.create_foreign_key('scans_sketch_id_fkey', 'scans', 'sketches', ['sketch_id'], ['id'])
op.alter_column('scans', 'status',
- existing_type=postgresql.ENUM('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', name='scannerstatus'),
+ existing_type=postgresql.ENUM('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', name='transformstatus'),
nullable=False)
# ### end Alembic commands ###
diff --git a/flowsint-api/alembic/versions/6dfa83113ad7_change_content_colum_of_log_to_json.py b/flowsint-api/alembic/versions/6dfa83113ad7_change_content_colum_of_log_to_json.py
index 5c102f30..8a8766c3 100644
--- a/flowsint-api/alembic/versions/6dfa83113ad7_change_content_colum_of_log_to_json.py
+++ b/flowsint-api/alembic/versions/6dfa83113ad7_change_content_colum_of_log_to_json.py
@@ -68,8 +68,8 @@ def downgrade() -> None:
# 1. Revert 'scans.status' to old ENUM type
op.execute("""
ALTER TABLE scans
- ALTER COLUMN status TYPE scannerstatus
- USING status::text::scannerstatus
+ ALTER COLUMN status TYPE transformstatus
+ USING status::text::transformstatus
""")
# 2. Revert 'logs.type' back to VARCHAR
diff --git a/flowsint-api/alembic/versions/71a3e5b4db2a_update_scan_status_enum.py b/flowsint-api/alembic/versions/71a3e5b4db2a_update_scan_status_enum.py
index b57033ae..d8d7cce7 100644
--- a/flowsint-api/alembic/versions/71a3e5b4db2a_update_scan_status_enum.py
+++ b/flowsint-api/alembic/versions/71a3e5b4db2a_update_scan_status_enum.py
@@ -21,7 +21,7 @@ depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
# Create the enum type first using raw SQL to ensure it exists
- op.execute("CREATE TYPE scannerstatus AS ENUM ('PENDING', 'RUNNING', 'COMPLETED', 'FAILED')")
+ op.execute("CREATE TYPE transformstatus AS ENUM ('PENDING', 'RUNNING', 'COMPLETED', 'FAILED')")
# Add new columns
op.add_column('scans', sa.Column('started_at', sa.DateTime(), nullable=True))
@@ -30,17 +30,17 @@ def upgrade() -> None:
op.add_column('scans', sa.Column('details', sa.JSON(), nullable=True))
# Add new status column with enum type
- op.add_column('scans', sa.Column('status_new', postgresql.ENUM('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', name='scannerstatus'), nullable=True))
+ op.add_column('scans', sa.Column('status_new', postgresql.ENUM('PENDING', 'RUNNING', 'COMPLETED', 'FAILED', name='transformstatus'), nullable=True))
# Copy data from old status to new status with proper casting
op.execute("""
UPDATE scans
SET status_new = CASE
- WHEN status = 'PENDING' THEN 'PENDING'::scannerstatus
- WHEN status = 'RUNNING' THEN 'RUNNING'::scannerstatus
- WHEN status = 'COMPLETED' THEN 'COMPLETED'::scannerstatus
- WHEN status = 'FAILED' THEN 'FAILED'::scannerstatus
- ELSE 'PENDING'::scannerstatus
+ WHEN status = 'PENDING' THEN 'PENDING'::transformstatus
+ WHEN status = 'RUNNING' THEN 'RUNNING'::transformstatus
+ WHEN status = 'COMPLETED' THEN 'COMPLETED'::transformstatus
+ WHEN status = 'FAILED' THEN 'FAILED'::transformstatus
+ ELSE 'PENDING'::transformstatus
END
""")
@@ -100,4 +100,4 @@ def downgrade() -> None:
op.drop_column('scans', 'started_at')
# Drop the enum type
- op.execute("DROP TYPE scannerstatus")
\ No newline at end of file
+ op.execute("DROP TYPE transformstatus")
\ No newline at end of file
diff --git a/flowsint-api/app/api/routes/events.py b/flowsint-api/app/api/routes/events.py
index 92b3f260..201aedb1 100644
--- a/flowsint-api/app/api/routes/events.py
+++ b/flowsint-api/app/api/routes/events.py
@@ -105,9 +105,9 @@ async def stream_events(
continue
# Handle different types of events
- if isinstance(data, dict) and data.get("type") == "scanner_complete":
- # Send scanner completion event
- yield json.dumps({"event": "scanner_complete", "data": data})
+ if isinstance(data, dict) and data.get("type") == "transform_complete":
+ # Send transform completion event
+ yield json.dumps({"event": "transform_complete", "data": data})
else:
# Send regular log event
yield json.dumps({"event": "log", "data": data})
diff --git a/flowsint-api/app/api/routes/flows.py b/flowsint-api/app/api/routes/flows.py
index b7da55d9..7bae3698 100644
--- a/flowsint-api/app/api/routes/flows.py
+++ b/flowsint-api/app/api/routes/flows.py
@@ -72,27 +72,27 @@ def get_flows(
# Returns the "raw_materials" for the flow editor
@router.get("/raw_materials")
async def get_material_list():
- scanners = TransformRegistry.list_by_categories()
- scanner_categories = {
+ transforms = TransformRegistry.list_by_categories()
+ transform_categories = {
category: [
{
- "class_name": scanner.get("class_name"),
- "category": scanner.get("category"),
- "name": scanner.get("name"),
- "module": scanner.get("module"),
- "documentation": scanner.get("documentation"),
- "description": scanner.get("description"),
- "inputs": scanner.get("inputs"),
- "outputs": scanner.get("outputs"),
- "type": "scanner",
- "params": scanner.get("params"),
- "params_schema": scanner.get("params_schema"),
- "required_params": scanner.get("required_params"),
- "icon": scanner.get("icon"),
+ "class_name": transform.get("class_name"),
+ "category": transform.get("category"),
+ "name": transform.get("name"),
+ "module": transform.get("module"),
+ "documentation": transform.get("documentation"),
+ "description": transform.get("description"),
+ "inputs": transform.get("inputs"),
+ "outputs": transform.get("outputs"),
+ "type": "transform",
+ "params": transform.get("params"),
+ "params_schema": transform.get("params_schema"),
+ "required_params": transform.get("required_params"),
+ "icon": transform.get("icon"),
}
- for scanner in scanner_list
+ for transform in transform_list
]
- for category, scanner_list in scanners.items()
+ for category, transform_list in transforms.items()
}
object_inputs = [
@@ -112,11 +112,11 @@ async def get_material_list():
extract_input_schema_flow(CryptoNFT),
]
- # Put types first, then add all scanner categories
- flattened_scanners = {"types": object_inputs}
- flattened_scanners.update(scanner_categories)
+ # Put types first, then add all transform categories
+ flattened_transforms = {"types": object_inputs}
+ flattened_transforms.update(transform_categories)
- return {"items": flattened_scanners}
+ return {"items": flattened_transforms}
# Returns the "raw_materials" for the flow editor
@@ -296,8 +296,8 @@ def compute_flow_branches(
node_map = {node.id: node for node in nodes}
branches = []
branch_counter = 0
- # Track scanner outputs across all branches
- scanner_outputs = {}
+ # Track transform outputs across all branches
+ transform_outputs = {}
def calculate_path_length(start_node: str, visited: set = None) -> int:
"""Calculate the shortest possible path length from a node to any leaf"""
@@ -340,7 +340,7 @@ def compute_flow_branches(
params=node_params,
inputs={} if is_input_node else input_data,
outputs=outputs,
- type="type" if is_input_node else "scanner",
+ type="type" if is_input_node else "transform",
status="pending",
branchId=branch_id,
depth=depth,
@@ -376,13 +376,13 @@ def compute_flow_branches(
)
current_outputs = {first_output_name: initial_value}
else:
- # Check if we already have outputs for this scanner
- if current_node_id in scanner_outputs:
- current_outputs = scanner_outputs[current_node_id]
+ # Check if we already have outputs for this transform
+ if current_node_id in transform_outputs:
+ current_outputs = transform_outputs[current_node_id]
else:
current_outputs = process_node_data(current_node, input_data)
# Store the outputs for future use
- scanner_outputs[current_node_id] = current_outputs
+ transform_outputs[current_node_id] = current_outputs
# Extract node parameters
node_params = current_node.data.get("params", {})
@@ -493,49 +493,49 @@ def process_node_data(node: Node, inputs: Dict[str, Any]) -> Dict[str, Any]:
for output in output_types:
output_name = output.get("name", "output")
class_name = node.data.get("class_name", "")
- # For simulation purposes, we'll return a placeholder value based on the scanner type
- if class_name in ["ReverseResolveScanner", "ResolveScanner"]:
- # IP/Domain resolution scanners
+ # For simulation purposes, we'll return a placeholder value based on the transform type
+ if class_name in ["ReverseResolveTransform", "ResolveTransform"]:
+ # IP/Domain resolution transforms
outputs[output_name] = (
"192.168.1.1" if "ip" in output_name.lower() else "example.com"
)
- elif class_name == "SubdomainScanner":
- # Subdomain scanner
+ elif class_name == "SubdomainTransform":
+ # Subdomain transform
outputs[output_name] = f"sub.{inputs.get('input', 'example.com')}"
- elif class_name == "WhoisScanner":
- # WHOIS scanner
+ elif class_name == "WhoisTransform":
+ # WHOIS transform
outputs[output_name] = {
"domain": inputs.get("input", "example.com"),
"registrar": "Example Registrar",
"creation_date": "2020-01-01",
}
- elif class_name == "IpToInfosScanner":
- # Geolocation scanner
+ elif class_name == "IpToInfosTransform":
+ # Geolocation transform
outputs[output_name] = {
"country": "France",
"city": "Paris",
"coordinates": {"lat": 48.8566, "lon": 2.3522},
}
- elif class_name == "MaigretScanner":
- # Social media scanner
+ elif class_name == "MaigretTransform":
+ # Social media transform
outputs[output_name] = {
"username": inputs.get("input", "user123"),
"platforms": ["twitter", "github", "linkedin"],
}
- elif class_name == "HoleheScanner":
- # Email verification scanner
+ elif class_name == "HoleheTransform":
+ # Email verification transform
outputs[output_name] = {
"email": inputs.get("input", "[email protected]"),
"exists": True,
"platforms": ["gmail", "github"],
}
- elif class_name == "SireneScanner":
- # Organization scanner
+ elif class_name == "SireneTransform":
+ # Organization transform
outputs[output_name] = {
"name": inputs.get("input", "Example Corp"),
"siret": "12345678901234",
@@ -543,7 +543,7 @@ def process_node_data(node: Node, inputs: Dict[str, Any]) -> Dict[str, Any]:
}
else:
- # For unknown scanners, pass through the input
+ # For unknown transforms, pass through the input
outputs[output_name] = inputs.get("input") or f"flowed_{output_name}"
return outputs
diff --git a/flowsint-api/app/utils.py b/flowsint-api/app/utils.py
index 5a6441ac..1a0bd16d 100644
--- a/flowsint-api/app/utils.py
+++ b/flowsint-api/app/utils.py
@@ -215,18 +215,18 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:
input_output = input_node["data"]["outputs"]
node_lookup = {node["id"]: node for node in nodes}
- scanners = []
+ transforms = []
for edge in edges:
target_id = edge["target"]
source_handle = edge["sourceHandle"]
target_handle = edge["targetHandle"]
- scanner_node = node_lookup.get(target_id)
- if scanner_node and scanner_node["data"]["type"] == "scanner":
- scanners.append(
+ transform_node = node_lookup.get(target_id)
+ if transform_node and transform_node["data"]["type"] == "transform":
+ transforms.append(
{
- "transform_name": scanner_node["data"]["name"],
- "module": scanner_node["data"]["module"],
+ "transform_name": transform_node["data"]["name"],
+ "module": transform_node["data"]["module"],
"input": source_handle,
"output": target_handle,
}
@@ -237,8 +237,8 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:
"name": input_node["data"]["name"],
"outputs": input_output,
},
- "scanners": scanners,
- "transform_names": [scanner["transform_name"] for scanner in scanners],
+ "transforms": transforms,
+ "transform_names": [transform["transform_name"] for transform in transforms],
}
diff --git a/flowsint-app/src/renderer/src/components/auth-debug.tsx b/flowsint-app/src/renderer/src/components/auth-debug.tsx
deleted file mode 100644
index e08a887d..00000000
--- a/flowsint-app/src/renderer/src/components/auth-debug.tsx
+++ /dev/null
@@ -1,17 +0,0 @@
-import { useAuthStore } from '@/stores/auth-store';
-import { isAuthenticated } from '@/lib/auth-utils';
-
-export function AuthDebug() {
- const authState = useAuthStore();
-
- return (
- <div className="fixed bottom-4 right-4 p-4 bg-black/80 text-white rounded-lg shadow-lg z-50">
- <h3 className="font-bold mb-2">Auth State Debug</h3>
- <div className="space-y-1 text-sm">
- <p>Is Authenticated: {isAuthenticated() ? '✅' : '❌'}</p>
- <p>Has Token: {authState.token ? '✅' : '❌'}</p>
- <p>User: {authState.user ? authState.user.username : 'None'}</p>
- </div>
- </div>
- );
-}
\ No newline at end of file
diff --git a/flowsint-app/src/renderer/src/components/flows/editor.tsx b/flowsint-app/src/renderer/src/components/flows/editor.tsx
index 52ffc4ff..535b36b0 100644
--- a/flowsint-app/src/renderer/src/components/flows/editor.tsx
+++ b/flowsint-app/src/renderer/src/components/flows/editor.tsx
@@ -18,9 +18,9 @@ import {
import "@xyflow/react/dist/style.css"
import { Play, Pause, SkipForward, RefreshCw } from "lucide-react"
import { Button } from "@/components/ui/button"
-import ScannerNode from "./scanner-node"
+import TransformNode from "./transform-node"
import TypeNode from "./type-node"
-import { type ScannerNodeData } from "@/types/transform"
+import { type TransformNodeData } from "@/types/transform"
import { FlowControls } from "./controls"
import { getDagreLayoutedElements } from "@/lib/utils"
import { toast } from "sonner"
@@ -42,11 +42,11 @@ import ParamsDialog from "./params-dialog"
import FlowSheet from "./flow-sheet"
import ContextMenu from "./context-menu"
import { useNodesDisplaySettings } from "@/stores/node-display-settings"
-import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query"
+import { useMutation, useQueryClient } from "@tanstack/react-query"
import { queryKeys } from "@/api/query-keys"
const nodeTypes: NodeTypes = {
- scanner: ScannerNode,
+ transform: TransformNode,
type: TypeNode,
}
@@ -111,13 +111,13 @@ const FlowEditor = memo(({ initialEdges, initialNodes, theme, flow }: FlowEditor
// Update flow mutation
const updateFlowMutation = useMutation({
- mutationFn: ({ flowId, body }: { flowId: string; body: BodyInit }) =>
+ mutationFn: ({ flowId, body }: { flowId: string; body: BodyInit }) =>
flowService.update(flowId, body),
onSuccess: () => {
toast.success("Transform saved successfully.")
// Invalidate the flow detail query
if (flowId) {
- queryClient.invalidateQueries({
+ queryClient.invalidateQueries({
queryKey: queryKeys.flows.detail(flowId)
})
}
@@ -134,7 +134,7 @@ const FlowEditor = memo(({ initialEdges, initialNodes, theme, flow }: FlowEditor
router.navigate({ to: "/dashboard/flows" })
toast.success("Flow deleted successfully.")
// Invalidate flows list
- queryClient.invalidateQueries({
+ queryClient.invalidateQueries({
queryKey: queryKeys.flows.list
})
},
@@ -145,7 +145,7 @@ const FlowEditor = memo(({ initialEdges, initialNodes, theme, flow }: FlowEditor
// Compute flow mutation
const computeFlowMutation = useMutation({
- mutationFn: ({ flowId, body }: { flowId: string; body: BodyInit }) =>
+ mutationFn: ({ flowId, body }: { flowId: string; body: BodyInit }) =>
flowService.compute(flowId, body),
onSuccess: (response) => {
setFlowBranches(response.flowBranches)
@@ -229,11 +229,11 @@ const FlowEditor = memo(({ initialEdges, initialNodes, theme, flow }: FlowEditor
event.preventDefault()
if (!reactFlowWrapper.current || !reactFlowInstance) return
const reactFlowBounds = reactFlowWrapper.current.getBoundingClientRect()
- const scannerData = JSON.parse(event.dataTransfer.getData("application/json")) as ScannerNodeData & {
+ const transformData = JSON.parse(event.dataTransfer.getData("application/json")) as TransformNodeData & {
category: string
}
- if (scannerData.type === "type") {
+ if (transformData.type === "type") {
const existsType = nodes.find((node) => node.data.type === "type")
if (existsType) {
return toast.error("Only one type node is allowed")
@@ -244,26 +244,26 @@ const FlowEditor = memo(({ initialEdges, initialNodes, theme, flow }: FlowEditor
y: event.clientY - reactFlowBounds.top,
})
const newNode: FlowNode = {
- id: `${scannerData.name}-${Date.now()}`,
- type: scannerData.type === "type" ? "type" : "scanner",
+ id: `${transformData.name}-${Date.now()}`,
+ type: transformData.type === "type" ? "type" : "transform",
position,
data: {
- id: scannerData.id,
- class_name: scannerData.class_name,
- module: scannerData.module || "",
- key: scannerData.name,
- color: colors[scannerData.category.toLowerCase()] || "#94a3b8",
- name: scannerData.name,
- category: scannerData.category,
- type: scannerData.type,
- inputs: scannerData.inputs,
- outputs: scannerData.outputs,
- documentation: scannerData.documentation,
- description: scannerData.description,
- required_params: scannerData.required_params,
- params: scannerData.params,
- params_schema: scannerData.params_schema,
- icon: scannerData.icon
+ id: transformData.id,
+ class_name: transformData.class_name,
+ module: transformData.module || "",
+ key: transformData.name,
+ color: colors[transformData.category.toLowerCase()] || "#94a3b8",
+ name: transformData.name,
+ category: transformData.category,
+ type: transformData.type,
+ inputs: transformData.inputs,
+ outputs: transformData.outputs,
+ documentation: transformData.documentation,
+ description: transformData.description,
+ required_params: transformData.required_params,
+ params: transformData.params,
+ params_schema: transformData.params_schema,
+ icon: transformData.icon
},
}
const updatedNodes = [...nodes, newNode]
@@ -335,7 +335,7 @@ const FlowEditor = memo(({ initialEdges, initialNodes, theme, flow }: FlowEditor
toast.error("Make sure your transform contains an input type.")
return
}
-
+
const body = JSON.stringify({
name: name,
description: description,
@@ -345,7 +345,7 @@ const FlowEditor = memo(({ initialEdges, initialNodes, theme, flow }: FlowEditor
edges,
},
})
-
+
if (flowId) {
await updateFlowMutation.mutateAsync({ flowId, body })
} else {
diff --git a/flowsint-app/src/renderer/src/components/flows/flow-sheet.tsx b/flowsint-app/src/renderer/src/components/flows/flow-sheet.tsx
index 5b204b4d..ea2a29b5 100644
--- a/flowsint-app/src/renderer/src/components/flows/flow-sheet.tsx
+++ b/flowsint-app/src/renderer/src/components/flows/flow-sheet.tsx
@@ -11,7 +11,7 @@ import { memo, useCallback, useMemo, useState } from "react"
import { TriangleAlert, Loader2, ArrowRight } from "lucide-react"
import { TooltipProvider, Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"
import { useNodesDisplaySettings } from "@/stores/node-display-settings"
-import { Scanner } from "@/types/transform"
+import { Transform } from "@/types/transform"
import { useQuery } from "@tanstack/react-query"
import { Input } from "../ui/input"
import { Alert, AlertDescription } from "../ui/alert"
@@ -33,48 +33,48 @@ const FlowSheet = ({ onLayout }: { onLayout: () => void }) => {
})
const [searchTerm, setSearchTerm] = useState<string>("")
- const filteredScanners = useMemo(() => {
+ const filteredTransforms = useMemo(() => {
if (!materials?.items || !selectedNode) return []
if (!searchTerm.trim()) {
return materials?.items
}
const term = searchTerm.toLowerCase()
- return materials?.items.filter((scanner: Scanner) => {
+ return materials?.items.filter((transform: Transform) => {
return (
- scanner.class_name.toLowerCase().includes(term) ||
- scanner.name.toLowerCase().includes(term) ||
- scanner.module.toLowerCase().includes(term) ||
- scanner.inputs.type.toLowerCase().includes(term) ||
- scanner.outputs.type.toLowerCase().includes(term) ||
- (scanner.documentation && scanner.documentation.toLowerCase().includes(term))
+ transform.class_name.toLowerCase().includes(term) ||
+ transform.name.toLowerCase().includes(term) ||
+ transform.module.toLowerCase().includes(term) ||
+ transform.inputs.type.toLowerCase().includes(term) ||
+ transform.outputs.type.toLowerCase().includes(term) ||
+ (transform.documentation && transform.documentation.toLowerCase().includes(term))
)
})
}, [searchTerm, materials?.items, selectedNode])
- const handleClick = useCallback((scanner: Scanner) => {
+ const handleClick = useCallback((transform: Transform) => {
if (!selectedNode) return
const position = { x: selectedNode.position.x + 350, y: selectedNode.position.y }
const newNode: FlowNode = {
- id: `${scanner.name}-${Date.now()}`,
- type: scanner.type === "type" ? "type" : "scanner",
+ id: `${transform.name}-${Date.now()}`,
+ type: transform.type === "type" ? "type" : "transform",
position,
data: {
- id: scanner.id,
- class_name: scanner.class_name,
- module: scanner.module || "",
- key: scanner.name,
- color: colors[scanner.category.toLowerCase()] || "#94a3b8",
- name: scanner.name,
- category: scanner.category,
- type: scanner.type,
- inputs: scanner.inputs,
- outputs: scanner.outputs,
- documentation: scanner.documentation,
- description: scanner.description,
- required_params: scanner.required_params,
- params: scanner.params,
- params_schema: scanner.params_schema,
- icon: scanner.icon
+ id: transform.id,
+ class_name: transform.class_name,
+ module: transform.module || "",
+ key: transform.name,
+ color: colors[transform.category.toLowerCase()] || "#94a3b8",
+ name: transform.name,
+ category: transform.category,
+ type: transform.type,
+ inputs: transform.inputs,
+ outputs: transform.outputs,
+ documentation: transform.documentation,
+ description: transform.description,
+ required_params: transform.required_params,
+ params: transform.params,
+ params_schema: transform.params_schema,
+ icon: transform.icon
},
}
setNodes((prev) => ([...prev, newNode]))
@@ -84,7 +84,7 @@ const FlowSheet = ({ onLayout }: { onLayout: () => void }) => {
source: selectedNode.id,
target: newNode.id,
sourceHandle: selectedNode.data.outputs.type,
- targetHandle: scanner.inputs.type
+ targetHandle: transform.inputs.type
}
setEdges((prev) => ([...prev, connection]))
// onLayout && onLayout()
@@ -125,13 +125,13 @@ const FlowSheet = ({ onLayout }: { onLayout: () => void }) => {
{!isLoading && !error && (
<div className="flex flex-col gap-2">
- {filteredScanners.length === 0 ? (
+ {filteredTransforms.length === 0 ? (
<div className="text-center py-8 text-muted-foreground">
{searchTerm.trim() ? 'No transforms found matching your search.' : 'No transforms available for this node type.'}
</div>
) : (
- filteredScanners.map((scanner: Scanner) => (
- <ScannerItem key={scanner.id} scanner={scanner} onClick={() => handleClick(scanner)} />
+ filteredTransforms.map((transform: Transform) => (
+ <TransformItem key={transform.id} transform={transform} onClick={() => handleClick(transform)} />
))
)}
</div>
@@ -146,22 +146,22 @@ const FlowSheet = ({ onLayout }: { onLayout: () => void }) => {
export default FlowSheet
-// Custom equality function for ScannerItem
-function areEqual(prevProps: { scanner: Scanner }, nextProps: { scanner: Scanner }) {
+// Custom equality function for TransformItem
+function areEqual(prevProps: { transform: Transform }, nextProps: { transform: Transform }) {
return (
- prevProps.scanner.class_name === nextProps.scanner.class_name &&
- prevProps.scanner.name === nextProps.scanner.name &&
- prevProps.scanner.module === nextProps.scanner.module &&
- prevProps.scanner.documentation === nextProps.scanner.documentation
+ prevProps.transform.class_name === nextProps.transform.class_name &&
+ prevProps.transform.name === nextProps.transform.name &&
+ prevProps.transform.module === nextProps.transform.module &&
+ prevProps.transform.documentation === nextProps.transform.documentation
)
}
-// Memoized scanner item component for the sidebar
-const ScannerItem = memo(({ scanner, onClick }: { scanner: Scanner, onClick: () => void }) => {
+// Memoized transform item component for the sidebar
+const TransformItem = memo(({ transform, onClick }: { transform: Transform, onClick: () => void }) => {
const colors = useNodesDisplaySettings(s => s.colors)
- const borderInputColor = colors[scanner.inputs.type.toLowerCase()]
- const borderOutputColor = colors[scanner.outputs.type.toLowerCase()]
- const Icon = scanner.type === "type" ? useIcon(scanner.outputs.type.toLowerCase() as string, null) : scanner.icon ? useIcon(scanner.icon, null) : null
+ const borderInputColor = colors[transform.inputs.type.toLowerCase()]
+ const borderOutputColor = colors[transform.outputs.type.toLowerCase()]
+ const Icon = transform.type === "type" ? useIcon(transform.outputs.type.toLowerCase() as string, null) : transform.icon ? useIcon(transform.icon, null) : null
return (
<TooltipProvider>
@@ -175,24 +175,24 @@ const ScannerItem = memo(({ scanner, onClick }: { scanner: Scanner, onClick: ()
<div className="space-y-1 truncate text-left">
<div className="flex items-center gap-2 truncate text-ellipsis">
{Icon && <Icon size={24} />}
- <h3 className="text-sm font-medium truncate text-ellipsis">{scanner.class_name}</h3>
+ <h3 className="text-sm font-medium truncate text-ellipsis">{transform.class_name}</h3>
</div>
- <p className="text-sm font-normal opacity-60 truncate text-ellipsis">{scanner.description}</p>
+ <p className="text-sm font-normal opacity-60 truncate text-ellipsis">{transform.description}</p>
<div className="mt-2 flex items-center gap-2 text-xs">
<div className="flex items-center gap-1">
<span className="text-muted-foreground">Takes</span>
- <span className="font-bold truncate text-ellipsis">{scanner.inputs.type}</span>
+ <span className="font-bold truncate text-ellipsis">{transform.inputs.type}</span>
</div>
<span><ArrowRight className="h-3 w-3" /></span>
<div className="flex items-center gap-1">
<span className="text-muted-foreground">Returns</span>
- <span className="font-bold truncate text-ellipsis">{scanner.outputs.type}</span>
+ <span className="font-bold truncate text-ellipsis">{transform.outputs.type}</span>
</div>
</div>
</div>
</div>
</div>
- {scanner.required_params &&
+ {transform.required_params &&
<div className="absolute bottom-3 right-3">
<Tooltip>
<TooltipTrigger asChild>
@@ -208,4 +208,4 @@ const ScannerItem = memo(({ scanner, onClick }: { scanner: Scanner, onClick: ()
)
}, areEqual)
-ScannerItem.displayName = "ScannerItem"
+TransformItem.displayName = "TransformItem"
diff --git a/flowsint-app/src/renderer/src/components/flows/params-dialog.tsx b/flowsint-app/src/renderer/src/components/flows/params-dialog.tsx
index cccf3845..5e306e10 100644
--- a/flowsint-app/src/renderer/src/components/flows/params-dialog.tsx
+++ b/flowsint-app/src/renderer/src/components/flows/params-dialog.tsx
@@ -2,7 +2,7 @@ import { useFlowStore } from '@/stores/flow-store'
import { DialogHeader, DialogFooter, Dialog, DialogContent, DialogTitle } from '../ui/dialog'
import { Button } from '../ui/button'
import { useCallback, useState, useEffect } from 'react'
-import { ScannerParamSchemaItem } from '@/types'
+import { TransformParamSchemaItem } from '@/types'
import { Label } from '../ui/label'
import { Input } from '../ui/input'
import KeySelector from '../keys/key-select'
@@ -78,7 +78,7 @@ const ParamsDialog = () => {
</TabsList>
<TabsContent value="parameters" className="space-y-4 mt-4">
<div className="grid gap-4">
- {selectedNode?.data?.params_schema?.map((param: ScannerParamSchemaItem) => (
+ {selectedNode?.data?.params_schema?.map((param: TransformParamSchemaItem) => (
<div className="space-y-2" key={param.name}>
<div className="flex items-start flex-col">
<Label htmlFor={param.name} className="text-sm font-medium">
diff --git a/flowsint-app/src/renderer/src/components/flows/raw-material.tsx b/flowsint-app/src/renderer/src/components/flows/raw-material.tsx
index 03e8cf7e..bd718255 100644
--- a/flowsint-app/src/renderer/src/components/flows/raw-material.tsx
+++ b/flowsint-app/src/renderer/src/components/flows/raw-material.tsx
@@ -1,7 +1,7 @@
import { useMemo, useState } from "react"
import { Search, X } from "lucide-react"
-import ScannerItem from "./scanner-item"
-import { type Scanner } from "@/types/transform"
+import TransformItem from "./transform-list-item"
+import { type Transform } from "@/types/transform"
import { Input } from "../ui/input"
import { Button } from "../ui/button"
import { useQuery } from "@tanstack/react-query"
@@ -17,9 +17,9 @@ export default function RawMaterial() {
})
const [searchTerm, setSearchTerm] = useState<string>("")
- const filteredScanners = useMemo(() => {
+ const filteredTransforms = useMemo(() => {
if (!materials?.items) return {}
- const result: Record<string, Scanner[]> = {}
+ const result: Record<string, Transform[]> = {}
if (!searchTerm.trim()) {
return materials?.items
}
@@ -50,7 +50,7 @@ export default function RawMaterial() {
<Search className="absolute left-2 top-2.5 h-4 w-4 text-muted-foreground" />
<Input
type="text"
- placeholder="Search scanners..."
+ placeholder="Search transforms..."
className="pl-8 !border border-border"
value={searchTerm}
onChange={(e) => setSearchTerm(e.target.value)}
@@ -67,24 +67,24 @@ export default function RawMaterial() {
)}
</div>
<div className="flex-1 w-full">
- {Object.entries(filteredScanners).map(([category, scanners]) => (
+ {Object.entries(filteredTransforms).map(([category, transforms]) => (
<div key={category} className="space-y-2 w-full">
<h3 className="text-sm font-medium capitalize mt-4">{category.replace("_", " ")}</h3>
<div className="space-y-2">
{/* @ts-ignore */}
- {scanners.map((scanner: Scanner) => (
- <ScannerItem
- key={scanner.name}
- scanner={scanner}
- category={scanner.category}
+ {transforms.map((transform: Transform) => (
+ <TransformItem
+ key={transform.name}
+ transform={transform}
+ category={transform.category}
/>
))}
</div>
</div>
))}
- {Object.keys(filteredScanners).length === 0 && (
+ {Object.keys(filteredTransforms).length === 0 && (
<div className="text-center py-4 text-muted-foreground">
- No scanners found matching "{searchTerm}"
+ No transforms found matching "{searchTerm}"
</div>
)}
</div>
diff --git a/flowsint-app/src/renderer/src/components/flows/scanner-item.tsx b/flowsint-app/src/renderer/src/components/flows/transform-list-item.tsx
similarity index 75%
rename from flowsint-app/src/renderer/src/components/flows/scanner-item.tsx
rename to flowsint-app/src/renderer/src/components/flows/transform-list-item.tsx
index 862692d1..441c3cfe 100644
--- a/flowsint-app/src/renderer/src/components/flows/scanner-item.tsx
+++ b/flowsint-app/src/renderer/src/components/flows/transform-list-item.tsx
@@ -7,41 +7,41 @@ import { TooltipProvider, Tooltip, TooltipContent, TooltipTrigger } from "@/comp
import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogTrigger } from "@/components/ui/dialog"
import { useNodesDisplaySettings } from "@/stores/node-display-settings"
import { Badge } from "../ui/badge"
-import { type ScannerItemProps } from "@/types/transform"
+import { type TransformItemProps } from "@/types/transform"
import { useIcon } from "@/hooks/use-icon"
-// Custom equality function for ScannerItem
-function areEqual(prevProps: ScannerItemProps, nextProps: ScannerItemProps) {
+// Custom equality function for TransformItem
+function areEqual(prevProps: TransformItemProps, nextProps: TransformItemProps) {
return (
- prevProps.scanner.class_name === nextProps.scanner.class_name &&
- prevProps.scanner.name === nextProps.scanner.name &&
- prevProps.scanner.module === nextProps.scanner.module &&
- prevProps.scanner.documentation === nextProps.scanner.documentation &&
- prevProps.scanner.description === nextProps.scanner.description &&
+ prevProps.transform.class_name === nextProps.transform.class_name &&
+ prevProps.transform.name === nextProps.transform.name &&
+ prevProps.transform.module === nextProps.transform.module &&
+ prevProps.transform.documentation === nextProps.transform.documentation &&
+ prevProps.transform.description === nextProps.transform.description &&
prevProps.category === nextProps.category
)
}
-// Memoized scanner item component for the sidebar
-const ScannerItem = memo(({ scanner, category }: ScannerItemProps) => {
+// Memoized transform item component for the sidebar
+const TransformItem = memo(({ transform, category }: TransformItemProps) => {
const colors = useNodesDisplaySettings(s => s.colors)
- const borderInputColor = colors[scanner.inputs.type.toLowerCase()]
- const borderOutputColor = colors[scanner.outputs.type.toLowerCase()]
- const Icon = scanner.type === "type" ? useIcon(scanner.outputs.type.toLowerCase() as string, null) : scanner.icon ? useIcon(scanner.icon, null) : null
+ const borderInputColor = colors[transform.inputs.type.toLowerCase()]
+ const borderOutputColor = colors[transform.outputs.type.toLowerCase()]
+ const Icon = transform.type === "type" ? useIcon(transform.outputs.type.toLowerCase() as string, null) : transform.icon ? useIcon(transform.icon, null) : null
const [isDialogOpen, setIsDialogOpen] = useState(false)
// Handler for drag start - using useCallback to prevent recreation on each render
const onDragStart = useCallback(
(event: React.DragEvent<HTMLDivElement>) => {
- const data = { ...scanner, category }
+ const data = { ...transform, category }
event.dataTransfer.setData("application/json", JSON.stringify(data))
event.dataTransfer.effectAllowed = "move"
},
- [scanner, category],
+ [transform, category],
)
- const isConfigurationRequired = scanner.required_params
+ const isConfigurationRequired = transform.required_params
return (
<TooltipProvider>
@@ -60,18 +60,18 @@ const ScannerItem = memo(({ scanner, category }: ScannerItemProps) => {
<div className="space-y-1 truncate">
<div className="flex items-center gap-2 truncate text-ellipsis">
{Icon && <Icon size={24} />}
- <h3 className="text-sm font-medium truncate text-ellipsis">{scanner.class_name}</h3>
+ <h3 className="text-sm font-medium truncate text-ellipsis">{transform.class_name}</h3>
</div>
- <p className="text-xs font-normal truncate text-ellipsis opacity-60">{scanner.description}</p>
- {scanner.type !== "type" &&
+ <p className="text-xs font-normal truncate text-ellipsis opacity-60">{transform.description}</p>
+ {transform.type !== "type" &&
<div className="mt-2 text-xs">
<div className="flex items-center gap-1">
<span className="font-medium">Input:</span>
- <span className="text-muted-foreground truncate text-ellipsis">{scanner.inputs.type}</span>
+ <span className="text-muted-foreground truncate text-ellipsis">{transform.inputs.type}</span>
</div>
<div className="flex items-center gap-1">
<span className="font-medium">Output:</span>
- <span className="text-muted-foreground truncate text-ellipsis">{scanner.outputs.type}</span>
+ <span className="text-muted-foreground truncate text-ellipsis">{transform.outputs.type}</span>
</div>
</div>}
</div>
@@ -100,7 +100,7 @@ const ScannerItem = memo(({ scanner, category }: ScannerItemProps) => {
<DialogHeader>
<DialogTitle className="flex items-center gap-2">
<div className="w-1 h-6 rounded-full" style={{ backgroundColor: borderInputColor }} />
- {scanner.class_name}
+ {transform.class_name}
</DialogTitle>
</DialogHeader>
<div className="grid gap-4 py-4">
@@ -112,16 +112,16 @@ const ScannerItem = memo(({ scanner, category }: ScannerItemProps) => {
</div>}
<div className="space-y-2">
<h4 className="font-medium text-sm" style={{ color: borderInputColor }}>Description</h4>
- <p className="text-sm text-muted-foreground">{scanner.description || "No description available"}</p>
+ <p className="text-sm text-muted-foreground">{transform.description || "No description available"}</p>
</div>
<div className="space-y-2">
<h4 className="font-medium text-sm" style={{ color: borderInputColor }}>Module</h4>
- <p className="text-sm text-muted-foreground">{scanner.module}</p>
+ <p className="text-sm text-muted-foreground">{transform.module}</p>
</div>
<div className="space-y-2">
<h4 className="font-medium text-sm" style={{ color: borderInputColor }}>Input Properties</h4>
<div className="space-y-1">
- {scanner?.inputs?.properties?.map((prop, index) => (
+ {transform?.inputs?.properties?.map((prop, index) => (
<div key={index} className="text-sm">
<span className="font-medium">{prop.name}:</span>{" "}
<span className="text-muted-foreground">{prop.type}</span>
@@ -132,7 +132,7 @@ const ScannerItem = memo(({ scanner, category }: ScannerItemProps) => {
<div className="space-y-2">
<h4 className="font-medium text-sm" style={{ color: borderOutputColor }}>Output Properties</h4>
<div className="space-y-1">
- {scanner?.outputs?.properties?.map((prop, index) => (
+ {transform?.outputs?.properties?.map((prop, index) => (
<div key={index} className="text-sm">
<span className="font-medium">{prop.name}:</span>{" "}
<span className="text-muted-foreground">{prop.type}</span>
@@ -147,6 +147,6 @@ const ScannerItem = memo(({ scanner, category }: ScannerItemProps) => {
)
}, areEqual)
-ScannerItem.displayName = "ScannerItem"
+TransformItem.displayName = "TransformItem"
-export default ScannerItem
+export default TransformItem
diff --git a/flowsint-app/src/renderer/src/components/flows/scanner-node.tsx b/flowsint-app/src/renderer/src/components/flows/transform-node.tsx
similarity index 95%
rename from flowsint-app/src/renderer/src/components/flows/scanner-node.tsx
rename to flowsint-app/src/renderer/src/components/flows/transform-node.tsx
index 9dd40193..c50a6378 100644
--- a/flowsint-app/src/renderer/src/components/flows/scanner-node.tsx
+++ b/flowsint-app/src/renderer/src/components/flows/transform-node.tsx
@@ -6,7 +6,7 @@ import { NodeStatusIndicator } from "../xyflow/node-status-indicator"
import { useNodesDisplaySettings } from "@/stores/node-display-settings"
import { cn } from "@/lib/utils"
import { Plus, TriangleAlert } from "lucide-react"
-import { type ScannerNodeProps } from "@/types/transform"
+import { type TransformNodeProps } from "@/types/transform"
import { FlowNode, useFlowStore } from "@/stores/flow-store"
import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"
import { Button } from "../ui/button"
@@ -14,7 +14,7 @@ import { ButtonHandle } from "../xyflow/button-handle"
import { useIcon } from "@/hooks/use-icon"
// Custom equality function to prevent unnecessary re-renders
-function areEqual(prevProps: ScannerNodeProps, nextProps: ScannerNodeProps) {
+function areEqual(prevProps: TransformNodeProps, nextProps: TransformNodeProps) {
return (
prevProps.data.class_name === nextProps.data.class_name &&
prevProps.data.name === nextProps.data.name &&
@@ -44,8 +44,8 @@ const getStateColor = (state?: string) => {
}
}
-// Scanner node component for transform/scanner nodes only
-const ScannerNode = memo(({ data, isConnectable }: ScannerNodeProps) => {
+// Transform node component for transform/transform nodes only
+const TransformNode = memo(({ data, isConnectable }: TransformNodeProps) => {
const colors = useNodesDisplaySettings(s => s.colors)
const inputColor = colors[data.inputs.type.toLowerCase()]
const outputColor = colors[data.outputs.type.toLowerCase()]
@@ -174,6 +174,6 @@ const ScannerNode = memo(({ data, isConnectable }: ScannerNodeProps) => {
)
}, areEqual)
-ScannerNode.displayName = "ScannerNode"
+TransformNode.displayName = "TransformNode"
-export default ScannerNode
+export default TransformNode
diff --git a/flowsint-app/src/renderer/src/components/flows/type-node.tsx b/flowsint-app/src/renderer/src/components/flows/type-node.tsx
index 12c8c30b..4eb9ee1c 100644
--- a/flowsint-app/src/renderer/src/components/flows/type-node.tsx
+++ b/flowsint-app/src/renderer/src/components/flows/type-node.tsx
@@ -4,11 +4,9 @@ import { useNodesDisplaySettings } from "@/stores/node-display-settings"
import { useIcon } from "@/hooks/use-icon"
import { cn } from "@/lib/utils"
import { Plus } from "lucide-react"
-import { type ScannerNodeProps } from "@/types/transform"
+import { type TransformNodeProps } from "@/types/transform"
import { ButtonHandle } from "@/components/xyflow/button-handle"
import { Button } from "../ui/button"
-
-
import {
NodeTooltip,
NodeTooltipContent,
@@ -18,7 +16,7 @@ import { BaseNode, BaseNodeContent } from "@/components/xyflow/base-node";
import { FlowNode, useFlowStore } from "@/stores/flow-store"
// Custom equality function to prevent unnecessary re-renders
-function areEqual(prevProps: ScannerNodeProps, nextProps: ScannerNodeProps) {
+function areEqual(prevProps: TransformNodeProps, nextProps: TransformNodeProps) {
return (
prevProps.data.class_name === nextProps.data.class_name &&
prevProps.data.name === nextProps.data.name &&
@@ -28,7 +26,7 @@ function areEqual(prevProps: ScannerNodeProps, nextProps: ScannerNodeProps) {
}
// Type node component for data types (domains, websites, IPs, etc.)
-const TypeNode = memo(({ data }: ScannerNodeProps) => {
+const TypeNode = memo(({ data }: TransformNodeProps) => {
const colors = useNodesDisplaySettings(s => s.colors)
const outputColor = colors[data.outputs.type.toLowerCase()]
const Icon = useIcon(data.outputs.type.toLowerCase() as string, null)
diff --git a/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx b/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx
index 9a0f73b5..ffb3e9d7 100644
--- a/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx
@@ -189,7 +189,7 @@ export default function ContextMenu({
</div>
<div className='flex items-center gap-1'>
{/* <FavoriteButton isFavorite={false} /> */}
- <InfoButton description={transform.description} />
+ <InfoButton description={transform.description ?? ""} />
</div>
</button>
))}
diff --git a/flowsint-app/src/renderer/src/components/investigations/sketch-list.tsx b/flowsint-app/src/renderer/src/components/investigations/sketch-list.tsx
index a35af897..08304ea3 100644
--- a/flowsint-app/src/renderer/src/components/investigations/sketch-list.tsx
+++ b/flowsint-app/src/renderer/src/components/investigations/sketch-list.tsx
@@ -24,7 +24,7 @@ const SketchList = () => {
if (!searchQuery.trim()) return investigation.sketches
const query = searchQuery.toLowerCase().trim()
- return investigation.sketches.filter((sketch) =>
+ return investigation.sketches.filter((sketch) =>
sketch.title.toLowerCase().includes(query)
)
}, [investigation?.sketches, searchQuery])
@@ -33,15 +33,15 @@ const SketchList = () => {
return (
<div className="w-full h-full bg-card flex flex-col overflow-hidden">
<div className="p-2 flex items-center gap-2 border-b shrink-0">
- <NewSketch noDropDown>
+ <NewSketch>
<Button variant="ghost" size="icon" className="h-7 w-7">
<PlusIcon className="h-4 w-4" />
</Button>
</NewSketch>
- <Input
- type="search"
- className="!border border-border h-7"
- placeholder="Search..."
+ <Input
+ type="search"
+ className="!border border-border h-7"
+ placeholder="Search..."
value={searchQuery}
onChange={(e) => setSearchQuery(e.target.value)}
/>
@@ -71,13 +71,13 @@ const SketchList = () => {
{searchQuery ? "No matching sketches" : "No sketches yet"}
</h3>
<p className="text-xs opacity-70 max-w-xs">
- {searchQuery
+ {searchQuery
? "Try adjusting your search query to find what you're looking for."
: "This investigation doesn't contain any sketch. Sketches let you organize and visualize your data as graphs."
}
</p>
</div>
- <NewSketch noDropDown>
+ <NewSketch>
<Button size="sm" variant="outline">
<PlusIcon className="w-4 h-4 mr-2" />
Create your first sketch
diff --git a/flowsint-app/src/renderer/src/components/layout/info.tsx b/flowsint-app/src/renderer/src/components/layout/info.tsx
index f1815e27..4bc799cd 100644
--- a/flowsint-app/src/renderer/src/components/layout/info.tsx
+++ b/flowsint-app/src/renderer/src/components/layout/info.tsx
@@ -41,7 +41,7 @@ const InfoDialog = () => {
<strong>Tracks and saves investigation states</strong> over time, letting you explore multiple hypotheses or revisit older threads without losing context.
</li>
<li>
- Supports <strong>live data enrichment</strong> from custom or built-in scanners, giving you <strong>actionable insights</strong> as you explore.
+ Supports <strong>live data enrichment</strong> from custom or built-in transforms, giving you <strong>actionable insights</strong> as you explore.
</li>
</ul>
diff --git a/flowsint-app/src/renderer/src/components/table/columns.tsx b/flowsint-app/src/renderer/src/components/table/columns.tsx
index 8a88d312..d01815f6 100644
--- a/flowsint-app/src/renderer/src/components/table/columns.tsx
+++ b/flowsint-app/src/renderer/src/components/table/columns.tsx
@@ -13,7 +13,8 @@ import {
} from "@/components/ui/dropdown-menu"
import { Checkbox } from "../ui/checkbox"
import { useCallback, memo } from "react"
-import { GraphNode, useGraphStore } from "@/stores/graph-store"
+import { useGraphStore } from "@/stores/graph-store"
+import { GraphNode } from "@/types"
// Memoized icon component to prevent unnecessary re-renders
const MemoizedIcon = memo(({ type, size = 16 }: { type: string; size?: number }) => {
@@ -65,7 +66,7 @@ const getConfidenceColor = (confidence: number) => {
export const columns: ColumnDef<GraphNode>[] = [
{
size: 50,
- minSize:50,
+ minSize: 50,
maxSize: 50,
enableResizing: false,
accessorKey: "icon",
@@ -100,7 +101,7 @@ export const columns: ColumnDef<GraphNode>[] = [
{
enableResizing: true,
size: 200,
- minSize:120,
+ minSize: 120,
maxSize: 500,
accessorKey: "data.label",
header: ({ column }) => {
@@ -156,7 +157,7 @@ export const columns: ColumnDef<GraphNode>[] = [
)
},
size: 120,
- minSize:80,
+ minSize: 80,
maxSize: 200,
accessorKey: "data.type",
cell: ({ row }) => {
@@ -171,7 +172,7 @@ export const columns: ColumnDef<GraphNode>[] = [
{
enableResizing: true,
size: 200,
- minSize:120,
+ minSize: 120,
maxSize: 500,
header: ({ column }) => {
return (
@@ -202,7 +203,7 @@ export const columns: ColumnDef<GraphNode>[] = [
{
enableResizing: true,
size: 200,
- minSize:120,
+ minSize: 120,
maxSize: 500,
header: ({ column }) => {
return (
@@ -218,7 +219,7 @@ export const columns: ColumnDef<GraphNode>[] = [
},
accessorKey: "data.source",
cell: ({ row }) => {
- const source = row.original.data.source || 'Scanner'
+ const source = row.original.data.source || 'Transform'
return (
<div className="text-left font-medium text-gray-700 dark:text-gray-200">
{source}
@@ -229,7 +230,7 @@ export const columns: ColumnDef<GraphNode>[] = [
{
enableResizing: true,
size: 200,
- minSize:120,
+ minSize: 120,
maxSize: 500,
header: ({ column }) => {
return (
@@ -266,7 +267,7 @@ export const columns: ColumnDef<GraphNode>[] = [
{
enableResizing: true,
size: 200,
- minSize:120,
+ minSize: 120,
maxSize: 500,
header: ({ column }) => {
return (
@@ -295,7 +296,7 @@ export const columns: ColumnDef<GraphNode>[] = [
{
enableResizing: true,
size: 200,
- minSize:120,
+ minSize: 120,
maxSize: 500,
header: ({ column }) => {
return (
@@ -322,7 +323,7 @@ export const columns: ColumnDef<GraphNode>[] = [
{
id: "actions",
size: 50,
- minSize:50,
+ minSize: 50,
maxSize: 50,
enableResizing: false,
header: () => {
diff --git a/flowsint-app/src/renderer/src/routes/_auth.dashboard.tsx b/flowsint-app/src/renderer/src/routes/_auth.dashboard.tsx
index 76994cf5..6daaffc8 100644
--- a/flowsint-app/src/renderer/src/routes/_auth.dashboard.tsx
+++ b/flowsint-app/src/renderer/src/routes/_auth.dashboard.tsx
@@ -12,7 +12,7 @@ function DashboardPage() {
return (
<RootLayout>
<Outlet />
- {Boolean(settings.general.showFlow.value) && <FloatingChat />}
+ {Boolean(settings?.general?.showFlow?.value) && <FloatingChat />}
</RootLayout>
)
}
diff --git a/flowsint-app/src/renderer/src/stores/flow-store.ts b/flowsint-app/src/renderer/src/stores/flow-store.ts
index d039086d..1e88a1ec 100644
--- a/flowsint-app/src/renderer/src/stores/flow-store.ts
+++ b/flowsint-app/src/renderer/src/stores/flow-store.ts
@@ -12,9 +12,9 @@ import {
applyEdgeChanges
} from "@xyflow/react"
import { toast } from "sonner"
-import { type ScannerNodeData } from "@/types/transform"
+import { type TransformNodeData } from "@/types/transform"
-export type NodeData = ScannerNodeData
+export type NodeData = TransformNodeData
export type FlowNode = Node<NodeData>
export type FlowEdge = Edge
diff --git a/flowsint-app/src/renderer/src/types/event.ts b/flowsint-app/src/renderer/src/types/event.ts
index c2198af1..2d4895ca 100644
--- a/flowsint-app/src/renderer/src/types/event.ts
+++ b/flowsint-app/src/renderer/src/types/event.ts
@@ -1,4 +1,4 @@
-import type { GraphEdge, GraphNode } from "@/stores/graph-store";
+import { GraphEdge, GraphNode } from "./graph"
export enum EventLevel {
// Standard log levels
@@ -7,7 +7,7 @@ export enum EventLevel {
FAILED = "FAILED",
SUCCESS = "SUCCESS",
DEBUG = "DEBUG",
- // Scanner-specific statuses
+ // Transform-specific statuses
PENDING = "PENDING",
RUNNING = "RUNNING",
COMPLETED = "COMPLETED",
diff --git a/flowsint-app/src/renderer/src/types/transform.ts b/flowsint-app/src/renderer/src/types/transform.ts
index 1e2470fd..db5fc932 100644
--- a/flowsint-app/src/renderer/src/types/transform.ts
+++ b/flowsint-app/src/renderer/src/types/transform.ts
@@ -1,14 +1,14 @@
-export interface ScannerProperty {
+export interface TransformProperty {
name: string
type: string
}
-export interface ScannerIO {
+export interface TransformIO {
type: string
- properties: ScannerProperty[]
+ properties: TransformProperty[]
}
-export interface ScannerParamSchemaItem {
+export interface TransformParamSchemaItem {
name: string
type: string
description: string
@@ -16,7 +16,7 @@ export interface ScannerParamSchemaItem {
required: boolean
}
-export interface Scanner {
+export interface Transform {
id: string
class_name: string
category: string
@@ -24,12 +24,12 @@ export interface Scanner {
module: string
documentation: string | null
description: string | null
- inputs: ScannerIO
- outputs: ScannerIO
+ inputs: TransformIO
+ outputs: TransformIO
type: string
required_params: boolean
params: Record<string, string>
- params_schema: ScannerParamSchemaItem[]
+ params_schema: TransformParamSchemaItem[]
settings?: Record<string, string>
icon: string | null
}
@@ -38,7 +38,7 @@ export interface Scanner {
// NODE DATA TYPE FOR TRANSFORM STORE
// ================================
-export interface ScannerNodeData extends Scanner, Record<string, unknown> {
+export interface TransformNodeData extends Transform, Record<string, unknown> {
color?: string
computationState?: "pending" | "processing" | "completed" | "error"
key: string
@@ -49,10 +49,10 @@ export interface ScannerNodeData extends Scanner, Record<string, unknown> {
// ================================
export interface ScansData {
- [category: string]: Scanner[]
+ [category: string]: Transform[]
}
-export interface ScannerData {
+export interface TransformData {
items: ScansData
}
@@ -60,31 +60,17 @@ export interface ScannerData {
// COMPONENT PROPS INTERFACES
// ================================
-export interface ScannerItemProps {
- scanner: Scanner
+export interface TransformItemProps {
+ transform: Transform
category: string
}
-export interface ScannerNodeProps {
- data: ScannerNodeData
+export interface TransformNodeProps {
+ data: TransformNodeData
isConnectable?: boolean
selected?: boolean
}
-// ================================
-// TRANSFORM TYPE DEFINITIONS
-// ================================
-
-export interface Transform {
- id: string;
- class_name: string;
- name: string;
- module: string;
- description: string;
- documentation: string;
- category: string;
-}
-
// ================================
// TRANSFORM DATA STRUCTURES
// ================================
diff --git a/flowsint-core/poetry.lock b/flowsint-core/poetry.lock
index 0cb2523e..d253d797 100644
--- a/flowsint-core/poetry.lock
+++ b/flowsint-core/poetry.lock
@@ -1781,7 +1781,7 @@ version = "2.1.0"
description = "brain-dead simple config-ini parsing"
optional = false
python-versions = ">=3.8"
-groups = ["dev"]
+groups = ["main", "dev"]
files = [
{file = "iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760"},
{file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"},
@@ -3194,7 +3194,7 @@ version = "1.6.0"
description = "plugin and hook calling mechanisms for python"
optional = false
python-versions = ">=3.9"
-groups = ["dev"]
+groups = ["main", "dev"]
files = [
{file = "pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"},
{file = "pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3"},
@@ -3778,7 +3778,7 @@ version = "2.19.2"
description = "Pygments is a syntax highlighting package written in Python."
optional = false
python-versions = ">=3.8"
-groups = ["main"]
+groups = ["main", "dev"]
files = [
{file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"},
{file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"},
@@ -3925,24 +3925,25 @@ files = [
[[package]]
name = "pytest"
-version = "7.4.4"
+version = "8.4.2"
description = "pytest: simple powerful testing with Python"
optional = false
-python-versions = ">=3.7"
-groups = ["dev"]
+python-versions = ">=3.9"
+groups = ["main", "dev"]
files = [
- {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"},
- {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"},
+ {file = "pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79"},
+ {file = "pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01"},
]
[package.dependencies]
-colorama = {version = "*", markers = "sys_platform == \"win32\""}
-iniconfig = "*"
-packaging = "*"
-pluggy = ">=0.12,<2.0"
+colorama = {version = ">=0.4", markers = "sys_platform == \"win32\""}
+iniconfig = ">=1"
+packaging = ">=20"
+pluggy = ">=1.5,<2"
+pygments = ">=2.7.2"
[package.extras]
-testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"]
+dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "requests", "setuptools", "xmlschema"]
[[package]]
name = "pytest-asyncio"
@@ -5690,4 +5691,4 @@ propcache = ">=0.2.1"
[metadata]
lock-version = "2.1"
python-versions = ">=3.11,<3.13"
-content-hash = "481c8b5a48b2d2b3a0e94c4078b7998597f787294de986eca9bed0ece0778e9e"
+content-hash = "ad8083b0650095db07fe8df95331cbf55aa672c6d72657f6bfcf51d084fcffda"
diff --git a/flowsint-core/pyproject.toml b/flowsint-core/pyproject.toml
index 10662a42..07b726ca 100644
--- a/flowsint-core/pyproject.toml
+++ b/flowsint-core/pyproject.toml
@@ -25,9 +25,9 @@ alembic = "1.13.0"
phonenumbers = "^9.0.8"
python-multipart = "^0.0.20"
docker = "^7.1.0"
+pytest = "^8.4.2"
[tool.poetry.group.dev.dependencies]
-pytest = "^7.4"
pytest-asyncio = "^0.21"
black = "^23.0"
isort = "^5.12"
diff --git a/flowsint-core/src/flowsint_core/core/enums.py b/flowsint-core/src/flowsint_core/core/enums.py
index 49471c73..a5c395e7 100644
--- a/flowsint-core/src/flowsint_core/core/enums.py
+++ b/flowsint-core/src/flowsint_core/core/enums.py
@@ -8,7 +8,7 @@ class EventLevel(str, Enum):
FAILED = "FAILED"
SUCCESS = "SUCCESS"
DEBUG = "DEBUG"
- # Scanner-specific statuses
+ # Transform-specific statuses
PENDING = "PENDING"
RUNNING = "RUNNING"
COMPLETED = "COMPLETED"
diff --git a/flowsint-core/src/flowsint_core/core/orchestrator.py b/flowsint-core/src/flowsint_core/core/orchestrator.py
index 46081833..4797e0ac 100644
--- a/flowsint-core/src/flowsint_core/core/orchestrator.py
+++ b/flowsint-core/src/flowsint_core/core/orchestrator.py
@@ -2,7 +2,7 @@ from typing import List, Dict, Any
from datetime import datetime
import time
from pydantic import ValidationError
-from .scanner_base import Scanner
+from .transform_base import Transform
from .registry import TransformRegistry
from .types import FlowBranch, FlowStep
from .logger import Logger
@@ -12,9 +12,9 @@ import json
import os
-class FlowOrchestrator(Scanner):
+class FlowOrchestrator(Transform):
"""
- Orchestrator for running a list of scanners.
+ Orchestrator for running a list of transforms.
"""
def __init__(
@@ -27,10 +27,10 @@ class FlowOrchestrator(Scanner):
):
super().__init__(sketch_id, scan_id, neo4j_conn=neo4j_conn, vault=vault)
self.transform_branches = transform_branches
- self.scanners = {} # Map of nodeId -> scanner instance
+ self.transforms = {} # Map of nodeId -> transform instance
self.execution_log_file = None # Path to the execution log file
self._create_execution_log()
- self._load_scanners()
+ self._load_transforms()
def _create_execution_log(self) -> None:
"""
@@ -210,44 +210,44 @@ class FlowOrchestrator(Scanner):
{"message": f"Failed to save transform branches: {str(e)}"},
)
- def _load_scanners(self) -> None:
+ def _load_transforms(self) -> None:
if not self.transform_branches:
raise ValueError("No transform branches provided")
- # Collect all scanner nodes across all branches
- scanner_nodes = []
+ # Collect all transform nodes across all branches
+ transform_nodes = []
for branch in self.transform_branches:
for step in branch.steps:
if step.type == "type":
continue
- scanner_nodes.append(step)
+ transform_nodes.append(step)
- if not scanner_nodes:
- raise ValueError("No scanner nodes found in transform branches")
+ if not transform_nodes:
+ raise ValueError("No transform nodes found in transform branches")
- # Create scanner instances for each node
- for node in scanner_nodes:
+ # Create transform instances for each node
+ for node in transform_nodes:
node_id = node.nodeId
- # Extract scanner name from nodeId (assuming format like "transform_name-1234567890")
+ # Extract transform name from nodeId (assuming format like "transform_name-1234567890")
transform_name = node_id.split("-")[0]
if not TransformRegistry.transform_exists(transform_name):
- raise ValueError(f"Scanner '{transform_name}' not found in registry")
+ raise ValueError(f"Transform '{transform_name}' not found in registry")
- # Pass the step params to the scanner instance
- scanner_params = (
+ # Pass the step params to the transform instance
+ transform_params = (
node.params if hasattr(node, "params") and node.params else {}
)
- scanner = TransformRegistry.get_scanner(
+ transform = TransformRegistry.get_transform(
transform_name,
self.sketch_id,
self.scan_id,
neo4j_conn=self.neo4j_conn,
vault=self.vault,
- params=scanner_params,
+ params=transform_params,
)
- self.scanners[node_id] = scanner
+ self.transforms[node_id] = transform
def resolve_reference(self, ref_value: str, results_mapping: Dict[str, Any]) -> Any:
"""
@@ -258,11 +258,11 @@ class FlowOrchestrator(Scanner):
return results_mapping[ref_value]
return None
- def prepare_scanner_inputs(
+ def prepare_transform_inputs(
self, step: FlowStep, results_mapping: Dict[str, Any], initial_values: List[str]
) -> List[Any]:
"""
- Prepare the inputs for a scanner based on the references and previous results.
+ Prepare the inputs for a transform based on the references and previous results.
Handles single references, lists, and direct values.
"""
inputs = {}
@@ -290,12 +290,12 @@ class FlowOrchestrator(Scanner):
# Si aucun input n'a été résolu, utiliser les valeurs initiales
if not inputs:
- scanner = self.scanners.get(step.nodeId)
- if scanner:
- primary_key = scanner.key()
+ transform = self.transforms.get(step.nodeId)
+ if transform:
+ primary_key = transform.key()
return {primary_key: initial_values}
else:
- scanner = self.scanners.get(step.nodeId)
+ transform = self.transforms.get(step.nodeId)
return inputs[input_key]
def update_results_mapping(
@@ -305,7 +305,7 @@ class FlowOrchestrator(Scanner):
results_mapping: Dict[str, Any],
) -> None:
"""
- Update the results mapping with new outputs from a scanner.
+ Update the results mapping with new outputs from a transform.
"""
for output_key, output_ref in step_outputs.items():
if output_key in outputs:
@@ -355,8 +355,8 @@ class FlowOrchestrator(Scanner):
# Global mapping of output references to actual values
results_mapping = {}
- # Cache for scanner results to avoid recomputation
- scanner_results_cache = {}
+ # Cache for transform results to avoid recomputation
+ transform_results_cache = {}
total_steps = sum(len(branch.steps) for branch in self.transform_branches)
completed_steps = 0
@@ -368,27 +368,27 @@ class FlowOrchestrator(Scanner):
branch_results = {"id": branch_id, "name": branch_name, "steps": []}
# Process each step in the branch
- scanner_inputs = values
+ transform_inputs = values
for step in branch.steps:
if step.type == "type":
continue
node_id = step.nodeId
- scanner = self.scanners.get(node_id)
+ transform = self.transforms.get(node_id)
- if not scanner:
+ if not transform:
Logger.error(
self.sketch_id,
- {"message": f"Scanner not found for node {node_id}"},
+ {"message": f"Transform not found for node {node_id}"},
)
continue
- transform_name = scanner.name()
+ transform_name = transform.name()
step_start_time = time.time()
step_result = {
"nodeId": node_id,
- "scanner": transform_name,
+ "transform": transform_name,
"status": "error", # Default to error, will update on success
}
@@ -399,7 +399,7 @@ class FlowOrchestrator(Scanner):
"branch_name": branch_name,
"node_id": node_id,
"transform_name": transform_name,
- "inputs": to_json_serializable(scanner_inputs),
+ "inputs": to_json_serializable(transform_inputs),
"outputs": None,
"status": "running",
"error": None,
@@ -410,7 +410,7 @@ class FlowOrchestrator(Scanner):
try:
# Update status for current step
completed_steps += 1
- if not scanner_inputs:
+ if not transform_inputs:
error_msg = "No inputs available"
step_result["error"] = error_msg
log_entry["status"] = "error"
@@ -422,20 +422,20 @@ class FlowOrchestrator(Scanner):
branch_results["steps"].append(step_result)
continue
- # Check if we already have results for this scanner with these inputs
- cache_key = f"{node_id}:{str(scanner_inputs)}"
- if cache_key in scanner_results_cache:
- outputs = scanner_results_cache[cache_key]
+ # Check if we already have results for this transform with these inputs
+ cache_key = f"{node_id}:{str(transform_inputs)}"
+ if cache_key in transform_results_cache:
+ outputs = transform_results_cache[cache_key]
log_entry["cache_hit"] = True
else:
- # Execute the scanner
- outputs = await scanner.execute(scanner_inputs)
+ # Execute the transform
+ outputs = await transform.execute(transform_inputs)
if not isinstance(outputs, (dict, list)):
raise ValueError(
- f"Scanner '{transform_name}' returned unsupported output format"
+ f"Transform '{transform_name}' returned unsupported output format"
)
# Cache the results
- scanner_results_cache[cache_key] = outputs
+ transform_results_cache[cache_key] = outputs
log_entry["cache_hit"] = False
# Store the outputs in the step result (serialize to avoid JSON issues)
@@ -453,7 +453,7 @@ class FlowOrchestrator(Scanner):
self.update_results_mapping(outputs, step.outputs, results_mapping)
# Also store the raw outputs in the main results
results["results"][node_id] = outputs
- scanner_inputs = outputs
+ transform_inputs = outputs
except ValidationError as e:
error_msg = f"Validation error: {str(e)}"
diff --git a/flowsint-core/src/flowsint_core/core/registry.py b/flowsint-core/src/flowsint_core/core/registry.py
index 1d92f87a..bb09a4a8 100644
--- a/flowsint-core/src/flowsint_core/core/registry.py
+++ b/flowsint-core/src/flowsint_core/core/registry.py
@@ -1,117 +1,117 @@
import inspect
from typing import Dict, Optional, Type, List
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
-# Domain-related scanners
-from flowsint_transforms.domain.to_subdomains import SubdomainScanner
-from flowsint_transforms.domain.to_whois import WhoisScanner
-from flowsint_transforms.domain.to_ip import ResolveScanner
-from flowsint_transforms.domain.to_website import DomainToWebsiteScanner
+# Domain-related transforms
+from flowsint_transforms.domain.to_subdomains import SubdomainTransform
+from flowsint_transforms.domain.to_whois import WhoisTransform
+from flowsint_transforms.domain.to_ip import ResolveTransform
+from flowsint_transforms.domain.to_website import DomainToWebsiteTransform
from flowsint_transforms.domain.to_root_domain import DomainToRootDomain
-from flowsint_transforms.domain.to_asn import DomainToAsnScanner
-from flowsint_transforms.domain.to_history import DomainToHistoryScanner
+from flowsint_transforms.domain.to_asn import DomainToAsnTransform
+from flowsint_transforms.domain.to_history import DomainToHistoryTransform
-# IP-related scanners
-from flowsint_transforms.email.to_domains import EmailToDomainsScanner
-from flowsint_transforms.individual.to_domains import IndividualToDomainsScanner
-from flowsint_transforms.ip.to_domain import ReverseResolveScanner
-from flowsint_transforms.ip.to_infos import IpToInfosScanner
-from flowsint_transforms.ip.to_asn import IpToAsnScanner
+# IP-related transforms
+from flowsint_transforms.email.to_domains import EmailToDomainsTransform
+from flowsint_transforms.individual.to_domains import IndividualToDomainsTransform
+from flowsint_transforms.ip.to_domain import ReverseResolveTransform
+from flowsint_transforms.ip.to_infos import IpToInfosTransform
+from flowsint_transforms.ip.to_asn import IpToAsnTransform
-# ASN-related scanners
-from flowsint_transforms.asn.to_cidrs import AsnToCidrsScanner
+# ASN-related transforms
+from flowsint_transforms.asn.to_cidrs import AsnToCidrsTransform
-# CIDR-related scanners
-from flowsint_transforms.cidr.to_ips import CidrToIpsScanner
+# CIDR-related transforms
+from flowsint_transforms.cidr.to_ips import CidrToIpsTransform
-# Social media scanners
-from flowsint_transforms.organization.to_domains import OrgToDomainsScanner
-from flowsint_transforms.social.to_maigret import MaigretScanner
+# Social media transforms
+from flowsint_transforms.organization.to_domains import OrgToDomainsTransform
+from flowsint_transforms.social.to_maigret import MaigretTransform
-# Organization-related scanners
-from flowsint_transforms.organization.to_asn import OrgToAsnScanner
-from flowsint_transforms.organization.to_infos import OrgToInfosScanner
+# Organization-related transforms
+from flowsint_transforms.organization.to_asn import OrgToAsnTransform
+from flowsint_transforms.organization.to_infos import OrgToInfosTransform
-# Cryptocurrency scanners
+# Cryptocurrency transforms
from flowsint_transforms.crypto.to_transactions import (
CryptoWalletAddressToTransactions,
)
from flowsint_transforms.crypto.to_nfts import CryptoWalletAddressToNFTs
-# Website-related scanners
+# Website-related transforms
from flowsint_transforms.website.to_crawler import WebsiteToCrawler
from flowsint_transforms.website.to_links import WebsiteToLinks
-from flowsint_transforms.website.to_domain import WebsiteToDomainScanner
+from flowsint_transforms.website.to_domain import WebsiteToDomainTransform
from flowsint_transforms.website.to_text import WebsiteToText
-from flowsint_transforms.website.to_webtrackers import WebsiteToWebtrackersScanner
+from flowsint_transforms.website.to_webtrackers import WebsiteToWebtrackersTransform
-# Email-related scanners
-from flowsint_transforms.email.to_gravatar import EmailToGravatarScanner
-from flowsint_transforms.email.to_leaks import EmailToBreachesScanner
+# Email-related transforms
+from flowsint_transforms.email.to_gravatar import EmailToGravatarTransform
+from flowsint_transforms.email.to_leaks import EmailToBreachesTransform
-# Phone-related scanners
-from flowsint_transforms.phone.to_leaks import PhoneToBreachesScanner
+# Phone-related transforms
+from flowsint_transforms.phone.to_leaks import PhoneToBreachesTransform
-# Individual-related scanners
-from flowsint_transforms.individual.to_org import IndividualToOrgScanner
+# Individual-related transforms
+from flowsint_transforms.individual.to_org import IndividualToOrgTransform
-# Integration scanners
+# Integration transforms
from flowsint_transforms.n8n.connector import N8nConnector
class TransformRegistry:
- _scanners: Dict[str, Type[Scanner]] = {}
+ _transforms: Dict[str, Type[Transform]] = {}
@classmethod
- def register(cls, scanner_class: Type[Scanner]) -> None:
- cls._scanners[scanner_class.name()] = scanner_class
+ def register(cls, transform_class: Type[Transform]) -> None:
+ cls._transforms[transform_class.name()] = transform_class
@classmethod
def transform_exists(cls, name: str) -> bool:
- return name in cls._scanners
+ return name in cls._transforms
@classmethod
- def get_scanner(cls, name: str, sketch_id: str, scan_id: str, **kwargs) -> Scanner:
- if name not in cls._scanners:
- raise Exception(f"Scanner '{name}' not found")
- return cls._scanners[name](sketch_id=sketch_id, scan_id=scan_id, **kwargs)
+ def get_transform(cls, name: str, sketch_id: str, scan_id: str, **kwargs) -> Transform:
+ if name not in cls._transforms:
+ raise Exception(f"Transform '{name}' not found")
+ return cls._transforms[name](sketch_id=sketch_id, scan_id=scan_id, **kwargs)
@classmethod
- def _create_scanner_metadata(cls, scanner: Type[Scanner]) -> Dict[str, str]:
- """Helper method to create scanner metadata dictionary."""
+ def _create_transform_metadata(cls, transform: Type[Transform]) -> Dict[str, str]:
+ """Helper method to create transform metadata dictionary."""
return {
- "class_name": scanner.__name__,
- "name": scanner.name(),
- "module": scanner.__module__,
- "description": scanner.__doc__,
- "documentation": inspect.cleandoc(scanner.documentation()),
- "category": scanner.category(),
- "inputs": scanner.input_schema(),
- "outputs": scanner.output_schema(),
+ "class_name": transform.__name__,
+ "name": transform.name(),
+ "module": transform.__module__,
+ "description": transform.__doc__,
+ "documentation": inspect.cleandoc(transform.documentation()),
+ "category": transform.category(),
+ "inputs": transform.input_schema(),
+ "outputs": transform.output_schema(),
"params": {},
- "params_schema": scanner.get_params_schema(),
- "required_params": scanner.required_params(),
- "icon": scanner.icon(),
+ "params_schema": transform.get_params_schema(),
+ "required_params": transform.required_params(),
+ "icon": transform.icon(),
}
@classmethod
def list(cls, exclude: Optional[List[str]] = []) -> Dict[str, Dict[str, str]]:
return {
- name: cls._create_scanner_metadata(scanner)
- for name, scanner in cls._scanners.items()
+ name: cls._create_transform_metadata(transform)
+ for name, transform in cls._transforms.items()
if name not in exclude
}
@classmethod
def list_by_categories(cls) -> Dict[str, List[Dict[str, str]]]:
- scanners_by_category = {}
- for _, scanner in cls._scanners.items():
- category = scanner.category()
- if category not in scanners_by_category:
- scanners_by_category[category] = []
- scanners_by_category[category].append(cls._create_scanner_metadata(scanner))
- return scanners_by_category
+ transforms_by_category = {}
+ for _, transform in cls._transforms.items():
+ category = transform.category()
+ if category not in transforms_by_category:
+ transforms_by_category[category] = []
+ transforms_by_category[category].append(cls._create_transform_metadata(transform))
+ return transforms_by_category
@classmethod
def list_by_input_type(
@@ -121,70 +121,70 @@ class TransformRegistry:
if input_type_lower == "any":
return [
- cls._create_scanner_metadata(scanner)
- for scanner in cls._scanners.values()
- if scanner.name() not in exclude
+ cls._create_transform_metadata(transform)
+ for transform in cls._transforms.values()
+ if transform.name() not in exclude
]
return [
- cls._create_scanner_metadata(scanner)
- for scanner in cls._scanners.values()
- if scanner.input_schema()["type"].lower() in ["any", input_type_lower]
- and scanner.name() not in exclude
+ cls._create_transform_metadata(transform)
+ for transform in cls._transforms.values()
+ if transform.input_schema()["type"].lower() in ["any", input_type_lower]
+ and transform.name() not in exclude
]
-# Register all scanners
+# Register all transforms
-# Domain-related scanners
-TransformRegistry.register(ReverseResolveScanner)
-TransformRegistry.register(ResolveScanner)
-TransformRegistry.register(SubdomainScanner)
-TransformRegistry.register(WhoisScanner)
-TransformRegistry.register(DomainToWebsiteScanner)
+# Domain-related transforms
+TransformRegistry.register(ReverseResolveTransform)
+TransformRegistry.register(ResolveTransform)
+TransformRegistry.register(SubdomainTransform)
+TransformRegistry.register(WhoisTransform)
+TransformRegistry.register(DomainToWebsiteTransform)
TransformRegistry.register(DomainToRootDomain)
-TransformRegistry.register(DomainToAsnScanner)
-TransformRegistry.register(DomainToHistoryScanner)
+TransformRegistry.register(DomainToAsnTransform)
+TransformRegistry.register(DomainToHistoryTransform)
-# IP-related scanners
-TransformRegistry.register(IpToInfosScanner)
-TransformRegistry.register(IpToAsnScanner)
+# IP-related transforms
+TransformRegistry.register(IpToInfosTransform)
+TransformRegistry.register(IpToAsnTransform)
-# ASN-related scanners
-TransformRegistry.register(AsnToCidrsScanner)
+# ASN-related transforms
+TransformRegistry.register(AsnToCidrsTransform)
-# CIDR-related scanners
-TransformRegistry.register(CidrToIpsScanner)
+# CIDR-related transforms
+TransformRegistry.register(CidrToIpsTransform)
-# Social media scanners
-TransformRegistry.register(MaigretScanner)
+# Social media transforms
+TransformRegistry.register(MaigretTransform)
-# Organization-related scanners
-TransformRegistry.register(OrgToAsnScanner)
-TransformRegistry.register(OrgToInfosScanner)
-TransformRegistry.register(OrgToDomainsScanner)
-# Cryptocurrency scanners
+# Organization-related transforms
+TransformRegistry.register(OrgToAsnTransform)
+TransformRegistry.register(OrgToInfosTransform)
+TransformRegistry.register(OrgToDomainsTransform)
+# Cryptocurrency transforms
TransformRegistry.register(CryptoWalletAddressToTransactions)
TransformRegistry.register(CryptoWalletAddressToNFTs)
-# Website-related scanners
+# Website-related transforms
TransformRegistry.register(WebsiteToCrawler)
TransformRegistry.register(WebsiteToLinks)
-TransformRegistry.register(WebsiteToDomainScanner)
-TransformRegistry.register(WebsiteToWebtrackersScanner)
+TransformRegistry.register(WebsiteToDomainTransform)
+TransformRegistry.register(WebsiteToWebtrackersTransform)
TransformRegistry.register(WebsiteToText)
-# Email-related scanners
-TransformRegistry.register(EmailToGravatarScanner)
-TransformRegistry.register(EmailToBreachesScanner)
-TransformRegistry.register(EmailToDomainsScanner)
+# Email-related transforms
+TransformRegistry.register(EmailToGravatarTransform)
+TransformRegistry.register(EmailToBreachesTransform)
+TransformRegistry.register(EmailToDomainsTransform)
-# Phone-related scanners
-TransformRegistry.register(PhoneToBreachesScanner)
+# Phone-related transforms
+TransformRegistry.register(PhoneToBreachesTransform)
-# Individual-related scanners
-TransformRegistry.register(IndividualToOrgScanner)
-TransformRegistry.register(IndividualToDomainsScanner)
+# Individual-related transforms
+TransformRegistry.register(IndividualToOrgTransform)
+TransformRegistry.register(IndividualToDomainsTransform)
-# Integration scanners
+# Integration transforms
TransformRegistry.register(N8nConnector)
diff --git a/flowsint-core/src/flowsint_core/core/scanner_base.py b/flowsint-core/src/flowsint_core/core/transform_base.py
similarity index 94%
rename from flowsint-core/src/flowsint_core/core/scanner_base.py
rename to flowsint-core/src/flowsint_core/core/transform_base.py
index c4bbf186..71f99237 100644
--- a/flowsint-core/src/flowsint_core/core/scanner_base.py
+++ b/flowsint-core/src/flowsint_core/core/transform_base.py
@@ -8,7 +8,7 @@ from .vault import VaultProtocol
from ..utils import resolve_type
-class InvalidScannerParams(Exception):
+class InvalidTransformParams(Exception):
pass
@@ -34,13 +34,13 @@ def build_params_model(params_schema: list) -> BaseModel:
return model
-class Scanner(ABC):
+class Transform(ABC):
"""
- Abstract base class for all scanners.
+ Abstract base class for all transforms.
## InputType and OutputType Pattern
- Scanners only need to define InputType and OutputType as class attributes.
+ Transforms only need to define InputType and OutputType as class attributes.
The base class automatically handles schema generation:
```python
@@ -48,14 +48,14 @@ class Scanner(ABC):
from flowsint_types import Domain
from flowsint_types import Ip
- class MyScanner(Scanner):
+ class MyTransform(Transform):
# Define types as class attributes
InputType = List[Domain]
OutputType = List[Ip]
@classmethod
def name(cls):
- return "my_scanner"
+ return "my_transform"
@classmethod
def category(cls):
@@ -76,15 +76,15 @@ class Scanner(ABC):
return results
# Make types available at module level for easy access
- InputType = MyScanner.InputType
- OutputType = MyScanner.OutputType
+ InputType = MyTransform.InputType
+ OutputType = MyTransform.OutputType
```
The base class automatically provides:
- input_schema() method using InputType
- output_schema() method using OutputType
- Error handling for missing type definitions
- - Consistent schema generation across all scanners
+ - Consistent schema generation across all transforms
Subclasses can override input_schema() or output_schema() if needed for special cases.
"""
@@ -128,8 +128,8 @@ class Scanner(ABC):
validated = self.ParamsModel(**resolved_params)
self.params = validated.model_dump()
except ValidationError as e:
- raise InvalidScannerParams(
- f"Scanner '{self.name()}' received invalid parameters: {e}"
+ raise InvalidTransformParams(
+ f"Transform '{self.name()}' received invalid parameters: {e}"
)
def resolve_params(self) -> Dict[str, Any]:
@@ -192,13 +192,13 @@ class Scanner(ABC):
@classmethod
@abstractmethod
def key(cls) -> str:
- """Primary key on which the scanner operates (e.g. domain, IP, etc.)"""
+ """Primary key on which the transform operates (e.g. domain, IP, etc.)"""
pass
@classmethod
def documentation(cls) -> str:
"""
- Return formatted markdown documentation for this scanner.
+ Return formatted markdown documentation for this transform.
Override this method to provide custom documentation.
Falls back to cleaned docstring if not overridden.
"""
@@ -327,7 +327,7 @@ class Scanner(ABC):
async def execute(self, values: List[str]) -> List[Dict[str, Any]]:
if self.name() != "transform_orchestrator":
- Logger.info(self.sketch_id, {"message": f"Scanner {self.name()} started."})
+ Logger.info(self.sketch_id, {"message": f"Transform {self.name()} started."})
try:
await self.async_init()
preprocessed = self.preprocess(values)
@@ -336,7 +336,7 @@ class Scanner(ABC):
if self.name() != "transform_orchestrator":
Logger.completed(
- self.sketch_id, {"message": f"Scanner {self.name()} finished."}
+ self.sketch_id, {"message": f"Transform {self.name()} finished."}
)
return processed
@@ -345,7 +345,7 @@ class Scanner(ABC):
if self.name() != "transform_orchestrator":
Logger.error(
self.sketch_id,
- {"message": f"Scanner {self.name()} errored: '{str(e)}'."},
+ {"message": f"Transform {self.name()} errored: '{str(e)}'."},
)
return []
@@ -373,7 +373,6 @@ class Scanner(ABC):
final_properties["type"] = node_type.lower()
final_properties["sketch_id"] = self.sketch_id
final_properties["label"] = final_properties.get("label", key_value)
-
set_clauses = [f"n.{prop} = ${prop}" for prop in final_properties.keys()]
params = {key_prop: key_value, **final_properties}
diff --git a/flowsint-core/src/flowsint_core/core/types.py b/flowsint-core/src/flowsint_core/core/types.py
index c32c0fe7..1f2d0edd 100644
--- a/flowsint-core/src/flowsint_core/core/types.py
+++ b/flowsint-core/src/flowsint_core/core/types.py
@@ -56,9 +56,9 @@ class FlowStep(BaseModel):
params: Optional[Dict[str, Any]] = Field(
None, description="Parameters for the step", title="Parameters"
)
- type: Literal["type", "scanner"] = Field(
+ type: Literal["type", "transform"] = Field(
...,
- description="Type of step - either type transformation or scanner",
+ description="Type of step - either type transformation or transform",
title="Step Type",
)
inputs: Dict[str, Any] = Field(
diff --git a/flowsint-core/src/flowsint_core/core/vault.py b/flowsint-core/src/flowsint_core/core/vault.py
index 10f51e91..2d96f6ce 100644
--- a/flowsint-core/src/flowsint_core/core/vault.py
+++ b/flowsint-core/src/flowsint_core/core/vault.py
@@ -15,6 +15,17 @@ class Vault(VaultProtocol):
raise ValueError("owner_id is required to use the vault.")
self.db = db
self.owner_id = owner_id
+
+ def set_secret(self, vault_ref, plain_key) -> None:
+ try:
+ ref_uuid = uuid.UUID(vault_ref)
+ stmt = select(Key).where(Key.id == ref_uuid)
+ except ValueError:
+ stmt = select(Key).where(Key.name == vault_ref)
+ stmt = stmt.where(Key.owner_id == self.owner_id)
+ result = self.db.execute(stmt)
+ row = result.scalars().first()
+ return row.encrypted_key if row else None
def get_secret(self, vault_ref: str) -> Optional[str]:
try:
diff --git a/flowsint-core/src/flowsint_core/tasks/flow.py b/flowsint-core/src/flowsint_core/tasks/flow.py
index f4bd6d76..0b718f22 100644
--- a/flowsint-core/src/flowsint_core/tasks/flow.py
+++ b/flowsint-core/src/flowsint_core/tasks/flow.py
@@ -62,7 +62,7 @@ def run_flow(
)
transform_branches = [FlowBranch(**branch) for branch in transform_branches]
- scanner = FlowOrchestrator(
+ transform = FlowOrchestrator(
sketch_id=sketch_id,
scan_id=str(scan_id),
transform_branches=transform_branches,
@@ -71,7 +71,7 @@ def run_flow(
)
# Use the synchronous scan method which internally handles the async operations
- results = scanner.scan(values=values)
+ results = transform.scan(values=values)
scan.status = EventLevel.COMPLETED
scan.results = to_json_serializable(results)
diff --git a/flowsint-core/src/flowsint_core/tasks/transform.py b/flowsint-core/src/flowsint_core/tasks/transform.py
index ade1241f..4a714bb6 100644
--- a/flowsint-core/src/flowsint_core/tasks/transform.py
+++ b/flowsint-core/src/flowsint_core/tasks/transform.py
@@ -60,9 +60,9 @@ def run_transform(
)
if not TransformRegistry.transform_exists(transform_name):
- raise ValueError(f"Scanner '{transform_name}' not found in registry")
+ raise ValueError(f"Transform '{transform_name}' not found in registry")
- scanner = TransformRegistry.get_scanner(
+ transform = TransformRegistry.get_transform(
name=transform_name,
sketch_id=sketch_id,
scan_id=scan_id,
@@ -70,7 +70,7 @@ def run_transform(
vault=vault,
)
- results = asyncio.run(scanner.execute(values=values))
+ results = asyncio.run(transform.execute(values=values))
scan.status = EventLevel.COMPLETED
scan.results = to_json_serializable(results)
diff --git a/flowsint-core/src/flowsint_core/tests/fixtures/database.py b/flowsint-core/src/flowsint_core/tests/fixtures/database.py
deleted file mode 100644
index dac66630..00000000
--- a/flowsint-core/src/flowsint_core/tests/fixtures/database.py
+++ /dev/null
@@ -1,132 +0,0 @@
-import pytest
-import uuid
-from unittest.mock import Mock
-from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker
-from app.models.models import Profile, Key
-from flowsint_core.core.vault import Vault
-
-
[email protected](scope="function")
-def test_db_session():
- """Create an in-memory SQLite database for testing"""
- # Create in-memory SQLite database
- engine = create_engine("sqlite:///:memory:", echo=False)
-
- # Only create specific tables we need for testing (to avoid ARRAY type issues)
- Profile.__table__.create(engine)
- Key.__table__.create(engine)
-
- # Create session
- TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
- session = TestingSessionLocal()
-
- yield session
-
- # Clean up
- session.close()
- engine.dispose()
-
-
[email protected](scope="function")
-def test_owner_id():
- """Generate a test owner ID"""
- return uuid.uuid4()
-
-
[email protected](scope="function")
-def test_profile(test_db_session, test_owner_id):
- """Create a test profile in the database"""
- profile = Profile(
- id=test_owner_id,
- email=f"test_{uuid.uuid4().hex[:8]}@example.com",
- hashed_password="hashed_test_password",
- is_active=True,
- first_name="Test",
- last_name="User",
- )
-
- test_db_session.add(profile)
- test_db_session.commit()
- test_db_session.refresh(profile)
-
- return profile
-
-
[email protected](scope="function")
-def test_api_key(test_db_session, test_owner_id):
- """Create a test API key in the database"""
- key = Key(
- id=uuid.uuid4(),
- name="test_etherscan_key",
- owner_id=test_owner_id,
- encrypted_key="test_api_key_12345",
- )
-
- test_db_session.add(key)
- test_db_session.commit()
- test_db_session.refresh(key)
-
- return key
-
-
[email protected](scope="function")
-def test_vault(test_db_session, test_owner_id, test_profile, test_api_key):
- """Create a test vault instance with test data"""
- return Vault(db=test_db_session, owner_id=test_owner_id)
-
-
[email protected](scope="function")
-def mock_logger():
- """Create a mock logger for testing"""
- logger = Mock()
- logger.info = Mock()
- logger.error = Mock()
- logger.warn = Mock()
- logger.debug = Mock()
- logger.success = Mock()
- return logger
-
-
[email protected](scope="function")
-def test_sketch_scan_ids():
- """Generate test sketch and scan IDs"""
- return {"sketch_id": str(uuid.uuid4()), "scan_id": str(uuid.uuid4())}
-
-
[email protected](scope="function")
-def multiple_test_keys(test_db_session, test_owner_id):
- """Create multiple test API keys for comprehensive testing"""
- keys = []
-
- # Create keys with different names
- key_configs = [
- {"name": "etherscan_api_key", "value": "test_etherscan_key_123"},
- {"name": "another_service_key", "value": "test_another_key_456"},
- {"name": "vaultKey1", "value": "test_vault_key_789"},
- ]
-
- for config in key_configs:
- key = Key(
- id=uuid.uuid4(),
- name=config["name"],
- owner_id=test_owner_id,
- encrypted_key=config["value"],
- )
- test_db_session.add(key)
- keys.append(key)
-
- test_db_session.commit()
-
- for key in keys:
- test_db_session.refresh(key)
-
- return keys
-
-
[email protected](scope="function")
-def test_vault_with_multiple_keys(
- test_db_session, test_owner_id, test_profile, multiple_test_keys
-):
- """Create a test vault with multiple keys for comprehensive testing"""
- return Vault(db=test_db_session, owner_id=test_owner_id)
diff --git a/flowsint-core/src/flowsint_core/tests/orchestrator.py b/flowsint-core/src/flowsint_core/tests/orchestrator.py
deleted file mode 100644
index 3c851684..00000000
--- a/flowsint-core/src/flowsint_core/tests/orchestrator.py
+++ /dev/null
@@ -1,111 +0,0 @@
-from flowsint_core.core.orchestrator import FlowOrchestrator
-from flowsint_types.domain import Domain, Domain
-from flowsint_types.whois import Whois
-
-import pytest
-
-
-def test_preprocess_valid_domains():
- scanner = FlowOrchestrator(
- "123", transform_names=["domain_resolve_scanner", "domain_whois_scanner"]
- )
- assert isinstance(scanner.scanners, list)
- assert len(scanner.scanners) == 2
-
-
-def test_preprocess_valid_domains():
- with pytest.raises(
- ValueError, match="Scanner 'this_scan_is_wrong' not found in registry"
- ):
- FlowOrchestrator(
- "123", transform_names=["domain_resolve_scanner", "this_scan_is_wrong"]
- )
-
-
-def test_execute_domain_subdomains_scanner():
- domains = ["example.com"]
- scanner = FlowOrchestrator("123", transform_names=["domain_subdomains_scanner"])
- results = scanner.execute(values=domains)
- assert results["initial_values"] == domains
- assert results["scanners"] == ["domain_subdomains_scanner"]
- assert isinstance(results["results"]["domain_subdomains_scanner"], list)
- assert isinstance(results["results"]["domain_subdomains_scanner"][0], Domain)
-
-
-def test_execute_domain_whois_scanner():
- domains = ["example.com"]
- scanner = FlowOrchestrator("123", transform_names=["domain_whois_scanner"])
- results = scanner.execute(values=domains)
- assert results["initial_values"] == domains
- assert results["scanners"] == ["domain_whois_scanner"]
- assert isinstance(results["results"]["domain_whois_scanner"], list)
- assert isinstance(results["results"]["domain_whois_scanner"][0], Whois)
-
-
-def test_execute_ip_resolve():
- ips = ["91.199.212.73"]
- scanner = FlowOrchestrator("123", transform_names=["ip_reverse_resolve_scanner"])
- results = scanner.execute(values=ips)
- assert results["initial_values"] == ips
- assert results["scanners"] == ["ip_reverse_resolve_scanner"]
- assert isinstance(results["results"]["ip_reverse_resolve_scanner"], list)
- if len(results["results"]["ip_reverse_resolve_scanner"]) != 0:
- assert isinstance(results["results"]["ip_reverse_resolve_scanner"][0], Domain)
-
-
-def test_execute_ip_resolve_and_whois():
- ips = ["91.199.212.73"]
- scanner = FlowOrchestrator(
- "123", transform_names=["ip_reverse_resolve_scanner", "domain_whois_scanner"]
- )
- results = scanner.execute(values=ips)
- assert results["initial_values"] == ips
- assert results["scanners"] == ["ip_reverse_resolve_scanner", "domain_whois_scanner"]
-
-
-def test_execute_ip_resolve_and_whois_multiple():
- ips = ["91.199.212.73", "76.76.21.21"]
- scanner = FlowOrchestrator(
- "123", transform_names=["ip_reverse_resolve_scanner", "domain_whois_scanner"]
- )
- results = scanner.execute(values=ips)
- assert results["initial_values"] == ips
- assert results["scanners"] == ["ip_reverse_resolve_scanner", "domain_whois_scanner"]
- print(results)
-
-
-def test_execute_ip_resolve_and_whois_multiple():
- ips = ["162.19.81.222"]
- scanner = FlowOrchestrator(
- "123",
- transform_names=[
- "ip_reverse_resolve_scanner",
- "domain_whois_scanner",
- "domain_subdomains_scanner",
- ],
- )
- results = scanner.execute(values=ips)
- assert results["initial_values"] == ips
- assert results["scanners"] == [
- "ip_reverse_resolve_scanner",
- "domain_whois_scanner",
- "domain_subdomains_scanner",
- ]
-
-
-def test_execute_domain_whois_and_subdomains():
- domains = ["alliage.io"]
- scanner = FlowOrchestrator(
- "123", transform_names=["domain_whois_scanner", "domain_subdomains_scanner"]
- )
- results = scanner.execute(values=domains)
- assert results["initial_values"] == domains
- assert results["scanners"] == ["domain_whois_scanner", "domain_subdomains_scanner"]
-
-
-def test_execute_domain_subdomains():
- domains = ["alliage.io"]
- scanner = FlowOrchestrator("123", transform_names=["domain_subdomains_scanner"])
- results = scanner.execute(values=domains)
- assert results["initial_values"] == domains
- assert results["scanners"] == ["domain_subdomains_scanner"]
diff --git a/flowsint-core/src/flowsint_core/tests/test_base_scanner.py b/flowsint-core/src/flowsint_core/tests/test_base_scanner.py
deleted file mode 100644
index 7d2078c3..00000000
--- a/flowsint-core/src/flowsint_core/tests/test_base_scanner.py
+++ /dev/null
@@ -1,203 +0,0 @@
-import pytest
-from typing import List, Dict, Any
-from ..core.scanner_base import Scanner
-from flowsint_types.domain import Domain
-from flowsint_types.ip import Ip
-
-
-class MockScanner(Scanner):
- """Mock scanner for testing base functionality"""
-
- # Define InputType and OutputType
- InputType = List[Domain]
- OutputType = List[Ip]
-
- @classmethod
- def name(cls) -> str:
- return "mock_scanner"
-
- @classmethod
- def category(cls) -> str:
- return "Test"
-
- @classmethod
- def key(cls) -> str:
- return "domain"
-
- @classmethod
- def input_schema(cls) -> Dict[str, Any]:
- return cls.generate_input_schema()
-
- @classmethod
- def output_schema(cls) -> Dict[str, Any]:
- return cls.generate_output_schema()
-
- async def scan(self, values: List[str]) -> List[Dict[str, Any]]:
- # Mock implementation
- return [{"address": "1.2.3.4"}]
-
-
-class IncompleteScanner(Scanner):
- """Scanner without InputType/OutputType for testing error cases"""
-
- @classmethod
- def name(cls) -> str:
- return "incomplete_scanner"
-
- @classmethod
- def category(cls) -> str:
- return "Test"
-
- @classmethod
- def key(cls) -> str:
- return "test"
-
- @classmethod
- def input_schema(cls) -> Dict[str, Any]:
- return cls.generate_input_schema()
-
- @classmethod
- def output_schema(cls) -> Dict[str, Any]:
- return cls.generate_output_schema()
-
- async def scan(self, values: List[str]) -> List[Dict[str, Any]]:
- return []
-
-
-class TestBaseScannerInputOutputTypes:
- """Test suite for Scanner InputType/OutputType functionality"""
-
- def test_input_type_output_type_class_attributes(self):
- """Test that InputType and OutputType are properly set as class attributes"""
- assert hasattr(MockScanner, "InputType")
- assert hasattr(MockScanner, "OutputType")
- assert MockScanner.InputType == List[Domain]
- assert MockScanner.OutputType == List[Ip]
-
- def test_generate_input_schema_success(self):
- """Test that generate_input_schema works correctly with valid InputType"""
- schema = MockScanner.generate_input_schema()
-
- assert isinstance(schema, dict)
- assert "type" in schema
- assert "properties" in schema
- assert schema["type"] == "Domain"
-
- # Check that properties are correctly extracted
- properties = schema["properties"]
- assert isinstance(properties, list)
-
- # Should have domain property
- domain_prop = next((p for p in properties if p["name"] == "domain"), None)
- assert domain_prop is not None
- assert domain_prop["type"] == "string"
-
- def test_generate_output_schema_success(self):
- """Test that generate_output_schema works correctly with valid OutputType"""
- schema = MockScanner.generate_output_schema()
-
- assert isinstance(schema, dict)
- assert "type" in schema
- assert "properties" in schema
- assert schema["type"] == "Ip"
-
- # Check that properties are correctly extracted
- properties = schema["properties"]
- assert isinstance(properties, list)
-
- # Should have address property
- address_prop = next((p for p in properties if p["name"] == "address"), None)
- assert address_prop is not None
- assert address_prop["type"] == "string"
-
- def test_generate_input_schema_not_implemented_error(self):
- """Test that generate_input_schema raises error when InputType is not defined"""
- with pytest.raises(NotImplementedError) as exc_info:
- IncompleteScanner.generate_input_schema()
-
- assert "InputType must be defined" in str(exc_info.value)
- assert "IncompleteScanner" in str(exc_info.value)
-
- def test_generate_output_schema_not_implemented_error(self):
- """Test that generate_output_schema raises error when OutputType is not defined"""
- with pytest.raises(NotImplementedError) as exc_info:
- IncompleteScanner.generate_output_schema()
-
- assert "OutputType must be defined" in str(exc_info.value)
- assert "IncompleteScanner" in str(exc_info.value)
-
- def test_input_output_schema_methods_use_generate_methods(self):
- """Test that the schema methods properly use the generate methods"""
- input_schema = MockScanner.input_schema()
- output_schema = MockScanner.output_schema()
-
- # These should be identical to calling generate methods directly
- assert input_schema == MockScanner.generate_input_schema()
- assert output_schema == MockScanner.generate_output_schema()
-
- def test_base_scanner_has_not_implemented_defaults(self):
- """Test that base Scanner class has NotImplemented defaults"""
- assert Scanner.InputType is NotImplemented
- assert Scanner.OutputType is NotImplemented
-
- def test_inheritance_preserves_input_output_types(self):
- """Test that InputType and OutputType are properly inherited"""
-
- class ChildScanner(MockScanner):
- pass
-
- # Child should inherit the types from MockScanner
- assert ChildScanner.InputType == List[Domain]
- assert ChildScanner.OutputType == List[Ip]
-
- # And schema generation should work
- input_schema = ChildScanner.generate_input_schema()
- output_schema = ChildScanner.generate_output_schema()
-
- assert input_schema["type"] == "Domain"
- assert output_schema["type"] == "Ip"
-
- def test_scanner_instance_can_access_class_types(self):
- """Test that scanner instances can access InputType and OutputType"""
- scanner = MockScanner("test_sketch", "test_scan")
-
- assert scanner.InputType == List[Domain]
- assert scanner.OutputType == List[Ip]
-
- # Instance should be able to call generate methods
- input_schema = scanner.generate_input_schema()
- output_schema = scanner.generate_output_schema()
-
- assert input_schema["type"] == "Domain"
- assert output_schema["type"] == "Ip"
-
-
-class TestScannerFunctionality:
- """Test other Scanner base functionality"""
-
- def test_scanner_initialization(self):
- """Test that Scanner initializes correctly"""
- scanner = MockScanner("test_sketch", "test_scan")
-
- assert scanner.sketch_id == "test_sketch"
- assert scanner.scan_id == "test_scan"
- assert scanner.params == {}
- assert scanner.params_schema == []
-
- def test_scanner_initialization_with_defaults(self):
- """Test Scanner initialization with default values"""
- scanner = MockScanner()
-
- assert scanner.sketch_id == "system"
- assert scanner.scan_id == "default"
-
- @pytest.mark.asyncio
- async def test_scanner_execute_method(self):
- """Test the execute method workflow"""
- scanner = MockScanner("test_sketch", "test_scan")
-
- result = await scanner.execute(["test.com"])
-
- assert isinstance(result, list)
- assert len(result) == 1
- assert result[0]["address"] == "1.2.3.4"
diff --git a/flowsint-core/src/flowsint_core/tests/test_crypto_with_vault.py b/flowsint-core/src/flowsint_core/tests/test_crypto_with_vault.py
deleted file mode 100644
index 03b5551d..00000000
--- a/flowsint-core/src/flowsint_core/tests/test_crypto_with_vault.py
+++ /dev/null
@@ -1,325 +0,0 @@
-import pytest
-import uuid
-from flowsint_transforms.crypto.wallet_to_transactions import (
- CryptoWalletAddressToTransactions,
-)
-from flowsint_transforms.crypto.wallet_to_nfts import CryptoWalletAddressToNFTs
-from ..core.vault import Vault
-from .fixtures.database import (
- test_db_session,
- test_owner_id,
- test_profile,
- test_api_key,
- test_vault,
- test_vault_with_multiple_keys,
- multiple_test_keys,
- test_sketch_scan_ids,
-)
-
-
-class TestVaultIntegration:
- """Test suite for Vault integration"""
-
- def test_vault_creation(self, test_vault, test_owner_id):
- """Test vault creation with valid owner ID"""
- assert test_vault.owner_id == test_owner_id
- assert test_vault.db is not None
-
- def test_vault_creation_without_owner_id(self, test_db_session):
- """Test vault creation fails without owner ID"""
- with pytest.raises(ValueError, match="owner_id is required to use the vault"):
- Vault(db=test_db_session, owner_id=None)
-
- def test_vault_get_secret_by_name(self, test_vault_with_multiple_keys):
- """Test retrieving secret by key name"""
- secret = test_vault_with_multiple_keys.get_secret("vaultKey1")
- assert secret == "test_vault_key_789"
-
- def test_vault_get_secret_by_uuid(
- self, test_vault_with_multiple_keys, multiple_test_keys
- ):
- """Test retrieving secret by key UUID"""
- # Get the UUID of the first key
- key_uuid = str(multiple_test_keys[0].id)
- secret = test_vault_with_multiple_keys.get_secret(key_uuid)
- assert secret == "test_etherscan_key_123"
-
- def test_vault_get_secret_nonexistent(self, test_vault):
- """Test retrieving non-existent secret returns None"""
- secret = test_vault.get_secret("nonexistent_key")
- assert secret is None
-
- def test_vault_get_secret_wrong_owner(self, test_db_session, test_api_key):
- """Test retrieving secret with wrong owner ID returns None"""
- wrong_owner_id = uuid.uuid4()
- vault = Vault(db=test_db_session, owner_id=wrong_owner_id)
- secret = vault.get_secret("test_etherscan_key")
- assert secret is None
-
-
-class TestCryptoWalletAddressToTransactions:
- """Test suite for CryptoWalletAddressToTransactions scanner with vault integration"""
-
- def test_scanner_creation_with_vault(self, test_vault, test_sketch_scan_ids):
- """Test scanner creation with vault"""
- scanner = CryptoWalletAddressToTransactions(
- sketch_id=test_sketch_scan_ids["sketch_id"],
- scan_id=test_sketch_scan_ids["scan_id"],
- vault=test_vault,
- params={
- "ETHERSCAN_API_KEY": "test_etherscan_key",
- "ETHERSCAN_API_URL": "https://api.etherscan.io/api",
- },
- )
-
- assert scanner.sketch_id == test_sketch_scan_ids["sketch_id"]
- assert scanner.scan_id == test_sketch_scan_ids["scan_id"]
- assert scanner.vault == test_vault
-
- def test_scanner_creation_without_vault(self, test_sketch_scan_ids):
- """Test scanner creation without vault"""
- scanner = CryptoWalletAddressToTransactions(
- sketch_id=test_sketch_scan_ids["sketch_id"],
- scan_id=test_sketch_scan_ids["scan_id"],
- params={
- "ETHERSCAN_API_KEY": "direct_api_key",
- "ETHERSCAN_API_URL": "https://api.etherscan.io/api",
- },
- )
-
- assert scanner.vault is None
-
- @pytest.mark.asyncio
- async def test_async_init_with_vault_key_exists(
- self, test_vault_with_multiple_keys, test_sketch_scan_ids
- ):
- """Test async_init when vault key exists"""
- scanner = CryptoWalletAddressToTransactions(
- sketch_id=test_sketch_scan_ids["sketch_id"],
- scan_id=test_sketch_scan_ids["scan_id"],
- vault=test_vault_with_multiple_keys,
- params={
- "ETHERSCAN_API_KEY": "vaultKey1", # This key exists in vault
- "ETHERSCAN_API_URL": "https://api.etherscan.io/api",
- },
- )
-
- # Get params before async_init
- params_before = scanner.get_params()
- assert params_before["ETHERSCAN_API_KEY"] == "vaultKey1"
-
- # Initialize scanner
- await scanner.async_init()
-
- # Get params after async_init - should be resolved from vault
- params_after = scanner.get_params()
- assert params_after["ETHERSCAN_API_KEY"] == "test_vault_key_789"
-
- @pytest.mark.asyncio
- async def test_async_init_with_vault_key_not_exists(
- self, test_vault, test_sketch_scan_ids
- ):
- """Test async_init when vault key doesn't exist"""
- scanner = CryptoWalletAddressToTransactions(
- sketch_id=test_sketch_scan_ids["sketch_id"],
- scan_id=test_sketch_scan_ids["scan_id"],
- vault=test_vault,
- params={
- "ETHERSCAN_API_KEY": "nonexistent_key", # This key doesn't exist
- "ETHERSCAN_API_URL": "https://api.etherscan.io/api",
- },
- )
-
- # Get params before async_init
- params_before = scanner.get_params()
- assert params_before["ETHERSCAN_API_KEY"] == "nonexistent_key"
-
- # Initialize scanner
- await scanner.async_init()
-
- # Get params after async_init - should remain unchanged since key not found
- params_after = scanner.get_params()
- assert params_after["ETHERSCAN_API_KEY"] == "nonexistent_key"
-
- @pytest.mark.asyncio
- async def test_async_init_without_vault(self, test_sketch_scan_ids):
- """Test async_init without vault"""
- scanner = CryptoWalletAddressToTransactions(
- sketch_id=test_sketch_scan_ids["sketch_id"],
- scan_id=test_sketch_scan_ids["scan_id"],
- params={
- "ETHERSCAN_API_KEY": "direct_api_key",
- "ETHERSCAN_API_URL": "https://api.etherscan.io/api",
- },
- )
-
- # Get params before async_init
- params_before = scanner.get_params()
- assert params_before["ETHERSCAN_API_KEY"] == "direct_api_key"
-
- # Initialize scanner
- await scanner.async_init()
-
- # Get params after async_init - should remain unchanged
- params_after = scanner.get_params()
- assert params_after["ETHERSCAN_API_KEY"] == "direct_api_key"
-
- def test_scanner_properties(self, test_vault, test_sketch_scan_ids):
- """Test scanner static properties"""
- assert CryptoWalletAddressToTransactions.name() == "wallet_to_transactions"
- assert CryptoWalletAddressToTransactions.category() == "CryptoWallet"
- assert isinstance(CryptoWalletAddressToTransactions.required_params(), bool)
-
-
-class TestCryptoWalletAddressToNFTs:
- """Test suite for CryptoWalletAddressToNFTs scanner with vault integration"""
-
- def test_scanner_creation_with_vault(self, test_vault, test_sketch_scan_ids):
- """Test scanner creation with vault"""
- scanner = CryptoWalletAddressToNFTs(
- sketch_id=test_sketch_scan_ids["sketch_id"],
- scan_id=test_sketch_scan_ids["scan_id"],
- vault=test_vault,
- params={
- "ETHERSCAN_API_KEY": "test_etherscan_key",
- "ETHERSCAN_API_URL": "https://api.etherscan.io/api",
- },
- )
-
- assert scanner.sketch_id == test_sketch_scan_ids["sketch_id"]
- assert scanner.scan_id == test_sketch_scan_ids["scan_id"]
- assert scanner.vault == test_vault
-
- @pytest.mark.asyncio
- async def test_async_init_with_vault_multiple_keys(
- self, test_vault_with_multiple_keys, test_sketch_scan_ids
- ):
- """Test async_init with multiple vault keys"""
- scanner = CryptoWalletAddressToNFTs(
- sketch_id=test_sketch_scan_ids["sketch_id"],
- scan_id=test_sketch_scan_ids["scan_id"],
- vault=test_vault_with_multiple_keys,
- params={
- "ETHERSCAN_API_KEY": "etherscan_api_key", # This key exists in vault
- "ETHERSCAN_API_URL": "https://api.etherscan.io/api",
- },
- )
-
- # Initialize scanner
- await scanner.async_init()
-
- # Get params after async_init - should be resolved from vault
- params_after = scanner.get_params()
- assert params_after["ETHERSCAN_API_KEY"] == "test_etherscan_key_123"
-
- def test_scanner_properties(self):
- """Test scanner static properties"""
- assert CryptoWalletAddressToNFTs.name() == "wallet_to_nfts"
- assert CryptoWalletAddressToNFTs.category() == "CryptoWallet"
- assert isinstance(CryptoWalletAddressToNFTs.required_params(), bool)
-
-
-class TestBothCryptoScannersIntegration:
- """Test suite for testing both crypto scanners together"""
-
- @pytest.mark.asyncio
- async def test_multiple_scanners_same_vault(
- self, test_vault_with_multiple_keys, test_sketch_scan_ids
- ):
- """Test multiple scanners using the same vault"""
- scanner1 = CryptoWalletAddressToTransactions(
- sketch_id=test_sketch_scan_ids["sketch_id"],
- scan_id=test_sketch_scan_ids["scan_id"],
- vault=test_vault_with_multiple_keys,
- params={
- "ETHERSCAN_API_KEY": "vaultKey1",
- "ETHERSCAN_API_URL": "https://api.etherscan.io/api",
- },
- )
-
- scanner2 = CryptoWalletAddressToNFTs(
- sketch_id=test_sketch_scan_ids["sketch_id"],
- scan_id=test_sketch_scan_ids["scan_id"],
- vault=test_vault_with_multiple_keys,
- params={
- "ETHERSCAN_API_KEY": "vaultKey1",
- "ETHERSCAN_API_URL": "https://api.etherscan.io/api",
- },
- )
-
- # Initialize both scanners
- await scanner1.async_init()
- await scanner2.async_init()
-
- # Both should have resolved the same key
- params1 = scanner1.get_params()
- params2 = scanner2.get_params()
-
- assert params1["ETHERSCAN_API_KEY"] == "test_vault_key_789"
- assert params2["ETHERSCAN_API_KEY"] == "test_vault_key_789"
-
- @pytest.mark.asyncio
- async def test_multiple_scanners_different_keys(
- self, test_vault_with_multiple_keys, test_sketch_scan_ids
- ):
- """Test multiple scanners using different vault keys"""
- scanner1 = CryptoWalletAddressToTransactions(
- sketch_id=test_sketch_scan_ids["sketch_id"],
- scan_id=test_sketch_scan_ids["scan_id"],
- vault=test_vault_with_multiple_keys,
- params={
- "ETHERSCAN_API_KEY": "vaultKey1",
- "ETHERSCAN_API_URL": "https://api.etherscan.io/api",
- },
- )
-
- scanner2 = CryptoWalletAddressToNFTs(
- sketch_id=test_sketch_scan_ids["sketch_id"],
- scan_id=test_sketch_scan_ids["scan_id"],
- vault=test_vault_with_multiple_keys,
- params={
- "ETHERSCAN_API_KEY": "etherscan_api_key",
- "ETHERSCAN_API_URL": "https://api.etherscan.io/api",
- },
- )
-
- # Initialize both scanners
- await scanner1.async_init()
- await scanner2.async_init()
-
- # Should have resolved different keys
- params1 = scanner1.get_params()
- params2 = scanner2.get_params()
-
- assert params1["ETHERSCAN_API_KEY"] == "test_vault_key_789"
- assert params2["ETHERSCAN_API_KEY"] == "test_etherscan_key_123"
-
- def test_scanner_schemas_consistency(self):
- """Test that both crypto scanners have consistent schemas"""
- transactions_schema = CryptoWalletAddressToTransactions.input_schema()
- nfts_schema = CryptoWalletAddressToNFTs.input_schema()
-
- # Both should accept the same input type (crypto wallet)
- assert transactions_schema["type"] == nfts_schema["type"]
-
- # Both should have param schemas with ETHERSCAN_API_KEY
- transactions_params = CryptoWalletAddressToTransactions.get_params_schema()
- nfts_params = CryptoWalletAddressToNFTs.get_params_schema()
-
- # Check that both have ETHERSCAN_API_KEY parameter
- transactions_api_key = next(
- (
- param
- for param in transactions_params
- if param["name"] == "ETHERSCAN_API_KEY"
- ),
- None,
- )
- nfts_api_key = next(
- (param for param in nfts_params if param["name"] == "ETHERSCAN_API_KEY"),
- None,
- )
-
- assert transactions_api_key is not None
- assert nfts_api_key is not None
- assert transactions_api_key["required"] == nfts_api_key["required"]
diff --git a/flowsint-core/src/flowsint_core/tests/test_input_output_migration.py b/flowsint-core/src/flowsint_core/tests/test_input_output_migration.py
deleted file mode 100644
index 62d223fd..00000000
--- a/flowsint-core/src/flowsint_core/tests/test_input_output_migration.py
+++ /dev/null
@@ -1,369 +0,0 @@
-"""
-Test migration from old TypeAlias pattern to new InputType/OutputType class attributes.
-
-This test demonstrates the benefits and proper usage of the new pattern.
-"""
-
-import pytest
-from typing import List, Dict, Any, TypeAlias
-from pydantic import TypeAdapter
-from ..core.scanner_base import Scanner
-from flowsint_types.domain import Domain
-from flowsint_types.ip import Ip
-from flowsint_types.email import Email
-from app.utils import resolve_type
-
-
-class OldPatternScanner(Scanner):
- """Example of old pattern using module-level TypeAlias"""
-
- @classmethod
- def name(cls) -> str:
- return "old_pattern_scanner"
-
- @classmethod
- def category(cls) -> str:
- return "Test"
-
- @classmethod
- def key(cls) -> str:
- return "domain"
-
- @classmethod
- def input_schema(cls) -> Dict[str, Any]:
- # Old pattern: manually defining TypeAlias and building schema
- InputType: TypeAlias = List[Domain]
- adapter = TypeAdapter(InputType)
- schema = adapter.json_schema()
- type_name, details = list(schema["$defs"].items())[0]
- return {
- "type": type_name,
- "properties": [
- {"name": prop, "type": resolve_type(info, schema)}
- for prop, info in details["properties"].items()
- ],
- }
-
- @classmethod
- def output_schema(cls) -> Dict[str, Any]:
- # Old pattern: manually defining TypeAlias and building schema
- OutputType: TypeAlias = List[Ip]
- adapter = TypeAdapter(OutputType)
- schema = adapter.json_schema()
- type_name, details = list(schema["$defs"].items())[0]
- return {
- "type": type_name,
- "properties": [
- {"name": prop, "type": resolve_type(info, schema)}
- for prop, info in details["properties"].items()
- ],
- }
-
- async def scan(self, values: List[str]) -> List[Dict[str, Any]]:
- return [{"address": "1.2.3.4"}]
-
-
-class NewPatternScanner(Scanner):
- """Example of new pattern using class attributes with automatic schema generation"""
-
- # New pattern: just define class attributes - base class handles the rest!
- InputType = List[Domain]
- OutputType = List[Ip]
-
- @classmethod
- def name(cls) -> str:
- return "new_pattern_scanner"
-
- @classmethod
- def category(cls) -> str:
- return "Test"
-
- @classmethod
- def key(cls) -> str:
- return "domain"
-
- # No need to implement input_schema() or output_schema() - base class does it automatically!
-
- # Methods can use InputType/OutputType directly (once made available at module level)
- def preprocess(self, data: List[Domain]) -> List[Domain]:
- # Using concrete type for clarity in test, but would use InputType in real implementation
- cleaned: List[Domain] = []
- for item in data:
- if isinstance(item, Domain):
- cleaned.append(item)
- return cleaned
-
- async def scan(self, values: List[Domain]) -> List[Ip]:
- # Using concrete type for clarity in test, but would use InputType/OutputType in real implementation
- results: List[Ip] = []
- for domain in values:
- results.append(Ip(address="1.2.3.4"))
- return results
-
-
-# Make types available at module level (this is what enables clean usage)
-NewPatternInputType = NewPatternScanner.InputType
-NewPatternOutputType = NewPatternScanner.OutputType
-
-
-class AdvancedNewPatternScanner(Scanner):
- """Example showing advanced usage with different types"""
-
- InputType = List[Email]
- OutputType = List[Domain]
-
- @classmethod
- def name(cls) -> str:
- return "advanced_pattern_scanner"
-
- @classmethod
- def category(cls) -> str:
- return "Test"
-
- @classmethod
- def key(cls) -> str:
- return "email"
-
- # Schema methods automatic!
-
- async def scan(self, values: List[str]) -> List[Dict[str, Any]]:
- return [{"domain": "example.com"}]
-
-
-class TestInputOutputMigrationPattern:
- """Test migration from old to new pattern"""
-
- def test_both_patterns_produce_same_schema(self):
- """Test that old and new patterns produce identical schemas"""
- old_input_schema = OldPatternScanner.input_schema()
- new_input_schema = NewPatternScanner.input_schema()
-
- old_output_schema = OldPatternScanner.output_schema()
- new_output_schema = NewPatternScanner.output_schema()
-
- # Schemas should be identical
- assert old_input_schema == new_input_schema
- assert old_output_schema == new_output_schema
-
- def test_new_pattern_benefits_code_reuse(self):
- """Test that new pattern reduces code duplication"""
- # With the new pattern, multiple scanners can easily reuse the same logic
-
- class Scanner1(Scanner):
- InputType = List[Domain]
- OutputType = List[Ip]
-
- @classmethod
- def name(cls):
- return "scanner1"
-
- @classmethod
- def category(cls):
- return "Test"
-
- @classmethod
- def key(cls):
- return "domain"
-
- # No need for input_schema() or output_schema() - automatic!
- async def scan(self, values):
- return []
-
- class Scanner2(Scanner):
- InputType = List[Domain]
- OutputType = List[Ip]
-
- @classmethod
- def name(cls):
- return "scanner2"
-
- @classmethod
- def category(cls):
- return "Test"
-
- @classmethod
- def key(cls):
- return "domain"
-
- # No need for input_schema() or output_schema() - automatic!
- async def scan(self, values):
- return []
-
- # Both should produce identical schemas with minimal code
- assert Scanner1.input_schema() == Scanner2.input_schema()
- assert Scanner1.output_schema() == Scanner2.output_schema()
-
- def test_new_pattern_type_introspection(self):
- """Test that new pattern allows for better type introspection"""
- # Can easily check what types a scanner uses
- assert NewPatternScanner.InputType == List[Domain]
- assert NewPatternScanner.OutputType == List[Ip]
-
- assert AdvancedNewPatternScanner.InputType == List[Email]
- assert AdvancedNewPatternScanner.OutputType == List[Domain]
-
- # This wasn't easily possible with the old pattern
-
- def test_new_pattern_inheritance_works(self):
- """Test that new pattern works well with inheritance"""
-
- class BaseDomainScanner(Scanner):
- InputType = List[Domain]
- OutputType = List[Ip]
-
- @classmethod
- def name(cls):
- return "base_domain"
-
- @classmethod
- def category(cls):
- return "Test"
-
- @classmethod
- def key(cls):
- return "domain"
-
- # Schema methods automatic!
- async def scan(self, values):
- return []
-
- class SpecializedDomainScanner(BaseDomainScanner):
- @classmethod
- def name(cls):
- return "specialized_domain"
-
- # Inherits InputType and OutputType
-
- # Child should inherit the types and schemas
- assert SpecializedDomainScanner.InputType == List[Domain]
- assert SpecializedDomainScanner.OutputType == List[Ip]
-
- specialized_input = SpecializedDomainScanner.input_schema()
- specialized_output = SpecializedDomainScanner.output_schema()
-
- assert specialized_input["type"] == "Domain"
- assert specialized_output["type"] == "Ip"
-
- def test_new_pattern_error_handling(self):
- """Test that new pattern provides better error handling"""
-
- class IncompleteDomainScanner(Scanner):
- # Forgot to define InputType and OutputType
- @classmethod
- def name(cls):
- return "incomplete"
-
- @classmethod
- def category(cls):
- return "Test"
-
- @classmethod
- def key(cls):
- return "domain"
-
- # Base class will try to generate schemas automatically and fail appropriately
- async def scan(self, values):
- return []
-
- # Should get clear error messages
- with pytest.raises(NotImplementedError) as exc_info:
- IncompleteDomainScanner.input_schema()
- assert "InputType must be defined" in str(exc_info.value)
- assert "IncompleteDomainScanner" in str(exc_info.value)
-
- with pytest.raises(NotImplementedError) as exc_info:
- IncompleteDomainScanner.output_schema()
- assert "OutputType must be defined" in str(exc_info.value)
- assert "IncompleteDomainScanner" in str(exc_info.value)
-
- def test_new_pattern_runtime_accessibility(self):
- """Test that types are accessible at runtime for dynamic operations"""
-
- # Can build registries or perform operations based on types
- scanners = [NewPatternScanner, AdvancedNewPatternScanner]
-
- domain_input_scanners = [
- scanner
- for scanner in scanners
- if hasattr(scanner, "InputType") and scanner.InputType == List[Domain]
- ]
-
- email_input_scanners = [
- scanner
- for scanner in scanners
- if hasattr(scanner, "InputType") and scanner.InputType == List[Email]
- ]
-
- assert len(domain_input_scanners) == 1
- assert domain_input_scanners[0] == NewPatternScanner
-
- assert len(email_input_scanners) == 1
- assert email_input_scanners[0] == AdvancedNewPatternScanner
-
- def test_new_pattern_with_clean_type_usage(self):
- """Test that the new pattern allows clean type usage without quotes"""
- scanner = NewPatternScanner("test", "test")
-
- # Test that we can create data of the expected types
- test_domains = [Domain(domain="example.com"), Domain(domain="test.com")]
-
- # Preprocess should work with clean type annotations
- result = scanner.preprocess(test_domains)
- assert len(result) == 2
- assert all(isinstance(d, Domain) for d in result)
-
- @pytest.mark.asyncio
- async def test_new_pattern_async_methods(self):
- """Test that async methods work correctly with clean type annotations"""
- scanner = NewPatternScanner("test", "test")
-
- test_domains = [Domain(domain="example.com")]
- result = await scanner.scan(test_domains)
-
- assert len(result) == 1
- assert isinstance(result[0], Ip)
- assert result[0].address == "1.2.3.4"
-
- def test_module_level_type_access(self):
- """Test that types are properly accessible at module level"""
- # These should be available after the class definition
- assert NewPatternInputType == List[Domain]
- assert NewPatternOutputType == List[Ip]
-
- # And they should match the class attributes
- assert NewPatternInputType == NewPatternScanner.InputType
- assert NewPatternOutputType == NewPatternScanner.OutputType
-
- def test_migration_checklist(self):
- """Test that demonstrates a complete migration checklist"""
-
- # Migration steps:
- # 1. Define InputType and OutputType as class attributes
- # 2. Remove input_schema() and output_schema() method implementations (base class handles automatically)
- # 3. Add module-level assignments: InputType = MyScanner.InputType (optional, for clean usage)
-
- # Verify the new pattern is simpler and more maintainable
- new_scanner_benefits = [
- "Just define InputType and OutputType class attributes",
- "Automatic schema generation by base class",
- "No boilerplate schema methods needed",
- "Consistent schema generation across all scanners",
- "Clean type usage throughout class methods",
- ]
-
- assert len(new_scanner_benefits) == 5
-
- # Verify functionality is preserved and automatic
- schema = NewPatternScanner.input_schema()
- assert schema["type"] == "Domain"
- assert any(prop["name"] == "domain" for prop in schema["properties"])
-
- # Verify schemas are generated automatically without manual implementation
- assert hasattr(NewPatternScanner, "input_schema")
- assert hasattr(NewPatternScanner, "output_schema")
-
- # The base class should be handling the schema generation
- input_schema = NewPatternScanner.input_schema()
- output_schema = NewPatternScanner.output_schema()
- assert input_schema is not None
- assert output_schema is not None
diff --git a/flowsint-core/src/flowsint_core/tests/test_registry.py b/flowsint-core/src/flowsint_core/tests/test_registry.py
deleted file mode 100644
index 66f0d529..00000000
--- a/flowsint-core/src/flowsint_core/tests/test_registry.py
+++ /dev/null
@@ -1,120 +0,0 @@
-import pytest
-from flowsint_core.core.registry import TransformRegistry
-from flowsint_core.core.scanner_base import Scanner
-from flowsint_types.domain import Domain
-from flowsint_types.ip import Ip
-from typing import List
-
-
-class MockScanner(Scanner):
- """Mock scanner for testing registry functionality"""
-
- InputType = List[Domain]
- OutputType = List[Ip]
-
- @classmethod
- def name(cls) -> str:
- return "mock_scanner"
-
- @classmethod
- def category(cls) -> str:
- return "Test"
-
- @classmethod
- def key(cls) -> str:
- return "domain"
-
- @classmethod
- def documentation(cls) -> str:
- return "Mock scanner for testing"
-
- @classmethod
- def get_params_schema(cls) -> List[dict]:
- return []
-
- @classmethod
- def required_params(cls) -> bool:
- return False
-
- @classmethod
- def icon(cls) -> str:
- return "test-icon"
-
- async def scan(self, data: InputType) -> OutputType:
- return []
-
-
-class TestTransformRegistry:
- """Test suite for TransformRegistry functionality"""
-
- def setup_method(self):
- """Clear registry before each test"""
- TransformRegistry.clear()
-
- def test_register_scanner(self):
- """Test registering a scanner"""
- TransformRegistry.register(MockScanner)
- assert TransformRegistry.transform_exists("mock_scanner")
-
- scanners = TransformRegistry.list()
- assert "mock_scanner" in scanners
- assert scanners["mock_scanner"]["class_name"] == "MockScanner"
-
- def test_register_invalid_scanner(self):
- """Test that registering a non-scanner class raises error"""
-
- class NotAScanner:
- pass
-
- with pytest.raises(ValueError, match="must inherit from Scanner"):
- TransformRegistry.register(NotAScanner)
-
- def test_get_scanner_instance(self):
- """Test getting a scanner instance"""
- TransformRegistry.register(MockScanner)
-
- scanner = TransformRegistry.get_scanner(
- "mock_scanner", sketch_id="test_sketch", scan_id="test_scan"
- )
-
- assert isinstance(scanner, MockScanner)
- assert scanner.sketch_id == "test_sketch"
- assert scanner.scan_id == "test_scan"
-
- def test_get_nonexistent_scanner(self):
- """Test that getting a non-existent scanner raises exception"""
- with pytest.raises(Exception, match="Scanner 'nonexistent' not found"):
- TransformRegistry.get_scanner(
- "nonexistent", sketch_id="test_sketch", scan_id="test_scan"
- )
-
- def test_list_by_categories(self):
- """Test listing scanners by category"""
- TransformRegistry.register(MockScanner)
-
- by_category = TransformRegistry.list_by_categories()
- assert "Test" in by_category
- assert len(by_category["Test"]) == 1
- assert by_category["Test"][0]["name"] == "mock_scanner"
-
- def test_list_by_input_type(self):
- """Test listing scanners by input type"""
- TransformRegistry.register(MockScanner)
-
- domain_scanners = TransformRegistry.list_by_input_type("Domain")
- assert len(domain_scanners) == 1
- assert domain_scanners[0]["name"] == "mock_scanner"
-
- # Test with "any" input type
- any_scanners = TransformRegistry.list_by_input_type("any")
- assert len(any_scanners) == 1
- assert any_scanners[0]["name"] == "mock_scanner"
-
- def test_register_module(self):
- """Test registering a module path"""
- TransformRegistry.register_module("test.module.path")
- assert "test.module.path" in TransformRegistry._scanner_modules
-
- # Registering the same module again should not duplicate
- TransformRegistry.register_module("test.module.path")
- assert TransformRegistry._scanner_modules.count("test.module.path") == 1
diff --git a/flowsint-core/src/flowsint_core/tests/test_schema_generation.py b/flowsint-core/src/flowsint_core/tests/test_schema_generation.py
deleted file mode 100644
index bb09fd8b..00000000
--- a/flowsint-core/src/flowsint_core/tests/test_schema_generation.py
+++ /dev/null
@@ -1,165 +0,0 @@
-"""
-Test schema generation for scanners with various InputType and OutputType combinations.
-"""
-
-from typing import List
-from flowsint_types.website import Website
-from flowsint_types.domain import Domain
-from flowsint_transforms.websites.to_domain import WebsiteToDomainScanner
-from flowsint_transforms.websites.to_webtrackers import WebsiteToWebtrackersScanner
-from flowsint_transforms.websites.to_crawler import WebsiteToCrawler
-from flowsint_transforms.domains.to_website import DomainToWebsiteScanner
-
-
-class TestSchemaGeneration:
- """Test that schema generation correctly identifies InputType and OutputType."""
-
- def test_website_to_domain_scanner_schemas(self):
- """Test that WebsiteToDomainScanner correctly shows Website as input type."""
- scanner = WebsiteToDomainScanner
-
- # Test InputType attribute
- assert scanner.InputType == List[Website]
-
- # Test input schema generation
- input_schema = scanner.input_schema()
- assert (
- input_schema["type"] == "Website"
- ), f"Expected 'Website', got '{input_schema['type']}'"
-
- # Test output schema generation
- output_schema = scanner.output_schema()
- assert (
- output_schema["type"] == "Domain"
- ), f"Expected 'Domain', got '{output_schema['type']}'"
-
- def test_website_to_webtrackers_scanner_schemas(self):
- """Test that WebsiteToWebtrackersScanner correctly shows Website as input type."""
- scanner = WebsiteToWebtrackersScanner
-
- # Test InputType attribute
- assert scanner.InputType == List[Website]
-
- # Test input schema generation
- input_schema = scanner.input_schema()
- assert (
- input_schema["type"] == "Website"
- ), f"Expected 'Website', got '{input_schema['type']}'"
-
- # Test output schema generation
- output_schema = scanner.output_schema()
- assert (
- output_schema["type"] == "WebTracker"
- ), f"Expected 'WebTracker', got '{output_schema['type']}'"
-
- def test_website_to_crawler_scanner_schemas(self):
- """Test that WebsiteToCrawler correctly shows Website as input type."""
- scanner = WebsiteToCrawler
-
- # Test InputType attribute
- assert scanner.InputType == List[Website]
-
- # Test input schema generation
- input_schema = scanner.input_schema()
- assert (
- input_schema["type"] == "Website"
- ), f"Expected 'Website', got '{input_schema['type']}'"
-
- def test_domain_to_website_scanner_schemas(self):
- """Test that DomainToWebsiteScanner correctly shows Domain as input and Website as output."""
- scanner = DomainToWebsiteScanner
-
- # Test InputType attribute
- assert scanner.InputType == List[Domain]
-
- # Test input schema generation
- input_schema = scanner.input_schema()
- assert (
- input_schema["type"] == "Domain"
- ), f"Expected 'Domain', got '{input_schema['type']}'"
-
- # Test output schema generation
- output_schema = scanner.output_schema()
- assert (
- output_schema["type"] == "Website"
- ), f"Expected 'Website', got '{output_schema['type']}'"
-
- def test_all_website_scanners_have_correct_input_types(self):
- """Test that all scanners taking Website input show Website in schema."""
- website_input_scanners = [
- (WebsiteToDomainScanner, "Website", "Domain"),
- (WebsiteToWebtrackersScanner, "Website", "WebTracker"),
- (WebsiteToCrawler, "Website", None), # Unknown output type
- ]
-
- for scanner_class, expected_input, expected_output in website_input_scanners:
- input_schema = scanner_class.input_schema()
- assert (
- input_schema["type"] == expected_input
- ), f"{scanner_class.__name__}: Expected input '{expected_input}', got '{input_schema['type']}'"
-
- if expected_output:
- output_schema = scanner_class.output_schema()
- assert (
- output_schema["type"] == expected_output
- ), f"{scanner_class.__name__}: Expected output '{expected_output}', got '{output_schema['type']}'"
-
- def test_schema_generation_debug_info(self):
- """Debug test to see what's actually in the schemas."""
- scanner = WebsiteToDomainScanner
-
- # Get the raw TypeAdapter schema
- from pydantic import TypeAdapter
-
- adapter = TypeAdapter(scanner.InputType)
- raw_schema = adapter.json_schema()
-
- print(f"\n=== Debug Info for {scanner.__name__} ===")
- print(f"InputType: {scanner.InputType}")
- print(f"Raw schema keys: {list(raw_schema.keys())}")
- if "$defs" in raw_schema:
- print(f"$defs keys: {list(raw_schema['$defs'].keys())}")
- print(f"Schema items: {raw_schema.get('items', 'No items')}")
- print(f"Generated input schema: {scanner.input_schema()}")
-
- # This test always passes, it's just for debugging
- assert True
-
- def test_schema_generation_follows_ref_not_first_def(self):
- """
- Regression test for the schema generation bug.
-
- Before the fix: generate_input_schema() picked the first definition in $defs
- (alphabetically "Domain" came before "Website"), so Website scanners incorrectly
- showed "Domain" as their input type.
-
- After the fix: generate_input_schema() follows the $ref in items to get the
- correct type name.
- """
- scanner = WebsiteToDomainScanner
-
- # Get the raw schema to understand the structure
- from pydantic import TypeAdapter
-
- adapter = TypeAdapter(scanner.InputType)
- raw_schema = adapter.json_schema()
-
- # Verify the raw schema structure that caused the bug
- assert "$defs" in raw_schema
- assert "Domain" in raw_schema["$defs"]
- assert "Website" in raw_schema["$defs"]
- assert raw_schema["items"]["$ref"] == "#/$defs/Website"
-
- # Before fix: list(raw_schema["$defs"].items())[0][0] would be "Domain" (first alphabetically)
- first_def_name = list(raw_schema["$defs"].items())[0][0]
- assert first_def_name == "Domain" # This would have been the bug
-
- # After fix: We follow the $ref to get "Website"
- ref_type = raw_schema["items"]["$ref"].split("/")[-1]
- assert ref_type == "Website" # This is what we should use
-
- # Verify our fix works correctly
- input_schema = scanner.input_schema()
- assert (
- input_schema["type"] == "Website"
- ), f"Schema generation should follow $ref, not pick first def. Got '{input_schema['type']}'"
diff --git a/flowsint-core/src/flowsint_core/tests/vault.py b/flowsint-core/src/flowsint_core/tests/vault.py
new file mode 100644
index 00000000..a105d90a
--- /dev/null
+++ b/flowsint-core/src/flowsint_core/tests/vault.py
@@ -0,0 +1,5 @@
+from flowsint_core.core.vault import Vault
+
+
+def test_description():
+ vault = Vault()
\ No newline at end of file
diff --git a/flowsint-core/src/flowsint_core/utils.py b/flowsint-core/src/flowsint_core/utils.py
index 5a6441ac..1a0bd16d 100644
--- a/flowsint-core/src/flowsint_core/utils.py
+++ b/flowsint-core/src/flowsint_core/utils.py
@@ -215,18 +215,18 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:
input_output = input_node["data"]["outputs"]
node_lookup = {node["id"]: node for node in nodes}
- scanners = []
+ transforms = []
for edge in edges:
target_id = edge["target"]
source_handle = edge["sourceHandle"]
target_handle = edge["targetHandle"]
- scanner_node = node_lookup.get(target_id)
- if scanner_node and scanner_node["data"]["type"] == "scanner":
- scanners.append(
+ transform_node = node_lookup.get(target_id)
+ if transform_node and transform_node["data"]["type"] == "transform":
+ transforms.append(
{
- "transform_name": scanner_node["data"]["name"],
- "module": scanner_node["data"]["module"],
+ "transform_name": transform_node["data"]["name"],
+ "module": transform_node["data"]["module"],
"input": source_handle,
"output": target_handle,
}
@@ -237,8 +237,8 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:
"name": input_node["data"]["name"],
"outputs": input_output,
},
- "scanners": scanners,
- "transform_names": [scanner["transform_name"] for scanner in scanners],
+ "transforms": transforms,
+ "transform_names": [transform["transform_name"] for transform in transforms],
}
diff --git a/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.py b/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.py
index b4fb0c21..724e174d 100644
--- a/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.py
+++ b/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.py
@@ -1,14 +1,14 @@
import json
import subprocess
from typing import List, Dict, Any, Union
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.cidr import CIDR
from flowsint_types.asn import ASN
from flowsint_core.utils import is_valid_asn, parse_asn
from flowsint_core.core.logger import Logger
-class AsnToCidrsScanner(Scanner):
+class AsnToCidrsTransform(Transform):
"""Takes an ASN and returns its corresponding CIDRs."""
# Define types as class attributes - base class handles schema generation automatically
@@ -202,5 +202,5 @@ class AsnToCidrsScanner(Scanner):
# Make types available at module level for easy access
-InputType = AsnToCidrsScanner.InputType
-OutputType = AsnToCidrsScanner.OutputType
+InputType = AsnToCidrsTransform.InputType
+OutputType = AsnToCidrsTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.py b/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.py
index 1192479f..1666b135 100644
--- a/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.py
+++ b/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.py
@@ -1,12 +1,12 @@
import subprocess
from typing import List, Union
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.cidr import CIDR
from flowsint_types.ip import Ip
from flowsint_core.core.logger import Logger
-class CidrToIpsScanner(Scanner):
+class CidrToIpsTransform(Transform):
"""Takes a CIDR and returns its corresponding IP addresses."""
# Define types as class attributes - base class handles schema generation automatically
@@ -120,5 +120,5 @@ class CidrToIpsScanner(Scanner):
# Make types available at module level for easy access
-InputType = CidrToIpsScanner.InputType
-OutputType = CidrToIpsScanner.OutputType
+InputType = CidrToIpsTransform.InputType
+OutputType = CidrToIpsTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py
index a3985ef1..de0d4c16 100644
--- a/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py
+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py
@@ -1,7 +1,7 @@
import os
from typing import List, Dict, Any, Optional, Union
import requests
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.wallet import CryptoWallet, CryptoNFT
from flowsint_core.core.logger import Logger
from flowsint_core.core.graph_db import Neo4jConnection
@@ -12,7 +12,7 @@ load_dotenv()
ETHERSCAN_API_URL = os.getenv("ETHERSCAN_API_URL")
-class CryptoWalletAddressToNFTs(Scanner):
+class CryptoWalletAddressToNFTs(Transform):
"""Resolve NFTs for a wallet address (ETH)."""
# Define types as class attributes - base class handles schema generation automatically
@@ -46,7 +46,7 @@ class CryptoWalletAddressToNFTs(Scanner):
@classmethod
def get_params_schema(cls) -> List[Dict[str, Any]]:
- """Declare required parameters for this scanner"""
+ """Declare required parameters for this transform"""
return [
{
"name": "ETHERSCAN_API_KEY",
diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py
index fc7b99e4..0b10dc6a 100644
--- a/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py
+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py
@@ -4,7 +4,7 @@ import requests
import requests.exceptions
from datetime import datetime
from dotenv import load_dotenv
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.wallet import CryptoWallet, CryptoWalletTransaction
from flowsint_core.core.graph_db import Neo4jConnection
from flowsint_core.core.logger import Logger
@@ -18,7 +18,7 @@ def wei_to_eth(wei_str):
return int(wei_str) / 10**18
-class CryptoWalletAddressToTransactions(Scanner):
+class CryptoWalletAddressToTransactions(Transform):
# Define types as class attributes - base class handles schema generation automatically
InputType = List[CryptoWallet]
@@ -51,7 +51,7 @@ class CryptoWalletAddressToTransactions(Scanner):
@classmethod
def get_params_schema(cls) -> List[Dict[str, Any]]:
- """Declare required parameters for this scanner"""
+ """Declare required parameters for this transform"""
return [
{
"name": "ETHERSCAN_API_KEY",
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_asn.py b/flowsint-transforms/src/flowsint_transforms/domain/to_asn.py
index 635ced2a..ec8684d4 100644
--- a/flowsint-transforms/src/flowsint_transforms/domain/to_asn.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_asn.py
@@ -2,14 +2,14 @@ import json
import socket
import subprocess
from typing import List, Union
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.domain import Domain
from flowsint_types.asn import ASN
from flowsint_core.utils import is_valid_domain
from flowsint_core.core.logger import Logger
-class DomainToAsnScanner(Scanner):
+class DomainToAsnTransform(Transform):
"""Takes a domain and returns its corresponding ASN."""
# Define types as class attributes - base class handles schema generation automatically
@@ -88,5 +88,5 @@ class DomainToAsnScanner(Scanner):
# Make types available at module level for easy access
-InputType = DomainToAsnScanner.InputType
-OutputType = DomainToAsnScanner.OutputType
+InputType = DomainToAsnTransform.InputType
+OutputType = DomainToAsnTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_history.py b/flowsint-transforms/src/flowsint_transforms/domain/to_history.py
index 49bacfc6..545421d3 100644
--- a/flowsint-transforms/src/flowsint_transforms/domain/to_history.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_history.py
@@ -2,7 +2,7 @@ import os
import re
import json
from typing import Any, List, Union, Dict, Set
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.domain import Domain
from flowsint_types.individual import Individual
from flowsint_types.organization import Organization
@@ -17,7 +17,7 @@ load_dotenv()
WHOXY_API_KEY = os.getenv("WHOXY_API_KEY")
-class DomainToHistoryScanner(Scanner):
+class DomainToHistoryTransform(Transform):
"""[WHOXY] Takes a domain and returns history infos about it (history, organization, owners, emails, etc.)."""
# Define types as class attributes - base class handles schema generation automatically
@@ -649,5 +649,5 @@ class DomainToHistoryScanner(Scanner):
return results
-InputType = DomainToHistoryScanner.InputType
-OutputType = DomainToHistoryScanner.OutputType
+InputType = DomainToHistoryTransform.InputType
+OutputType = DomainToHistoryTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py b/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py
index e5b5590a..8fd21493 100644
--- a/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py
@@ -1,12 +1,12 @@
import socket
from typing import List, Union
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.domain import Domain
from flowsint_types.ip import Ip
from flowsint_core.utils import is_valid_domain, is_root_domain
-class ResolveScanner(Scanner):
+class ResolveTransform(Transform):
"""Resolve domain names to IP addresses."""
# Define the input and output types as class attributes
@@ -27,15 +27,15 @@ class ResolveScanner(Scanner):
@classmethod
def documentation(cls) -> str:
- """Return formatted markdown documentation for the domain resolver scanner."""
+ """Return formatted markdown documentation for the domain resolver transform."""
return """
-# Domain Resolver Scanner
+# Domain Resolver Transform
-Resolve domain names to their corresponding IP addresses using DNS queries. This scanner performs forward DNS resolution to discover the IP addresses associated with domain names and subdomains.
+Resolve domain names to their corresponding IP addresses using DNS queries. This transform performs forward DNS resolution to discover the IP addresses associated with domain names and subdomains.
## Overview
-The Domain Resolver Scanner takes domain names as input and returns their resolved IP addresses. It automatically handles different input formats and validates domains before attempting resolution.
+The Domain Resolver Transform takes domain names as input and returns their resolved IP addresses. It automatically handles different input formats and validates domains before attempting resolution.
## Input/Output Types
@@ -44,7 +44,7 @@ The Domain Resolver Scanner takes domain names as input and returns their resolv
## Input Format Support
-The scanner accepts multiple input formats and automatically converts them:
+The transform accepts multiple input formats and automatically converts them:
### String Format
@@ -151,7 +151,7 @@ MERGE (d)-[:RESOLVES_TO {sketch_id: "sketch-uuid"}]->(ip)
## Domain Type Classification
-The scanner automatically classifies domains:
+The transform automatically classifies domains:
- **Root Domain**: `example.com` → `type: "domain"`
- **Subdomain**: `api.example.com` → `type: "subdomain"`
@@ -160,7 +160,7 @@ The scanner automatically classifies domains:
### Resolution Failures
-When DNS resolution fails, the scanner:
+When DNS resolution fails, the transform:
- Logs the error with domain name
- Continues processing remaining domains
@@ -174,7 +174,7 @@ Common resolution failures:
### Invalid Input Handling
-The scanner filters out:
+The transform filters out:
- Malformed domain names
- Empty strings
@@ -207,7 +207,7 @@ The scanner filters out:
```
[Domain Discovery] → [Domain Resolver] → [IP Geolocation]
- → [Port Scanner]
+ → [Port Transform]
→ [ASN Lookup]
```
@@ -234,7 +234,7 @@ The scanner filters out:
### Debug Information
-The scanner provides logging for:
+The transform provides logging for:
- Input validation results
- DNS resolution attempts
- Graph database operations
@@ -311,5 +311,5 @@ Check FlowSint logs for detailed resolution information.
# Make types available at module level for easy access
-InputType = ResolveScanner.InputType
-OutputType = ResolveScanner.OutputType
+InputType = ResolveTransform.InputType
+OutputType = ResolveTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py b/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py
index 30a0a98e..c5e99eee 100644
--- a/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py
@@ -1,11 +1,11 @@
from typing import List, Union
from flowsint_transforms.utils import is_valid_domain, get_root_domain
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.domain import Domain
from flowsint_core.core.logger import Logger
-class DomainToRootDomain(Scanner):
+class DomainToRootDomain(Transform):
"""Subdomain to root domain."""
InputType = List[Domain]
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py b/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py
index 55cd497b..3514e441 100644
--- a/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py
@@ -1,7 +1,7 @@
import shutil
import requests
from typing import List, Union
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.domain import Domain
from flowsint_core.utils import is_valid_domain
from flowsint_core.core.logger import Logger
@@ -9,8 +9,8 @@ from tools.network.subfinder import SubfinderTool
from flowsint_core.core.logger import Logger
-class SubdomainScanner(Scanner):
- """Scanner to find subdomains associated with a domain."""
+class SubdomainTransform(Transform):
+ """Transform to find subdomains associated with a domain."""
# Define types as class attributes - base class handles schema generation automatically
InputType = List[Domain | str]
@@ -144,5 +144,5 @@ class SubdomainScanner(Scanner):
return output
-InputType = SubdomainScanner.InputType
-OutputType = SubdomainScanner.OutputType
+InputType = SubdomainTransform.InputType
+OutputType = SubdomainTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_website.py b/flowsint-transforms/src/flowsint_transforms/domain/to_website.py
index 3194d4a6..d2c3a7fe 100644
--- a/flowsint-transforms/src/flowsint_transforms/domain/to_website.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_website.py
@@ -1,13 +1,13 @@
from typing import List, Union
import requests
from flowsint_core.utils import is_valid_domain
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.domain import Domain
from flowsint_types.website import Website
from flowsint_core.core.logger import Logger
-class DomainToWebsiteScanner(Scanner):
+class DomainToWebsiteTransform(Transform):
"""From domain to website."""
# Define types as class attributes - base class handles schema generation automatically
@@ -141,5 +141,5 @@ class DomainToWebsiteScanner(Scanner):
# Make types available at module level for easy access
-InputType = DomainToWebsiteScanner.InputType
-OutputType = DomainToWebsiteScanner.OutputType
+InputType = DomainToWebsiteTransform.InputType
+OutputType = DomainToWebsiteTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py b/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py
index 8671a101..77cfe3a0 100644
--- a/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py
@@ -1,7 +1,7 @@
from typing import List, Union
import whois
from flowsint_core.utils import is_valid_domain
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.domain import Domain, Domain
from flowsint_types.whois import Whois
from flowsint_types.email import Email
@@ -9,7 +9,7 @@ from flowsint_core.core.logger import Logger
from datetime import datetime
-class WhoisScanner(Scanner):
+class WhoisTransform(Transform):
"""Scan for WHOIS information of a domain."""
# Define types as class attributes - base class handles schema generation automatically
@@ -203,5 +203,5 @@ class WhoisScanner(Scanner):
# Make types available at module level for easy access
-InputType = WhoisScanner.InputType
-OutputType = WhoisScanner.OutputType
+InputType = WhoisTransform.InputType
+OutputType = WhoisTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/email/to_domains.py b/flowsint-transforms/src/flowsint_transforms/email/to_domains.py
index ff2d2883..a741a6dc 100644
--- a/flowsint-transforms/src/flowsint_transforms/email/to_domains.py
+++ b/flowsint-transforms/src/flowsint_transforms/email/to_domains.py
@@ -1,7 +1,7 @@
import os
import re
from typing import Any, List, Union, Dict, Set
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.domain import Domain
from flowsint_types.individual import Individual
from flowsint_types.email import Email
@@ -15,7 +15,7 @@ load_dotenv()
WHOXY_API_KEY = os.getenv("WHOXY_API_KEY")
-class EmailToDomainsScanner(Scanner):
+class EmailToDomainsTransform(Transform):
"""[WHOXY] Takes an email and returns the domains it registered."""
# Define types as class attributes - base class handles schema generation automatically
@@ -405,5 +405,5 @@ class EmailToDomainsScanner(Scanner):
)
-InputType = EmailToDomainsScanner.InputType
-OutputType = EmailToDomainsScanner.OutputType
+InputType = EmailToDomainsTransform.InputType
+OutputType = EmailToDomainsTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py b/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py
index 318f9918..b6864fd7 100644
--- a/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py
+++ b/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py
@@ -1,13 +1,13 @@
import hashlib
from typing import List, Union
import requests
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_core.core.logger import Logger
from flowsint_types.email import Email
from flowsint_types.gravatar import Gravatar
-class EmailToGravatarScanner(Scanner):
+class EmailToGravatarTransform(Transform):
"""From md5 hash of email to gravatar."""
InputType = List[Email]
@@ -129,5 +129,5 @@ class EmailToGravatarScanner(Scanner):
# Make types available at module level for easy access
-InputType = EmailToGravatarScanner.InputType
-OutputType = EmailToGravatarScanner.OutputType
+InputType = EmailToGravatarTransform.InputType
+OutputType = EmailToGravatarTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py
index 70149614..34b78faa 100644
--- a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py
+++ b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py
@@ -2,7 +2,7 @@ import os
from typing import Any, Dict, List, Optional, Union
import requests
from urllib.parse import urljoin
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_core.core.logger import Logger
from flowsint_types.email import Email
from flowsint_types.breach import Breach
@@ -15,7 +15,7 @@ load_dotenv()
HIBP_API_KEY = os.getenv("HIBP_API_KEY")
-class EmailToBreachesScanner(Scanner):
+class EmailToBreachesTransform(Transform):
"""[HIBPWNED] Get the breaches the email might be invovled in."""
InputType = List[Email]
@@ -56,7 +56,7 @@ class EmailToBreachesScanner(Scanner):
@classmethod
def get_params_schema(cls) -> List[Dict[str, Any]]:
- """Declare required parameters for this scanner"""
+ """Declare required parameters for this transform"""
return [
{
"name": "HIBP_API_KEY",
@@ -102,7 +102,7 @@ class EmailToBreachesScanner(Scanner):
Logger.error(
self.sketch_id, {"message": "Could not find HIBP_API_URL in params."}
)
- headers = {"hibp-api-key": api_key, "User-Agent": "FlowsInt-Scanner"}
+ headers = {"hibp-api-key": api_key, "User-Agent": "FlowsInt-Transform"}
Logger.info(self.sketch_id, {"message": f"HIBP API URL: {api_url}"})
for email in data:
try:
@@ -235,5 +235,5 @@ class EmailToBreachesScanner(Scanner):
# Make types available at module level for easy access
-InputType = EmailToBreachesScanner.InputType
-OutputType = EmailToBreachesScanner.OutputType
+InputType = EmailToBreachesTransform.InputType
+OutputType = EmailToBreachesTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py b/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py
index 5f2a2d2d..6d61b3bf 100644
--- a/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py
+++ b/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py
@@ -1,7 +1,7 @@
import os
import re
from typing import Any, List, Union, Dict, Set
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.domain import Domain
from flowsint_types.organization import Organization
from flowsint_types.individual import Individual
@@ -15,7 +15,7 @@ load_dotenv()
WHOXY_API_KEY = os.getenv("WHOXY_API_KEY")
-class IndividualToDomainsScanner(Scanner):
+class IndividualToDomainsTransform(Transform):
"""[WHOXY] Takes an individual and returns the domains it registered."""
# Define types as class attributes - base class handles schema generation automatically
@@ -418,5 +418,5 @@ class IndividualToDomainsScanner(Scanner):
# Make types available at module level for easy access
-InputType = IndividualToDomainsScanner.InputType
-OutputType = IndividualToDomainsScanner.OutputType
+InputType = IndividualToDomainsTransform.InputType
+OutputType = IndividualToDomainsTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/individual/to_org.py b/flowsint-transforms/src/flowsint_transforms/individual/to_org.py
index 3ab74da9..9d376c3a 100644
--- a/flowsint-transforms/src/flowsint_transforms/individual/to_org.py
+++ b/flowsint-transforms/src/flowsint_transforms/individual/to_org.py
@@ -1,13 +1,13 @@
import requests
from typing import List, Dict, Union, Set
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.organization import Organization
from flowsint_types.individual import Individual
from flowsint_core.core.logger import Logger
from tools.organizations.sirene import SireneTool
-class IndividualToOrgScanner(Scanner):
+class IndividualToOrgTransform(Transform):
"""[SIRENE] Find organization from a person with data from SIRENE (France only)."""
# Define types as class attributes - base class handles schema generation automatically
@@ -340,5 +340,5 @@ class IndividualToOrgScanner(Scanner):
# Make types available at module level for easy access
-InputType = IndividualToOrgScanner.InputType
-OutputType = IndividualToOrgScanner.OutputType
+InputType = IndividualToOrgTransform.InputType
+OutputType = IndividualToOrgTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/ip/to_asn.py b/flowsint-transforms/src/flowsint_transforms/ip/to_asn.py
index 75364cfe..63281d96 100644
--- a/flowsint-transforms/src/flowsint_transforms/ip/to_asn.py
+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_asn.py
@@ -2,14 +2,14 @@ import json
import socket
import subprocess
from typing import List, Union
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.ip import Ip
from flowsint_types.asn import ASN
from flowsint_core.utils import is_valid_ip
from flowsint_core.core.logger import Logger
-class IpToAsnScanner(Scanner):
+class IpToAsnTransform(Transform):
"""[ASNMAP] Takes an IP address and returns its corresponding ASN."""
# Define types as class attributes - base class handles schema generation automatically
@@ -18,7 +18,7 @@ class IpToAsnScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "ip_to_asn_scanner"
+ return "ip_to_asn_transform"
@classmethod
def category(cls) -> str:
@@ -108,5 +108,5 @@ class IpToAsnScanner(Scanner):
# Make types available at module level for easy access
-InputType = IpToAsnScanner.InputType
-OutputType = IpToAsnScanner.OutputType
+InputType = IpToAsnTransform.InputType
+OutputType = IpToAsnTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py b/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py
index fb4910d3..2cecd107 100644
--- a/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py
+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py
@@ -3,7 +3,7 @@ import socket
import requests
from typing import List, Union
from flowsint_core.core.logger import Logger
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.domain import Domain
from flowsint_types.ip import Ip
from flowsint_core.utils import is_valid_ip
@@ -11,7 +11,7 @@ from flowsint_core.utils import is_valid_ip
PTR_BLACKLIST = re.compile(r"^ip\d+\.ip-\d+-\d+-\d+-\d+\.")
-class ReverseResolveScanner(Scanner):
+class ReverseResolveTransform(Transform):
"""Resolve IP addresses to domain names using PTR, Certificate Transparency and optional API calls."""
# Define types as class attributes - base class handles schema generation automatically
@@ -120,5 +120,5 @@ class ReverseResolveScanner(Scanner):
# Make types available at module level for easy access
-InputType = ReverseResolveScanner.InputType
-OutputType = ReverseResolveScanner.OutputType
+InputType = ReverseResolveTransform.InputType
+OutputType = ReverseResolveTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/ip/to_infos.py b/flowsint-transforms/src/flowsint_transforms/ip/to_infos.py
index 6f563cea..c76270e0 100644
--- a/flowsint-transforms/src/flowsint_transforms/ip/to_infos.py
+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_infos.py
@@ -1,7 +1,7 @@
import requests
from typing import List, Dict, Any, TypeAlias, Union
from pydantic import TypeAdapter
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.ip import Ip, Ip
from flowsint_core.utils import resolve_type, is_valid_ip
@@ -9,7 +9,7 @@ InputType: TypeAlias = List[Ip]
OutputType: TypeAlias = List[Ip]
-class IpToInfosScanner(Scanner):
+class IpToInfosTransform(Transform):
"""[ip-api.com] Get information data for IP addresses."""
# Define types as class attributes - base class handles schema generation automatically
@@ -131,5 +131,5 @@ class IpToInfosScanner(Scanner):
return {}
-InputType = IpToInfosScanner.InputType
-OutputType = IpToInfosScanner.OutputType
+InputType = IpToInfosTransform.InputType
+OutputType = IpToInfosTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.py
index c5b310e3..55b0dba8 100644
--- a/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.py
+++ b/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.py
@@ -1,6 +1,6 @@
from typing import Dict, Any, List, Union
import hibpwned
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_core.core.logger import Logger
import os
from dotenv import load_dotenv
@@ -11,7 +11,7 @@ load_dotenv()
HIBP_API_KEY = os.getenv("HIBP_API_KEY")
-class HibpScanner(Scanner):
+class HibpTransform(Transform):
"""Queries HaveIBeenPwned for potential leaks."""
# Define types as class attributes - base class handles schema generation automatically
@@ -116,5 +116,5 @@ class HibpScanner(Scanner):
# Make types available at module level for easy access
-InputType = HibpScanner.InputType
-OutputType = HibpScanner.OutputType
+InputType = HibpTransform.InputType
+OutputType = HibpTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/n8n/connector.py b/flowsint-transforms/src/flowsint_transforms/n8n/connector.py
index 36089513..46d0cb87 100644
--- a/flowsint-transforms/src/flowsint_transforms/n8n/connector.py
+++ b/flowsint-transforms/src/flowsint_transforms/n8n/connector.py
@@ -1,12 +1,12 @@
import json
import aiohttp
from typing import List, Dict, Any, Optional
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_core.core.logger import Logger
from flowsint_core.core.graph_db import Neo4jConnection
-class N8nConnector(Scanner):
+class N8nConnector(Transform):
"""
Connect to your custom n8n workflows to process data through webhooks.
"""
diff --git a/flowsint-transforms/src/flowsint_transforms/organization/to_asn.py b/flowsint-transforms/src/flowsint_transforms/organization/to_asn.py
index c7ca4c62..6a368635 100644
--- a/flowsint-transforms/src/flowsint_transforms/organization/to_asn.py
+++ b/flowsint-transforms/src/flowsint_transforms/organization/to_asn.py
@@ -1,13 +1,13 @@
import json
import subprocess
from typing import List, Dict, Any, Union
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.organization import Organization
from flowsint_types.asn import ASN
from flowsint_core.core.logger import Logger
-class OrgToAsnScanner(Scanner):
+class OrgToAsnTransform(Transform):
"""Takes an organization and returns its corresponding ASN."""
# Define types as class attributes - base class handles schema generation automatically
@@ -159,5 +159,5 @@ class OrgToAsnScanner(Scanner):
# Make types available at module level for easy access
-InputType = OrgToAsnScanner.InputType
-OutputType = OrgToAsnScanner.OutputType
+InputType = OrgToAsnTransform.InputType
+OutputType = OrgToAsnTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py b/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py
index 609ee776..d4cd1b70 100644
--- a/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py
+++ b/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py
@@ -1,7 +1,7 @@
import os
import re
from typing import Any, List, Union, Dict, Set
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.domain import Domain
from flowsint_types.organization import Organization
from flowsint_types.individual import Individual
@@ -17,7 +17,7 @@ load_dotenv()
WHOXY_API_KEY = os.getenv("WHOXY_API_KEY")
-class OrgToDomainsScanner(Scanner):
+class OrgToDomainsTransform(Transform):
"""[WHOXY] Takes an organization and returns the domains it registered."""
# Define types as class attributes - base class handles schema generation automatically
@@ -406,5 +406,5 @@ class OrgToDomainsScanner(Scanner):
)
-InputType = OrgToDomainsScanner.InputType
-OutputType = OrgToDomainsScanner.OutputType
+InputType = OrgToDomainsTransform.InputType
+OutputType = OrgToDomainsTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/organization/to_infos.py b/flowsint-transforms/src/flowsint_transforms/organization/to_infos.py
index 24d13e9f..4d241a0a 100644
--- a/flowsint-transforms/src/flowsint_transforms/organization/to_infos.py
+++ b/flowsint-transforms/src/flowsint_transforms/organization/to_infos.py
@@ -1,11 +1,11 @@
from typing import List, Dict, Any, Union
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.organization import Organization
from flowsint_core.core.logger import Logger
from tools.organizations.sirene import SireneTool
-class OrgToInfosScanner(Scanner):
+class OrgToInfosTransform(Transform):
"""Enrich Organization with data from SIRENE (France only)."""
# Define types as class attributes - base class handles schema generation automatically
@@ -412,5 +412,5 @@ class OrgToInfosScanner(Scanner):
# Make types available at module level for easy access
-InputType = OrgToInfosScanner.InputType
-OutputType = OrgToInfosScanner.OutputType
+InputType = OrgToInfosTransform.InputType
+OutputType = OrgToInfosTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py b/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py
index cbcebeb5..a041ecbd 100644
--- a/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py
+++ b/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py
@@ -1,13 +1,13 @@
import asyncio
from typing import Dict, Any, List, Union
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_core.utils import is_valid_number
from flowsint_core.core.logger import Logger
from flowsint_types.phone import Phone
import httpx
-class IgnorantScanner(Scanner):
+class IgnorantTransform(Transform):
# Define types as class attributes - base class handles schema generation automatically
InputType = List[str] # Phone numbers as strings
@@ -120,5 +120,5 @@ class IgnorantScanner(Scanner):
# Make types available at module level for easy access
-InputType = IgnorantScanner.InputType
-OutputType = IgnorantScanner.OutputType
+InputType = IgnorantTransform.InputType
+OutputType = IgnorantTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.py
index a7dc6a89..e40cfe32 100644
--- a/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.py
+++ b/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.py
@@ -2,7 +2,7 @@ import os
from typing import Any, Dict, List, Optional, Union
import requests
from urllib.parse import urljoin
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_core.core.logger import Logger
from flowsint_types.phone import Phone
from flowsint_types.breach import Breach
@@ -15,7 +15,7 @@ load_dotenv()
HIBP_API_KEY = os.getenv("HIBP_API_KEY")
-class PhoneToBreachesScanner(Scanner):
+class PhoneToBreachesTransform(Transform):
"""[HIBPWNED] Get the breaches the phone number might be invovled in."""
InputType = List[Phone]
@@ -56,7 +56,7 @@ class PhoneToBreachesScanner(Scanner):
@classmethod
def get_params_schema(cls) -> List[Dict[str, Any]]:
- """Declare required parameters for this scanner"""
+ """Declare required parameters for this transform"""
return [
{
"name": "HIBP_API_KEY",
@@ -102,7 +102,7 @@ class PhoneToBreachesScanner(Scanner):
Logger.error(
self.sketch_id, {"message": "Could not find HIBP_API_URL in params."}
)
- headers = {"hibp-api-key": api_key, "User-Agent": "FlowsInt-Scanner"}
+ headers = {"hibp-api-key": api_key, "User-Agent": "FlowsInt-Transform"}
Logger.info(self.sketch_id, {"message": f"HIBP API URL: {api_url}"})
for phone in data:
try:
@@ -235,5 +235,5 @@ class PhoneToBreachesScanner(Scanner):
# Make types available at module level for easy access
-InputType = PhoneToBreachesScanner.InputType
-OutputType = PhoneToBreachesScanner.OutputType
+InputType = PhoneToBreachesTransform.InputType
+OutputType = PhoneToBreachesTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py b/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py
index 53d2d206..29e53567 100644
--- a/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py
+++ b/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py
@@ -3,14 +3,14 @@ import subprocess
from pathlib import Path
from typing import List, Union
from flowsint_core.utils import is_valid_username
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.social import SocialProfile
from flowsint_core.core.logger import Logger
false_positives = ["LeagueOfLegends"]
-class MaigretScanner(Scanner):
+class MaigretTransform(Transform):
"""[MAIGRET] Scans usernames for associated social accounts using Maigret."""
# Define types as class attributes - base class handles schema generation automatically
@@ -182,5 +182,5 @@ class MaigretScanner(Scanner):
# Make types available at module level for easy access
-InputType = MaigretScanner.InputType
-OutputType = MaigretScanner.OutputType
+InputType = MaigretTransform.InputType
+OutputType = MaigretTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py b/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py
index 2303b633..5081ca59 100644
--- a/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py
+++ b/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py
@@ -3,11 +3,11 @@ from pathlib import Path
from typing import List, Union
from flowsint_core.utils import is_valid_username
from flowsint_types.social import SocialProfile
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_core.core.logger import Logger
-class SherlockScanner(Scanner):
+class SherlockTransform(Transform):
"""[SHERLOCK] Scans the usernames for associated social accounts using Sherlock."""
# Define types as class attributes - base class handles schema generation automatically
@@ -127,5 +127,5 @@ class SherlockScanner(Scanner):
# Make types available at module level for easy access
-InputType = SherlockScanner.InputType
-OutputType = SherlockScanner.OutputType
+InputType = SherlockTransform.InputType
+OutputType = SherlockTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/utils.py b/flowsint-transforms/src/flowsint_transforms/utils.py
index 6a752679..b912a880 100644
--- a/flowsint-transforms/src/flowsint_transforms/utils.py
+++ b/flowsint-transforms/src/flowsint_transforms/utils.py
@@ -268,18 +268,18 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:
input_output = input_node["data"]["outputs"]
node_lookup = {node["id"]: node for node in nodes}
- scanners = []
+ transforms = []
for edge in edges:
target_id = edge["target"]
source_handle = edge["sourceHandle"]
target_handle = edge["targetHandle"]
- scanner_node = node_lookup.get(target_id)
- if scanner_node and scanner_node["data"]["type"] == "scanner":
- scanners.append(
+ transform_node = node_lookup.get(target_id)
+ if transform_node and transform_node["data"]["type"] == "transform":
+ transforms.append(
{
- "transform_name": scanner_node["data"]["name"],
- "module": scanner_node["data"]["module"],
+ "transform_name": transform_node["data"]["name"],
+ "module": transform_node["data"]["module"],
"input": source_handle,
"output": target_handle,
}
@@ -290,8 +290,8 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]:
"name": input_node["data"]["name"],
"outputs": input_output,
},
- "scanners": scanners,
- "transform_names": [scanner["transform_name"] for scanner in scanners],
+ "transforms": transforms,
+ "transform_names": [transform["transform_name"] for transform in transforms],
}
diff --git a/flowsint-transforms/src/flowsint_transforms/website/to_crawler.py b/flowsint-transforms/src/flowsint_transforms/website/to_crawler.py
index ab2fb295..67e86044 100644
--- a/flowsint-transforms/src/flowsint_transforms/website/to_crawler.py
+++ b/flowsint-transforms/src/flowsint_transforms/website/to_crawler.py
@@ -1,6 +1,6 @@
from typing import List, Optional, Union
from urllib.parse import urlparse
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.website import Website
from flowsint_types.phone import Phone
from flowsint_types.email import Email
@@ -15,7 +15,7 @@ class ReturnType(BaseModel):
phones: Optional[Phone]
-class WebsiteToCrawler(Scanner):
+class WebsiteToCrawler(Transform):
"""From website to crawler."""
# Define types as class attributes - base class handles schema generation automatically
diff --git a/flowsint-transforms/src/flowsint_transforms/website/to_domain.py b/flowsint-transforms/src/flowsint_transforms/website/to_domain.py
index a9e0e5a5..cb400323 100644
--- a/flowsint-transforms/src/flowsint_transforms/website/to_domain.py
+++ b/flowsint-transforms/src/flowsint_transforms/website/to_domain.py
@@ -1,12 +1,12 @@
from typing import List, Union
from urllib.parse import urlparse
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.website import Website
from flowsint_types.domain import Domain
from flowsint_core.core.logger import Logger
-class WebsiteToDomainScanner(Scanner):
+class WebsiteToDomainTransform(Transform):
"""From website to domain."""
# Define types as class attributes - base class handles schema generation automatically
@@ -110,5 +110,5 @@ class WebsiteToDomainScanner(Scanner):
# Make types available at module level for easy access
-InputType = WebsiteToDomainScanner.InputType
-OutputType = WebsiteToDomainScanner.OutputType
+InputType = WebsiteToDomainTransform.InputType
+OutputType = WebsiteToDomainTransform.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/website/to_links.py b/flowsint-transforms/src/flowsint_transforms/website/to_links.py
index bc90ee31..f31e3f37 100644
--- a/flowsint-transforms/src/flowsint_transforms/website/to_links.py
+++ b/flowsint-transforms/src/flowsint_transforms/website/to_links.py
@@ -1,12 +1,12 @@
from typing import List, Union
from urllib.parse import urlparse
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.website import Website
from flowsint_core.core.logger import Logger
from reconspread import Crawler
-class WebsiteToLinks(Scanner):
+class WebsiteToLinks(Transform):
"""From website to spread crawler that extracts domains and internal/external links."""
# Define types as class attributes - base class handles schema generation automatically
diff --git a/flowsint-transforms/src/flowsint_transforms/website/to_text.py b/flowsint-transforms/src/flowsint_transforms/website/to_text.py
index 825ef88d..a23757a1 100644
--- a/flowsint-transforms/src/flowsint_transforms/website/to_text.py
+++ b/flowsint-transforms/src/flowsint_transforms/website/to_text.py
@@ -1,12 +1,12 @@
from typing import List, Union
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.phrase import Phrase
from flowsint_types.website import Website
import requests
from bs4 import BeautifulSoup
-class WebsiteToText(Scanner):
+class WebsiteToText(Transform):
"""Extracts the texts in a webpage."""
InputType = List[Website]
diff --git a/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py b/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py
index cb5f77e8..4ddefaac 100644
--- a/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py
+++ b/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py
@@ -1,5 +1,5 @@
from typing import List, Dict, Any, Union, Optional
-from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.transform_base import Transform
from flowsint_types.website import Website
from flowsint_types.web_tracker import WebTracker
from flowsint_core.core.logger import Logger
@@ -8,7 +8,7 @@ from flowsint_core.core.vault import VaultProtocol
from recontrack import TrackingCodeExtractor
-class WebsiteToWebtrackersScanner(Scanner):
+class WebsiteToWebtrackersTransform(Transform):
"""From website to webtrackers."""
# Define types as class attributes - base class handles schema generation automatically
@@ -128,5 +128,5 @@ class WebsiteToWebtrackersScanner(Scanner):
# Make types available at module level for easy access
-InputType = WebsiteToWebtrackersScanner.InputType
-OutputType = WebsiteToWebtrackersScanner.OutputType
+InputType = WebsiteToWebtrackersTransform.InputType
+OutputType = WebsiteToWebtrackersTransform.OutputType
diff --git a/flowsint-transforms/tests/insert_command.py b/flowsint-transforms/tests/insert_command.py
index 8e6cc2de..4df5a035 100644
--- a/flowsint-transforms/tests/insert_command.py
+++ b/flowsint-transforms/tests/insert_command.py
@@ -7,7 +7,7 @@ if __name__ == "__main__":
from flowsint_types.domain import Domain
from flowsint_types.ip import Ip
-from flowsint_transforms.domains.resolve import ResolveScanner
+from flowsint_transforms.domains.resolve import ResolveTransform
async def main():
@@ -15,11 +15,11 @@ async def main():
domains = [Domain(domain="adaltas.com")]
ips = [Ip(address="12.23.34.45"), Ip(address="56.67.78.89")]
- # Test the scanner
- scanner = ResolveScanner("sketch_123", "scan_123")
+ # Test the transform
+ transform = ResolveTransform("sketch_123", "scan_123")
# Test the new KISS postprocess method
- scanner.postprocess(ips[:1], domains) # Only use first IP to match domains length
+ transform.postprocess(ips[:1], domains) # Only use first IP to match domains length
print("Postprocess test completed successfully!")
diff --git a/flowsint-transforms/tests/scanners/__init__.py b/flowsint-transforms/tests/transforms/__init__.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/__init__.py
rename to flowsint-transforms/tests/transforms/__init__.py
diff --git a/flowsint-transforms/tests/scanners/crypto/__init__.py b/flowsint-transforms/tests/transforms/crypto/__init__.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/crypto/__init__.py
rename to flowsint-transforms/tests/transforms/crypto/__init__.py
diff --git a/flowsint-transforms/tests/scanners/crypto/wallet_to_nfts.py b/flowsint-transforms/tests/transforms/crypto/wallet_to_nfts.py
similarity index 83%
rename from flowsint-transforms/tests/scanners/crypto/wallet_to_nfts.py
rename to flowsint-transforms/tests/transforms/crypto/wallet_to_nfts.py
index 9e7778e6..6ad61434 100644
--- a/flowsint-transforms/tests/scanners/crypto/wallet_to_nfts.py
+++ b/flowsint-transforms/tests/transforms/crypto/wallet_to_nfts.py
@@ -2,24 +2,24 @@ from flowsint_transforms.crypto.wallet_to_nfts import CryptoWalletAddressToNFTs
from flowsint_types.wallet import CryptoWallet, CryptoNFT
from pydantic import HttpUrl
-scanner = CryptoWalletAddressToNFTs("sketch_123", "scan_123")
+transform = CryptoWalletAddressToNFTs("sketch_123", "scan_123")
def test_wallet_address_to_transactions_name():
- assert scanner.name() == "wallet_to_nfts"
+ assert transform.name() == "wallet_to_nfts"
def test_wallet_address_to_transactions_category():
- assert scanner.category() == "crypto"
+ assert transform.category() == "crypto"
def test_wallet_address_to_transactions_key():
- assert scanner.key() == "address"
+ assert transform.key() == "address"
def test_preprocess_with_string():
input_data = ["0x742d35Cc6634C0532925a3b844Bc454e4438f44e"]
- result = scanner.preprocess(input_data)
+ result = transform.preprocess(input_data)
assert len(result) == 1
assert isinstance(result[0], CryptoWallet)
assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"
@@ -27,7 +27,7 @@ def test_preprocess_with_string():
def test_preprocess_with_dict():
input_data = [{"address": "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"}]
- result = scanner.preprocess(input_data)
+ result = transform.preprocess(input_data)
assert len(result) == 1
assert isinstance(result[0], CryptoWallet)
assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"
@@ -36,7 +36,7 @@ def test_preprocess_with_dict():
def test_preprocess_with_wallet_object():
wallet = CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")
input_data = [wallet]
- result = scanner.preprocess(input_data)
+ result = transform.preprocess(input_data)
assert len(result) == 1
assert isinstance(result[0], CryptoWallet)
assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"
@@ -59,10 +59,10 @@ def test_scan_mocked_transactions(monkeypatch):
)
]
- monkeypatch.setattr(scanner, "_get_nfts", mock_get_nfts)
+ monkeypatch.setattr(transform, "_get_nfts", mock_get_nfts)
input_data = [CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")]
- result = scanner.scan(input_data)
+ result = transform.scan(input_data)
assert len(result) == 1
assert len(result[0]) == 1
diff --git a/flowsint-transforms/tests/scanners/crypto/wallet_to_transactions.py b/flowsint-transforms/tests/transforms/crypto/wallet_to_transactions.py
similarity index 76%
rename from flowsint-transforms/tests/scanners/crypto/wallet_to_transactions.py
rename to flowsint-transforms/tests/transforms/crypto/wallet_to_transactions.py
index af0deb74..fd4c4090 100644
--- a/flowsint-transforms/tests/scanners/crypto/wallet_to_transactions.py
+++ b/flowsint-transforms/tests/transforms/crypto/wallet_to_transactions.py
@@ -4,7 +4,7 @@ from flowsint_transforms.crypto.wallet_to_transactions import (
)
from flowsint_types.wallet import CryptoWallet, CryptoWalletTransaction
-scanner = CryptoWalletAddressToTransactions(
+transform = CryptoWalletAddressToTransactions(
"sketch_123",
"scan_123",
params={"ETHERSCAN_API_KEY": "ta-clef-api"},
@@ -12,20 +12,20 @@ scanner = CryptoWalletAddressToTransactions(
def test_wallet_address_to_transactions_name():
- assert scanner.name() == "wallet_to_transactions"
+ assert transform.name() == "wallet_to_transactions"
def test_wallet_address_to_transactions_category():
- assert scanner.category() == "CryptoCryptoWallet"
+ assert transform.category() == "CryptoCryptoWallet"
def test_wallet_address_to_transactions_key():
- assert scanner.key() == "address"
+ assert transform.key() == "address"
def test_preprocess_with_string():
input_data = ["0x742d35Cc6634C0532925a3b844Bc454e4438f44e"]
- result = scanner.preprocess(input_data)
+ result = transform.preprocess(input_data)
assert len(result) == 1
assert isinstance(result[0], CryptoWallet)
assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"
@@ -33,7 +33,7 @@ def test_preprocess_with_string():
def test_preprocess_with_dict():
input_data = [{"address": "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"}]
- result = scanner.preprocess(input_data)
+ result = transform.preprocess(input_data)
assert len(result) == 1
assert isinstance(result[0], CryptoWallet)
assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"
@@ -42,7 +42,7 @@ def test_preprocess_with_dict():
def test_preprocess_with_wallet_object():
wallet = CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")
input_data = [wallet]
- result = scanner.preprocess(input_data)
+ result = transform.preprocess(input_data)
assert len(result) == 1
assert isinstance(result[0], CryptoWallet)
assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"
@@ -74,10 +74,10 @@ async def test_scan_mocked_transactions(monkeypatch):
)
]
- monkeypatch.setattr(scanner, "_get_transactions", mock_get_transactions)
+ monkeypatch.setattr(transform, "_get_transactions", mock_get_transactions)
input_data = [CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")]
- result = await scanner.scan(input_data)
+ result = await transform.scan(input_data)
assert len(result) == 1
assert len(result[0]) == 1
@@ -88,18 +88,18 @@ async def test_scan_mocked_transactions(monkeypatch):
assert result[0][0].timestamp == "1234567890"
-def test_scanner_requires_api_key():
- """Test that the scanner validates required ETHERSCAN_API_KEY parameter at construction"""
+def test_transform_requires_api_key():
+ """Test that the transform validates required ETHERSCAN_API_KEY parameter at construction"""
with pytest.raises(
- ValueError, match="Scanner wallet_to_transactions received invalid params"
+ ValueError, match="Transform wallet_to_transactions received invalid params"
):
CryptoWalletAddressToTransactions("sketch_123", "scan_123", params={})
-def test_scanner_with_invalid_api_key_type():
- """Test that the scanner validates parameter types"""
+def test_transform_with_invalid_api_key_type():
+ """Test that the transform validates parameter types"""
with pytest.raises(
- ValueError, match="Scanner wallet_to_transactions received invalid params"
+ ValueError, match="Transform wallet_to_transactions received invalid params"
):
CryptoWalletAddressToTransactions(
"sketch_123", "scan_123", params={"ETHERSCAN_API_KEY": 123}
diff --git a/flowsint-transforms/tests/scanners/domain/__init__.py b/flowsint-transforms/tests/transforms/domain/__init__.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/domain/__init__.py
rename to flowsint-transforms/tests/transforms/domain/__init__.py
diff --git a/flowsint-transforms/tests/scanners/domain/resolve.py b/flowsint-transforms/tests/transforms/domain/resolve.py
similarity index 71%
rename from flowsint-transforms/tests/scanners/domain/resolve.py
rename to flowsint-transforms/tests/transforms/domain/resolve.py
index de72d91a..bccf0101 100644
--- a/flowsint-transforms/tests/scanners/domain/resolve.py
+++ b/flowsint-transforms/tests/transforms/domain/resolve.py
@@ -1,10 +1,10 @@
-from flowsint_transforms.domains.resolve import ResolveScanner
+from flowsint_transforms.domains.resolve import ResolveTransform
from flowsint_types.domain import Domain
from flowsint_types.ip import Ip
from typing import List
import pytest
-scanner = ResolveScanner("sketch_123", "scan_123")
+transform = ResolveTransform("sketch_123", "scan_123")
def test_preprocess_valid_domains():
@@ -12,7 +12,7 @@ def test_preprocess_valid_domains():
Domain(domain="example.com"),
Domain(domain="example2.com"),
]
- result = scanner.preprocess(domains)
+ result = transform.preprocess(domains)
result_domains = [d.domain for d in result]
expected_domains = [d.domain for d in domains]
@@ -25,7 +25,7 @@ def test_unprocessed_valid_domains():
"example.com",
"example2.com",
]
- result = scanner.preprocess(domains)
+ result = transform.preprocess(domains)
result_domains = [d for d in result]
expected_domains = [Domain(domain=d) for d in domains]
assert result_domains == expected_domains
@@ -37,7 +37,7 @@ def test_preprocess_invalid_domains():
Domain(domain="invalid_domain"),
Domain(domain="example.org"),
]
- result = scanner.preprocess(domains)
+ result = transform.preprocess(domains)
result_domains = [d.domain for d in result]
assert "example.com" in result_domains
@@ -52,7 +52,7 @@ def test_preprocess_multiple_formats():
Domain(domain="example.org"),
"example.org",
]
- result = scanner.preprocess(domains)
+ result = transform.preprocess(domains)
result_domains = [d.domain for d in result]
assert "example.com" in result_domains
@@ -70,15 +70,15 @@ async def test_scan_returns_ip(monkeypatch):
monkeypatch.setattr("socket.gethostbyname", mock_gethostbyname)
input_data = [Domain(domain="example.com")]
- output = await scanner.execute(input_data)
+ output = await transform.execute(input_data)
print(output)
assert isinstance(output, list)
assert output[0].address == "12.23.34.45"
def test_schemas():
- input_schema = scanner.input_schema()
- output_schema = scanner.output_schema()
+ input_schema = transform.input_schema()
+ output_schema = transform.output_schema()
# Test the structure and key properties rather than exact match
assert input_schema["type"] == "Domain"
@@ -93,20 +93,20 @@ def test_schemas():
class TestResolveInputOutputTypes:
- """Test the InputType/OutputType functionality for ResolveScanner"""
+ """Test the InputType/OutputType functionality for ResolveTransform"""
def test_input_output_types_are_defined(self):
"""Test that InputType and OutputType are properly defined"""
- assert hasattr(ResolveScanner, "InputType")
- assert hasattr(ResolveScanner, "OutputType")
- assert ResolveScanner.InputType == List[Domain]
- assert ResolveScanner.OutputType == List[Ip]
+ assert hasattr(ResolveTransform, "InputType")
+ assert hasattr(ResolveTransform, "OutputType")
+ assert ResolveTransform.InputType == List[Domain]
+ assert ResolveTransform.OutputType == List[Ip]
def test_schemas_use_generate_methods(self):
"""Test that schema methods use the new generate methods"""
# These should work without error
- input_schema = ResolveScanner.generate_input_schema()
- output_schema = ResolveScanner.generate_output_schema()
+ input_schema = ResolveTransform.generate_input_schema()
+ output_schema = ResolveTransform.generate_output_schema()
assert isinstance(input_schema, dict)
assert isinstance(output_schema, dict)
@@ -115,12 +115,12 @@ class TestResolveInputOutputTypes:
def test_schema_methods_return_same_as_generate_methods(self):
"""Test that input_schema() and output_schema() return the same as generate methods"""
- assert ResolveScanner.input_schema() == ResolveScanner.generate_input_schema()
- assert ResolveScanner.output_schema() == ResolveScanner.generate_output_schema()
+ assert ResolveTransform.input_schema() == ResolveTransform.generate_input_schema()
+ assert ResolveTransform.output_schema() == ResolveTransform.generate_output_schema()
def test_input_schema_properties(self):
"""Test input schema has expected properties"""
- schema = ResolveScanner.input_schema()
+ schema = ResolveTransform.input_schema()
properties = schema["properties"]
property_names = [p["name"] for p in properties]
@@ -130,7 +130,7 @@ class TestResolveInputOutputTypes:
def test_output_schema_properties(self):
"""Test output schema has expected properties"""
- schema = ResolveScanner.output_schema()
+ schema = ResolveTransform.output_schema()
properties = schema["properties"]
property_names = [p["name"] for p in properties]
@@ -139,15 +139,15 @@ class TestResolveInputOutputTypes:
assert "address" in property_names
def test_type_accessibility_from_instance(self):
- """Test that types are accessible from scanner instance"""
- scanner_instance = ResolveScanner("test", "test")
+ """Test that types are accessible from transform instance"""
+ transform_instance = ResolveTransform("test", "test")
- assert scanner_instance.InputType == List[Domain]
- assert scanner_instance.OutputType == List[Ip]
+ assert transform_instance.InputType == List[Domain]
+ assert transform_instance.OutputType == List[Ip]
# Should be able to generate schemas from instance
- input_schema = scanner_instance.generate_input_schema()
- output_schema = scanner_instance.generate_output_schema()
+ input_schema = transform_instance.generate_input_schema()
+ output_schema = transform_instance.generate_output_schema()
assert input_schema["type"] == "Domain"
assert output_schema["type"] == "Ip"
diff --git a/flowsint-transforms/tests/scanners/domain/subdomains.py b/flowsint-transforms/tests/transforms/domain/subdomains.py
similarity index 85%
rename from flowsint-transforms/tests/scanners/domain/subdomains.py
rename to flowsint-transforms/tests/transforms/domain/subdomains.py
index d323721c..16ee1835 100644
--- a/flowsint-transforms/tests/scanners/domain/subdomains.py
+++ b/flowsint-transforms/tests/transforms/domain/subdomains.py
@@ -1,7 +1,7 @@
-from flowsint_transforms.domains.subdomains import SubdomainScanner
+from flowsint_transforms.domains.subdomains import SubdomainTransform
from flowsint_types.domain import Domain, Domain
-scanner = SubdomainScanner("sketch_123", "scan_123")
+transform = SubdomainTransform("sketch_123", "scan_123")
def test_preprocess_valid_domains():
@@ -9,7 +9,7 @@ def test_preprocess_valid_domains():
Domain(domain="example.com"),
Domain(domain="example2.com"),
]
- result = scanner.preprocess(domains)
+ result = transform.preprocess(domains)
result_domains = [d.domain for d in result]
expected_domains = [d.domain for d in domains]
@@ -22,7 +22,7 @@ def test_unprocessed_valid_domains():
"example.com",
"example2.com",
]
- result = scanner.preprocess(domains)
+ result = transform.preprocess(domains)
result_domains = [d for d in result]
expected_domains = [Domain(domain=d) for d in domains]
assert result_domains == expected_domains
@@ -34,7 +34,7 @@ def test_preprocess_invalid_domains():
Domain(domain="invalid_domain"),
Domain(domain="example.org"),
]
- result = scanner.preprocess(domains)
+ result = transform.preprocess(domains)
result_domains = [d.domain for d in result]
assert "example.com" in result_domains
@@ -49,7 +49,7 @@ def test_preprocess_multiple_formats():
Domain(domain="example.org"),
"example.org",
]
- result = scanner.preprocess(domains)
+ result = transform.preprocess(domains)
result_domains = [d.domain for d in result]
assert "example.com" in result_domains
@@ -81,11 +81,11 @@ def test_scan_extracts_subdomains(monkeypatch):
assert "example.com" in url
return MockRequestsResponse(mock_response)
- # Patch la requête réseau dans le module scanner
+ # Patch la requête réseau dans le module transform
monkeypatch.setattr("requests.get", mock_get)
input_data = [Domain(domain="example.com")]
- domains = scanner.execute(input_data)
+ domains = transform.execute(input_data)
assert isinstance(domains, list)
for sub in domains:
print(sub)
diff --git a/flowsint-transforms/tests/scanners/domain/to_history.py b/flowsint-transforms/tests/transforms/domain/to_history.py
similarity index 69%
rename from flowsint-transforms/tests/scanners/domain/to_history.py
rename to flowsint-transforms/tests/transforms/domain/to_history.py
index bdb62db5..73619dbd 100644
--- a/flowsint-transforms/tests/scanners/domain/to_history.py
+++ b/flowsint-transforms/tests/transforms/domain/to_history.py
@@ -2,7 +2,7 @@ import pytest
import json
import os
from unittest.mock import Mock
-from flowsint_transforms.domain.to_history import DomainToHistoryScanner
+from flowsint_transforms.domain.to_history import DomainToHistoryTransform
from flowsint_types.domain import Domain
@@ -37,7 +37,7 @@ class MockNeo4jConn:
pass
-class MockScanner(DomainToHistoryScanner):
+class MockTransform(DomainToHistoryTransform):
def __init__(self):
self.sketch_id = "test_sketch_123"
self.neo4j_conn = MockNeo4jConn()
@@ -50,10 +50,10 @@ class MockScanner(DomainToHistoryScanner):
@pytest.fixture
-def scanner():
- """Create a scanner instance for testing."""
- scanner = MockScanner()
- return scanner
+def transform():
+ """Create a transform instance for testing."""
+ transform = MockTransform()
+ return transform
@pytest.fixture
@@ -65,13 +65,13 @@ def test_data():
return json.load(f)
-def test_preprocess_valid_domains(scanner):
+def test_preprocess_valid_domains(transform):
"""Test preprocessing with valid domains."""
domains = [
Domain(domain="example.com"),
Domain(domain="example2.com"),
]
- result = scanner.preprocess(domains)
+ result = transform.preprocess(domains)
result_domains = [d.domain for d in result]
expected_domains = [d.domain for d in domains]
@@ -79,10 +79,10 @@ def test_preprocess_valid_domains(scanner):
assert result_domains == expected_domains
-def test_preprocess_string_domains(scanner):
+def test_preprocess_string_domains(transform):
"""Test preprocessing with string domains."""
domains = ["example.com", "example2.com"]
- result = scanner.preprocess(domains)
+ result = transform.preprocess(domains)
assert len(result) == 2
assert all(isinstance(d, Domain) for d in result)
@@ -90,10 +90,10 @@ def test_preprocess_string_domains(scanner):
assert result[1].domain == "example2.com"
-def test_preprocess_dict_domains(scanner):
+def test_preprocess_dict_domains(transform):
"""Test preprocessing with dict domains."""
domains = [{"domain": "example.com"}, {"domain": "example2.com"}]
- result = scanner.preprocess(domains)
+ result = transform.preprocess(domains)
assert len(result) == 2
assert all(isinstance(d, Domain) for d in result)
@@ -101,14 +101,14 @@ def test_preprocess_dict_domains(scanner):
assert result[1].domain == "example2.com"
-def test_preprocess_invalid_domains(scanner):
+def test_preprocess_invalid_domains(transform):
"""Test preprocessing with invalid domains."""
domains = [
Domain(domain="example.com"),
Domain(domain="invalid_domain"),
Domain(domain="example.org"),
]
- result = scanner.preprocess(domains)
+ result = transform.preprocess(domains)
result_domains = [d.domain for d in result]
assert "example.com" in result_domains
@@ -116,21 +116,21 @@ def test_preprocess_invalid_domains(scanner):
assert "invalid_domain" not in result_domains
-def test_is_redacted(scanner):
+def test_is_redacted(transform):
"""Test the __is_redacted method."""
# Should be redacted
- assert scanner._DomainToHistoryScanner__is_redacted("REDACTED FOR PRIVACY")
- assert scanner._DomainToHistoryScanner__is_redacted("redacted for privacy")
- assert scanner._DomainToHistoryScanner__is_redacted("Some text with PRIVACY in it")
+ assert transform._DomainToHistoryTransform__is_redacted("REDACTED FOR PRIVACY")
+ assert transform._DomainToHistoryTransform__is_redacted("redacted for privacy")
+ assert transform._DomainToHistoryTransform__is_redacted("Some text with PRIVACY in it")
# Should NOT be redacted
- assert not scanner._DomainToHistoryScanner__is_redacted("JOHN DOE")
- assert not scanner._DomainToHistoryScanner__is_redacted("[email protected]")
- assert not scanner._DomainToHistoryScanner__is_redacted("123 JOHN STREET")
- assert not scanner._DomainToHistoryScanner__is_redacted("DOE CITY")
+ assert not transform._DomainToHistoryTransform__is_redacted("JOHN DOE")
+ assert not transform._DomainToHistoryTransform__is_redacted("[email protected]")
+ assert not transform._DomainToHistoryTransform__is_redacted("123 JOHN STREET")
+ assert not transform._DomainToHistoryTransform__is_redacted("DOE CITY")
-def test_has_non_redacted_info(scanner):
+def test_has_non_redacted_info(transform):
"""Test the __has_non_redacted_info method."""
# Contact with valid information
valid_contact = {
@@ -142,7 +142,7 @@ def test_has_non_redacted_info(scanner):
"zip_code": "12345",
"country_name": "United States",
}
- assert scanner._DomainToHistoryScanner__has_non_redacted_info(valid_contact)
+ assert transform._DomainToHistoryTransform__has_non_redacted_info(valid_contact)
# Contact with all redacted information
redacted_contact = {
@@ -154,13 +154,13 @@ def test_has_non_redacted_info(scanner):
"zip_code": "REDACTED FOR PRIVACY",
"country_name": "REDACTED FOR PRIVACY",
}
- assert not scanner._DomainToHistoryScanner__has_non_redacted_info(redacted_contact)
+ assert not transform._DomainToHistoryTransform__has_non_redacted_info(redacted_contact)
# Empty contact
- assert not scanner._DomainToHistoryScanner__has_non_redacted_info({})
+ assert not transform._DomainToHistoryTransform__has_non_redacted_info({})
-def test_extract_individual_from_contact(scanner):
+def test_extract_individual_from_contact(transform):
"""Test the __extract_individual_from_contact method."""
# Valid contact
valid_contact = {
@@ -173,7 +173,7 @@ def test_extract_individual_from_contact(scanner):
"country_name": "United States",
}
- individual = scanner._DomainToHistoryScanner__extract_individual_from_contact(
+ individual = transform._DomainToHistoryTransform__extract_individual_from_contact(
valid_contact, "REGISTRANT"
)
@@ -187,7 +187,7 @@ def test_extract_individual_from_contact(scanner):
assert individual.phone_numbers == ["+123456789"]
-def test_extract_individual_redacted_name(scanner):
+def test_extract_individual_redacted_name(transform):
"""Test that individuals with redacted names are skipped."""
redacted_contact = {
"full_name": "REDACTED FOR PRIVACY",
@@ -195,27 +195,27 @@ def test_extract_individual_redacted_name(scanner):
"phone_number": "+1234567890",
}
- individual = scanner._DomainToHistoryScanner__extract_individual_from_contact(
+ individual = transform._DomainToHistoryTransform__extract_individual_from_contact(
redacted_contact, "REGISTRANT"
)
assert individual is None
-def test_is_valid_email(scanner):
+def test_is_valid_email(transform):
"""Test the __is_valid_email method."""
# Valid emails
- assert scanner._DomainToHistoryScanner__is_valid_email("[email protected]")
- assert scanner._DomainToHistoryScanner__is_valid_email("[email protected]")
- assert scanner._DomainToHistoryScanner__is_valid_email("[email protected]")
+ assert transform._DomainToHistoryTransform__is_valid_email("[email protected]")
+ assert transform._DomainToHistoryTransform__is_valid_email("[email protected]")
+ assert transform._DomainToHistoryTransform__is_valid_email("[email protected]")
# Invalid emails
- assert not scanner._DomainToHistoryScanner__is_valid_email("invalid-email")
- assert not scanner._DomainToHistoryScanner__is_valid_email("@example.com")
- assert not scanner._DomainToHistoryScanner__is_valid_email("test@")
- assert not scanner._DomainToHistoryScanner__is_valid_email("")
+ assert not transform._DomainToHistoryTransform__is_valid_email("invalid-email")
+ assert not transform._DomainToHistoryTransform__is_valid_email("@example.com")
+ assert not transform._DomainToHistoryTransform__is_valid_email("test@")
+ assert not transform._DomainToHistoryTransform__is_valid_email("")
-def test_extract_physical_address(scanner):
+def test_extract_physical_address(transform):
"""Test the __extract_physical_address method."""
# Valid address
valid_contact = {
@@ -225,7 +225,7 @@ def test_extract_physical_address(scanner):
"country_name": "United States",
}
- address = scanner._DomainToHistoryScanner__extract_physical_address(valid_contact)
+ address = transform._DomainToHistoryTransform__extract_physical_address(valid_contact)
assert address is not None
assert address.address == "123 JOHN STREET"
@@ -241,14 +241,14 @@ def test_extract_physical_address(scanner):
"country_name": "United States",
}
- address = scanner._DomainToHistoryScanner__extract_physical_address(
+ address = transform._DomainToHistoryTransform__extract_physical_address(
redacted_contact
)
assert address is None
@pytest.mark.asyncio
-async def test_scan_with_test_data(scanner, test_data, monkeypatch):
+async def test_scan_with_test_data(transform, test_data, monkeypatch):
"""Test the scan method with test data."""
# Mock the __get_infos_from_whoxy method to return test data
@@ -258,30 +258,30 @@ async def test_scan_with_test_data(scanner, test_data, monkeypatch):
return {}
monkeypatch.setattr(
- scanner, "_DomainToHistoryScanner__get_infos_from_whoxy", mock_get_infos
+ transform, "_DomainToHistoryTransform__get_infos_from_whoxy", mock_get_infos
)
# Test with epios.com domain
input_domains = [Domain(domain="epios.com")]
- results = await scanner.scan(input_domains)
+ results = await transform.scan(input_domains)
# Should find the domain (one for each WHOIS record)
assert len(results) == 16 # 16 WHOIS records in the test data
assert all(r.domain == "epios.com" for r in results)
# Should have extracted data
- assert len(scanner._extracted_data) == 16
+ assert len(transform._extracted_data) == 16
# Should have extracted individuals
assert (
- len(scanner._extracted_individuals) > 0
+ len(transform._extracted_individuals) > 0
), "Should have extracted some individuals"
# Check that JOHN DOE is in the extracted individuals
marc_found = False
marc_individuals = []
- for individual_info in scanner._extracted_individuals:
+ for individual_info in transform._extracted_individuals:
individual = individual_info["individual"]
if "JOHN DOE" in individual.full_name:
marc_found = True
@@ -299,7 +299,7 @@ async def test_scan_with_test_data(scanner, test_data, monkeypatch):
# Print summary of all extracted individuals
print(f"\n=== Summary of extracted individuals ===")
- for individual_info in scanner._extracted_individuals:
+ for individual_info in transform._extracted_individuals:
individual = individual_info["individual"]
print(
f"- {individual.full_name} ({individual_info['contact_type']}) for {individual_info['domain_name']}"
@@ -310,7 +310,7 @@ async def test_scan_with_test_data(scanner, test_data, monkeypatch):
print(f" Phones: {individual.phone_numbers}")
-def test_postprocess_creates_nodes_and_relationships(scanner, test_data, monkeypatch):
+def test_postprocess_creates_nodes_and_relationships(transform, test_data, monkeypatch):
"""Test that postprocess creates the expected nodes and relationships."""
# Mock the __get_infos_from_whoxy method
@@ -320,7 +320,7 @@ def test_postprocess_creates_nodes_and_relationships(scanner, test_data, monkeyp
return {}
monkeypatch.setattr(
- scanner, "_DomainToHistoryScanner__get_infos_from_whoxy", mock_get_infos
+ transform, "_DomainToHistoryTransform__get_infos_from_whoxy", mock_get_infos
)
# First run scan to populate _extracted_data and _extracted_individuals
@@ -331,13 +331,13 @@ def test_postprocess_creates_nodes_and_relationships(scanner, test_data, monkeyp
try:
input_domains = [Domain(domain="epios.com")]
- results = loop.run_until_complete(scanner.scan(input_domains))
+ results = loop.run_until_complete(transform.scan(input_domains))
# Debug: Check what individuals were extracted
print(
- f"\n=== DEBUG: _extracted_individuals has {len(scanner._extracted_individuals)} individuals ==="
+ f"\n=== DEBUG: _extracted_individuals has {len(transform._extracted_individuals)} individuals ==="
)
- for i, individual_info in enumerate(scanner._extracted_individuals):
+ for i, individual_info in enumerate(transform._extracted_individuals):
individual = individual_info["individual"]
print(
f"Individual {i+1}: {individual.full_name} ({individual_info['contact_type']}) for {individual_info['domain_name']}"
@@ -349,28 +349,28 @@ def test_postprocess_creates_nodes_and_relationships(scanner, test_data, monkeyp
# Now run postprocess
print(f"\n=== Running postprocess ===")
- scanner.postprocess(results, input_domains)
+ transform.postprocess(results, input_domains)
# Debug: Check what happened during postprocess
print(f"=== Postprocess completed ===")
- print(f"Nodes created: {len(scanner.neo4j_conn.nodes_created)}")
- print(f"Relationships created: {len(scanner.neo4j_conn.relationships_created)}")
+ print(f"Nodes created: {len(transform.neo4j_conn.nodes_created)}")
+ print(f"Relationships created: {len(transform.neo4j_conn.relationships_created)}")
# Should have created some nodes
- assert len(scanner.neo4j_conn.nodes_created) > 0
+ assert len(transform.neo4j_conn.nodes_created) > 0
# Should have created some relationships
- assert len(scanner.neo4j_conn.relationships_created) > 0
+ assert len(transform.neo4j_conn.relationships_created) > 0
# Check for domain node
domain_nodes = [
- n for n in scanner.neo4j_conn.nodes_created if n["label"] == "domain"
+ n for n in transform.neo4j_conn.nodes_created if n["label"] == "domain"
]
assert len(domain_nodes) > 0
# Check for individual nodes (should include JOHN DOE)
individual_nodes = [
- n for n in scanner.neo4j_conn.nodes_created if n["label"] == "individual"
+ n for n in transform.neo4j_conn.nodes_created if n["label"] == "individual"
]
assert len(individual_nodes) > 0
@@ -384,10 +384,10 @@ def test_postprocess_creates_nodes_and_relationships(scanner, test_data, monkeyp
loop.close()
-def test_schemas(scanner):
- """Test that the scanner has the expected schemas."""
- input_schema = scanner.input_schema()
- output_schema = scanner.output_schema()
+def test_schemas(transform):
+ """Test that the transform has the expected schemas."""
+ input_schema = transform.input_schema()
+ output_schema = transform.output_schema()
assert input_schema is not None
assert output_schema is not None
diff --git a/flowsint-transforms/tests/scanners/domain/whois.py b/flowsint-transforms/tests/transforms/domain/whois.py
similarity index 86%
rename from flowsint-transforms/tests/scanners/domain/whois.py
rename to flowsint-transforms/tests/transforms/domain/whois.py
index c03eac81..333d90f5 100644
--- a/flowsint-transforms/tests/scanners/domain/whois.py
+++ b/flowsint-transforms/tests/transforms/domain/whois.py
@@ -1,7 +1,7 @@
-from flowsint_transforms.domains.whois import WhoisScanner
+from flowsint_transforms.domains.whois import WhoisTransform
from flowsint_types.domain import Domain
-scanner = WhoisScanner("sketch_123", "scan_123")
+transform = WhoisTransform("sketch_123", "scan_123")
def test_preprocess_valid_domains():
@@ -9,7 +9,7 @@ def test_preprocess_valid_domains():
Domain(domain="example.com"),
Domain(domain="example2.com"),
]
- result = scanner.preprocess(domains)
+ result = transform.preprocess(domains)
result_domains = [d.domain for d in result]
expected_domains = [d.domain for d in domains]
@@ -22,7 +22,7 @@ def test_unprocessed_valid_domains():
"example.com",
"example2.com",
]
- result = scanner.preprocess(domains)
+ result = transform.preprocess(domains)
result_domains = [d for d in result]
expected_domains = [Domain(domain=d) for d in domains]
assert result_domains == expected_domains
@@ -34,7 +34,7 @@ def test_preprocess_invalid_domains():
Domain(domain="invalid_domain"),
Domain(domain="example.org"),
]
- result = scanner.preprocess(domains)
+ result = transform.preprocess(domains)
result_domains = [d.domain for d in result]
assert "example.com" in result_domains
@@ -49,7 +49,7 @@ def test_preprocess_multiple_formats():
Domain(domain="example.org"),
"example.org",
]
- result = scanner.preprocess(domains)
+ result = transform.preprocess(domains)
result_domains = [d.domain for d in result]
assert "example.com" in result_domains
@@ -73,7 +73,7 @@ def test_scan_returns_whois_objects(monkeypatch):
monkeypatch.setattr("whois.whois", mock_whois)
input_data = [Domain(domain="example.com")]
- output = scanner.execute(input_data)
+ output = transform.execute(input_data)
assert isinstance(output, list)
assert isinstance(output[0], Domain)
assert output[0].whois.org == "MockOrg"
@@ -81,8 +81,8 @@ def test_scan_returns_whois_objects(monkeypatch):
def test_schemas():
- input_schema = scanner.input_schema()
- output_schema = scanner.output_schema()
+ input_schema = transform.input_schema()
+ output_schema = transform.output_schema()
assert input_schema == {
"type": "Domain",
"properties": [
diff --git a/flowsint-transforms/tests/scanners/email/__init__.py b/flowsint-transforms/tests/transforms/email/__init__.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/email/__init__.py
rename to flowsint-transforms/tests/transforms/email/__init__.py
diff --git a/flowsint-transforms/tests/scanners/email/to_gravatar.py b/flowsint-transforms/tests/transforms/email/to_gravatar.py
similarity index 86%
rename from flowsint-transforms/tests/scanners/email/to_gravatar.py
rename to flowsint-transforms/tests/transforms/email/to_gravatar.py
index 871b5b4f..0e230039 100644
--- a/flowsint-transforms/tests/scanners/email/to_gravatar.py
+++ b/flowsint-transforms/tests/transforms/email/to_gravatar.py
@@ -1,30 +1,30 @@
import hashlib
from unittest.mock import Mock, patch
-from flowsint_transforms.emails.to_gravatar import EmailToGravatarScanner
+from flowsint_transforms.emails.to_gravatar import EmailToGravatarTransform
from flowsint_types.email import Email
from flowsint_types.gravatar import Gravatar
-scanner = EmailToGravatarScanner("sketch_123", "scan_123")
+transform = EmailToGravatarTransform("sketch_123", "scan_123")
-class TestEmailToGravatarScanner:
- """Test suite for EmailToGravatarScanner"""
+class TestEmailToGravatarTransform:
+ """Test suite for EmailToGravatarTransform"""
def test_name(self):
- """Test the scanner name"""
- assert EmailToGravatarScanner.name() == "to_gravatar"
+ """Test the transform name"""
+ assert EmailToGravatarTransform.name() == "to_gravatar"
def test_category(self):
- """Test the scanner category"""
- assert EmailToGravatarScanner.category() == "Email"
+ """Test the transform category"""
+ assert EmailToGravatarTransform.category() == "Email"
def test_key(self):
- """Test the scanner key"""
- assert EmailToGravatarScanner.key() == "email"
+ """Test the transform key"""
+ assert EmailToGravatarTransform.key() == "email"
def test_input_schema(self):
"""Test the input schema generation"""
- schema = EmailToGravatarScanner.input_schema()
+ schema = EmailToGravatarTransform.input_schema()
assert schema["type"] == "Email"
assert "properties" in schema
# Check that email property is present
@@ -36,7 +36,7 @@ class TestEmailToGravatarScanner:
def test_output_schema(self):
"""Test the output schema generation"""
- schema = EmailToGravatarScanner.output_schema()
+ schema = EmailToGravatarTransform.output_schema()
assert schema["type"] == "Gravatar"
assert "properties" in schema
# Check that required properties are present
@@ -55,7 +55,7 @@ class TestEmailToGravatarScanner:
]
- result = scanner.preprocess(emails)
+ result = transform.preprocess(emails)
assert len(result) == 2
assert all(isinstance(email, Email) for email in result)
assert result[0].email == "[email protected]"
@@ -67,7 +67,7 @@ class TestEmailToGravatarScanner:
{"email": "[email protected]"},
{"email": "[email protected]"},
]
- result = scanner.preprocess(emails)
+ result = transform.preprocess(emails)
assert len(result) == 2
assert all(isinstance(email, Email) for email in result)
assert result[0].email == "[email protected]"
@@ -79,7 +79,7 @@ class TestEmailToGravatarScanner:
Email(email="[email protected]"),
Email(email="[email protected]"),
]
- result = scanner.preprocess(emails)
+ result = transform.preprocess(emails)
assert len(result) == 2
assert all(isinstance(email, Email) for email in result)
assert result[0].email == "[email protected]"
@@ -92,7 +92,7 @@ class TestEmailToGravatarScanner:
{"email": "[email protected]"},
Email(email="[email protected]"),
]
- result = scanner.preprocess(emails)
+ result = transform.preprocess(emails)
assert len(result) == 3
assert all(isinstance(email, Email) for email in result)
assert result[0].email == "[email protected]"
@@ -108,7 +108,7 @@ class TestEmailToGravatarScanner:
None,
123,
]
- result = scanner.preprocess(emails)
+ result = transform.preprocess(emails)
# The preprocess method doesn't validate email format, it just creates Email objects
# for valid string inputs and dicts with email key
assert len(result) == 2 # "not-an-email" and "invalid-email" are processed
@@ -117,7 +117,7 @@ class TestEmailToGravatarScanner:
def test_preprocess_empty_list(self):
"""Test preprocessing with empty list"""
- result = scanner.preprocess([])
+ result = transform.preprocess([])
assert result == []
@patch("requests.get")
@@ -129,7 +129,7 @@ class TestEmailToGravatarScanner:
mock_get.return_value = mock_response
emails = [Email(email="[email protected]")]
- result = scanner.scan(emails)
+ result = transform.scan(emails)
assert len(result) == 1
assert isinstance(result[0], Gravatar)
@@ -145,7 +145,7 @@ class TestEmailToGravatarScanner:
mock_get.return_value = mock_response
emails = [Email(email="[email protected]")]
- result = scanner.scan(emails)
+ result = transform.scan(emails)
assert len(result) == 0
@@ -156,7 +156,7 @@ class TestEmailToGravatarScanner:
mock_get.side_effect = Exception("Network error")
emails = [Email(email="[email protected]")]
- result = scanner.scan(emails)
+ result = transform.scan(emails)
assert len(result) == 0
@@ -173,7 +173,7 @@ class TestEmailToGravatarScanner:
Email(email="[email protected]"),
Email(email="[email protected]"),
]
- result = scanner.scan(emails)
+ result = transform.scan(emails)
assert len(result) == 3
assert all(isinstance(gravatar, Gravatar) for gravatar in result)
@@ -200,7 +200,7 @@ class TestEmailToGravatarScanner:
Email(email="[email protected]"),
Email(email="[email protected]"),
]
- result = scanner.scan(emails)
+ result = transform.scan(emails)
# Should get 1 result for the first email (success) and 0 for the second (failure)
assert len(result) == 1
@@ -210,7 +210,7 @@ class TestEmailToGravatarScanner:
"""Test postprocessing with Neo4j connection"""
# Mock Neo4j connection
mock_neo4j = Mock()
- scanner_with_neo4j = EmailToGravatarScanner(
+ transform_with_neo4j = EmailToGravatarTransform(
"sketch_123", "scan_123", neo4j_conn=mock_neo4j
)
@@ -223,7 +223,7 @@ class TestEmailToGravatarScanner:
Email(email="[email protected]"),
]
- result = scanner_with_neo4j.postprocess(gravatars, original_input)
+ result = transform_with_neo4j.postprocess(gravatars, original_input)
# Verify Neo4j queries were executed
assert mock_neo4j.query.call_count == 2
@@ -238,7 +238,7 @@ class TestEmailToGravatarScanner:
]
original_input = [Email(email="[email protected]")]
- result = scanner.postprocess(gravatars, original_input)
+ result = transform.postprocess(gravatars, original_input)
# Should return results unchanged
assert result == gravatars
@@ -250,7 +250,7 @@ class TestEmailToGravatarScanner:
]
original_input = [] # Empty list
- result = scanner.postprocess(gravatars, original_input)
+ result = transform.postprocess(gravatars, original_input)
# Should handle gracefully and return results
assert result == gravatars
@@ -263,7 +263,7 @@ class TestEmailToGravatarScanner:
# The postprocess method doesn't handle None input properly
# Let's test with an empty list instead
- result = scanner.postprocess(gravatars, [])
+ result = transform.postprocess(gravatars, [])
# Should handle gracefully and return results
assert result == gravatars
@@ -277,7 +277,7 @@ class TestEmailToGravatarScanner:
mock_get.return_value = mock_response
emails = ["[email protected]"]
- result = scanner.execute(emails)
+ result = transform.execute(emails)
assert len(result) == 1
assert isinstance(result[0], Gravatar)
@@ -295,9 +295,9 @@ class TestEmailToGravatarScanner:
mock_response.status_code = 200
mock_get.return_value = mock_response
- result = scanner.execute(emails)
+ result = transform.execute(emails)
- # The scanner processes any string as an email, so it will create Email objects
+ # The transform processes any string as an email, so it will create Email objects
# and attempt to get gravatars for them
assert len(result) == 2
assert all(isinstance(gravatar, Gravatar) for gravatar in result)
@@ -313,7 +313,7 @@ class TestEmailToGravatarScanner:
mock_get.return_value = mock_response
emails = [Email(email=email)]
- result = scanner.scan(emails)
+ result = transform.scan(emails)
assert len(result) == 1
assert result[0].hash == expected_hash
@@ -330,7 +330,7 @@ class TestEmailToGravatarScanner:
mock_get.return_value = mock_response
emails = [Email(email=email)]
- result = scanner.scan(emails)
+ result = transform.scan(emails)
assert len(result) == 1
assert str(result[0].src) == expected_url
diff --git a/flowsint-transforms/tests/scanners/email/to_leaks.py b/flowsint-transforms/tests/transforms/email/to_leaks.py
similarity index 82%
rename from flowsint-transforms/tests/scanners/email/to_leaks.py
rename to flowsint-transforms/tests/transforms/email/to_leaks.py
index 77d68e37..b05113d6 100644
--- a/flowsint-transforms/tests/scanners/email/to_leaks.py
+++ b/flowsint-transforms/tests/transforms/email/to_leaks.py
@@ -1,22 +1,22 @@
import pytest
from unittest.mock import patch, MagicMock
-from flowsint_transforms.emails.to_leaks import EmailToBreachesScanner
+from flowsint_transforms.emails.to_leaks import EmailToBreachesTransform
from flowsint_types.email import Email
from flowsint_types.breach import Breach
-scanner = EmailToBreachesScanner("sketch_123", "scan_123")
+transform = EmailToBreachesTransform("sketch_123", "scan_123")
def test_transform_name():
- assert EmailToBreachesScanner.name() == "to_leaks"
+ assert EmailToBreachesTransform.name() == "to_leaks"
-def test_scanner_category():
- assert EmailToBreachesScanner.category() == "Email"
+def test_transform_category():
+ assert EmailToBreachesTransform.category() == "Email"
-def test_scanner_key():
- assert EmailToBreachesScanner.key() == "email"
+def test_transform_key():
+ assert EmailToBreachesTransform.key() == "email"
def test_preprocess_string_emails():
@@ -24,7 +24,7 @@ def test_preprocess_string_emails():
]
- result = scanner.preprocess(emails)
+ result = transform.preprocess(emails)
expected_emails = [Email(email=email) for email in emails]
assert result == expected_emails
@@ -34,7 +34,7 @@ def test_preprocess_dict_emails():
{"email": "[email protected]"},
{"email": "[email protected]"},
]
- result = scanner.preprocess(emails)
+ result = transform.preprocess(emails)
expected_emails = [Email(email=email["email"]) for email in emails]
assert result == expected_emails
@@ -44,7 +44,7 @@ def test_preprocess_email_objects():
Email(email="[email protected]"),
Email(email="[email protected]"),
]
- result = scanner.preprocess(emails)
+ result = transform.preprocess(emails)
assert result == emails
@@ -55,7 +55,7 @@ def test_preprocess_mixed_formats():
Email(email="[email protected]"),
{"invalid_key": "[email protected]"},
]
- result = scanner.preprocess(emails)
+ result = transform.preprocess(emails)
result_emails = [email.email for email in result]
assert "[email protected]" in result_emails
@@ -76,7 +76,7 @@ def test_scan_successful_response(mock_get):
mock_get.return_value = mock_response
emails = [Email(email="[email protected]")]
- result = scanner.scan(emails)
+ result = transform.scan(emails)
assert len(result) == 2
assert isinstance(result[0], Breach)
@@ -95,7 +95,7 @@ def test_scan_no_breaches_found(mock_get):
mock_get.return_value = mock_response
emails = [Email(email="[email protected]")]
- result = scanner.scan(emails)
+ result = transform.scan(emails)
assert len(result) == 0
@@ -106,7 +106,7 @@ def test_scan_api_error(mock_get):
mock_get.side_effect = Exception("API Error")
emails = [Email(email="[email protected]")]
- result = scanner.scan(emails)
+ result = transform.scan(emails)
assert len(result) == 0
@@ -123,7 +123,7 @@ def test_scan_missing_name_field(mock_get):
mock_get.return_value = mock_response
emails = [Email(email="[email protected]")]
- result = scanner.scan(emails)
+ result = transform.scan(emails)
assert len(result) == 2
assert result[0].name == "unknown" # Should default to "unknown"
@@ -134,16 +134,16 @@ def test_scan_missing_name_field(mock_get):
@patch("src.transforms.emails.to_leaks.HIBP_API_KEY", None)
def test_scan_no_api_key():
- """Test that scanner raises ValueError when HIBP_API_KEY is not set."""
+ """Test that transform raises ValueError when HIBP_API_KEY is not set."""
emails = [Email(email="[email protected]")]
with pytest.raises(ValueError, match="HIBP_API_KEY not set"):
- scanner.scan(emails)
+ transform.scan(emails)
def test_postprocess():
# Test postprocess method with mocked neo4j connection
- scanner.neo4j_conn = MagicMock()
+ transform.neo4j_conn = MagicMock()
# Create breach objects with the new structure
breach1 = Breach(
@@ -164,7 +164,7 @@ def test_postprocess():
breaches = [breach1, breach2]
original_input = [Email(email="[email protected]")]
- result = scanner.postprocess(breaches, original_input)
+ result = transform.postprocess(breaches, original_input)
assert result == breaches
# Verify that neo4j queries were called:
@@ -172,4 +172,4 @@ def test_postprocess():
# - 1 email node creation query
# - 2 relationship creation queries
# Total: 5 queries
- assert scanner.neo4j_conn.query.call_count == 5
+ assert transform.neo4j_conn.query.call_count == 5
diff --git a/flowsint-transforms/tests/scanners/ip/__init__.py b/flowsint-transforms/tests/transforms/ip/__init__.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/ip/__init__.py
rename to flowsint-transforms/tests/transforms/ip/__init__.py
diff --git a/flowsint-transforms/tests/scanners/ip/asn_to_cidrs.py b/flowsint-transforms/tests/transforms/ip/asn_to_cidrs.py
similarity index 82%
rename from flowsint-transforms/tests/scanners/ip/asn_to_cidrs.py
rename to flowsint-transforms/tests/transforms/ip/asn_to_cidrs.py
index fc40c860..4cc7a05e 100644
--- a/flowsint-transforms/tests/scanners/ip/asn_to_cidrs.py
+++ b/flowsint-transforms/tests/transforms/ip/asn_to_cidrs.py
@@ -1,7 +1,7 @@
-from flowsint_transforms.ips.asn_to_cidrs import AsnToCidrsScanner
+from flowsint_transforms.ips.asn_to_cidrs import AsnToCidrsTransform
from flowsint_types.asn import ASN
-scanner = AsnToCidrsScanner("sketch_123", "scan_123")
+transform = AsnToCidrsTransform("sketch_123", "scan_123")
def test_preprocess_valid_asns():
@@ -9,7 +9,7 @@ def test_preprocess_valid_asns():
ASN(number=15169),
ASN(number=13335),
]
- result = scanner.preprocess(asns)
+ result = transform.preprocess(asns)
result_numbers = [asn.number for asn in result]
expected_numbers = [asn.number for asn in asns]
@@ -22,7 +22,7 @@ def test_unprocessed_valid_asns():
"15169",
"13335",
]
- result = scanner.preprocess(asns)
+ result = transform.preprocess(asns)
result_asns = [asn for asn in result]
expected_asns = [ASN(number=int(asn)) for asn in asns]
assert result_asns == expected_asns
@@ -34,7 +34,7 @@ def test_preprocess_invalid_asns():
ASN(number=999999999999), # Invalid ASN number
ASN(number=13335),
]
- result = scanner.preprocess(asns)
+ result = transform.preprocess(asns)
result_numbers = [asn.number for asn in result]
assert 15169 in result_numbers
@@ -49,7 +49,7 @@ def test_preprocess_multiple_formats():
ASN(number=13335),
"15169",
]
- result = scanner.preprocess(asns)
+ result = transform.preprocess(asns)
result_numbers = [asn.number for asn in result]
assert 15169 in result_numbers
@@ -60,8 +60,8 @@ def test_preprocess_multiple_formats():
def test_schemas():
- input_schema = scanner.input_schema()
- output_schema = scanner.output_schema()
+ input_schema = transform.input_schema()
+ output_schema = transform.output_schema()
# Input schema should have number field
assert "properties" in input_schema
diff --git a/flowsint-transforms/tests/scanners/ip/cidr_to_ips.py b/flowsint-transforms/tests/transforms/ip/cidr_to_ips.py
similarity index 86%
rename from flowsint-transforms/tests/scanners/ip/cidr_to_ips.py
rename to flowsint-transforms/tests/transforms/ip/cidr_to_ips.py
index 0803c9da..1aac0493 100644
--- a/flowsint-transforms/tests/scanners/ip/cidr_to_ips.py
+++ b/flowsint-transforms/tests/transforms/ip/cidr_to_ips.py
@@ -1,10 +1,10 @@
-from flowsint_transforms.ips.cidr_to_ips import CidrToIpsScanner
+from flowsint_transforms.ips.cidr_to_ips import CidrToIpsTransform
from flowsint_types.cidr import CIDR
from flowsint_types.ip import Ip
from tests.logger import TestLogger
logger = TestLogger()
-scanner = CidrToIpsScanner("sketch_123", "scan_123", logger)
+transform = CidrToIpsTransform("sketch_123", "scan_123", logger)
def test_preprocess_valid_cidrs():
@@ -12,7 +12,7 @@ def test_preprocess_valid_cidrs():
CIDR(network="8.8.8.0/24"),
CIDR(network="1.1.1.0/24"),
]
- result = scanner.preprocess(cidrs)
+ result = transform.preprocess(cidrs)
result_networks = [cidr.network for cidr in result]
expected_networks = [cidr.network for cidr in cidrs]
@@ -25,7 +25,7 @@ def test_preprocess_unprocessed_valid_cidrs():
"8.8.8.0/24",
"1.1.1.0/24",
]
- result = scanner.preprocess(cidrs)
+ result = transform.preprocess(cidrs)
result_cidrs = [c for c in result]
expected_cidrs = [CIDR(network=c) for c in cidrs]
assert result_cidrs == expected_cidrs
@@ -37,7 +37,7 @@ def test_preprocess_invalid_cidrs():
"invalid-cidr",
"not-a-cidr",
]
- result = scanner.preprocess(cidrs)
+ result = transform.preprocess(cidrs)
result_networks = [str(cidr.network) for cidr in result]
assert "8.8.8.0/24" in result_networks
assert "invalid-cidr" not in result_networks
@@ -51,7 +51,7 @@ def test_preprocess_multiple_formats():
CIDR(network="9.9.9.0/24"),
"InvalidCIDR",
]
- result = scanner.preprocess(cidrs)
+ result = transform.preprocess(cidrs)
result_networks = [str(cidr.network) for cidr in result]
assert "8.8.8.0/24" in result_networks
assert "9.9.9.0/24" in result_networks
@@ -75,11 +75,11 @@ def test_scan_extracts_ips(monkeypatch):
assert "-ptr" in cmd
return MockSubprocessResult(mock_dnsx_output)
- # Patch the subprocess call in the scanner
+ # Patch the subprocess call in the transform
monkeypatch.setattr("subprocess.run", mock_subprocess_run)
input_data = [CIDR(network="8.35.200.0/24")]
- ips = scanner.scan(input_data)
+ ips = transform.scan(input_data)
assert isinstance(ips, list)
assert len(ips) == 4
@@ -103,7 +103,7 @@ def test_scan_handles_empty_output(monkeypatch):
monkeypatch.setattr("subprocess.run", mock_subprocess_run)
input_data = [CIDR(network="8.8.8.0/24")]
- ips = scanner.scan(input_data)
+ ips = transform.scan(input_data)
assert isinstance(ips, list)
assert len(ips) == 0
@@ -116,7 +116,7 @@ def test_scan_handles_subprocess_exception(monkeypatch):
monkeypatch.setattr("subprocess.run", mock_subprocess_run)
input_data = [CIDR(network="8.8.8.0/24")]
- ips = scanner.scan(input_data)
+ ips = transform.scan(input_data)
assert isinstance(ips, list)
assert len(ips) == 0
diff --git a/flowsint-transforms/tests/scanners/ip/ip_to_asn.py b/flowsint-transforms/tests/transforms/ip/ip_to_asn.py
similarity index 89%
rename from flowsint-transforms/tests/scanners/ip/ip_to_asn.py
rename to flowsint-transforms/tests/transforms/ip/ip_to_asn.py
index 091017e7..4768dd23 100644
--- a/flowsint-transforms/tests/scanners/ip/ip_to_asn.py
+++ b/flowsint-transforms/tests/transforms/ip/ip_to_asn.py
@@ -1,14 +1,14 @@
import json
from unittest.mock import Mock
-from flowsint_transforms.ips.ip_to_asn import IpToAsnScanner
+from flowsint_transforms.ips.ip_to_asn import IpToAsnTransform
from flowsint_types.ip import Ip
from flowsint_types.asn import ASN
from flowsint_types.cidr import CIDR
from tests.logger import TestLogger
logger = TestLogger()
-# The scanner will get a mock logger from conftest.py automatically
-scanner = IpToAsnScanner("sketch_123", "scan_123", logger)
+# The transform will get a mock logger from conftest.py automatically
+transform = IpToAsnTransform("sketch_123", "scan_123", logger)
def test_preprocess_valid_ips():
@@ -16,7 +16,7 @@ def test_preprocess_valid_ips():
Ip(address="8.8.8.8"),
Ip(address="1.1.1.1"),
]
- result = scanner.preprocess(ips)
+ result = transform.preprocess(ips)
result_addresses = [ip.address for ip in result]
expected_addresses = [ip.address for ip in ips]
@@ -29,7 +29,7 @@ def test_unprocessed_valid_ips():
"8.8.8.8",
"1.1.1.1",
]
- result = scanner.preprocess(ips)
+ result = transform.preprocess(ips)
result_ips = [ip for ip in result]
expected_ips = [Ip(address=ip) for ip in ips]
assert result_ips == expected_ips
@@ -41,7 +41,7 @@ def test_preprocess_invalid_ips():
Ip(address="invalid_ip"),
Ip(address="192.168.1.1"),
]
- result = scanner.preprocess(ips)
+ result = transform.preprocess(ips)
result_addresses = [ip.address for ip in result]
assert "8.8.8.8" in result_addresses
@@ -56,7 +56,7 @@ def test_preprocess_multiple_formats():
Ip(address="192.168.1.1"),
"10.0.0.1",
]
- result = scanner.preprocess(ips)
+ result = transform.preprocess(ips)
result_addresses = [ip.address for ip in result]
assert "8.8.8.8" in result_addresses
@@ -86,11 +86,11 @@ def test_scan_extracts_asn_info(monkeypatch):
assert input == "8.8.8.8"
return MockSubprocessResult(json.dumps(mock_asnmap_output))
- # Patch the subprocess call in the scanner
+ # Patch the subprocess call in the transform
monkeypatch.setattr("subprocess.run", mock_subprocess_run)
input_data = [Ip(address="8.8.8.8")]
- asns = scanner.scan(input_data)
+ asns = transform.scan(input_data)
assert isinstance(asns, list)
assert len(asns) == 1
@@ -118,7 +118,7 @@ def test_scan_handles_no_asn_found(monkeypatch):
monkeypatch.setattr("subprocess.run", mock_subprocess_run)
input_data = [Ip(address="192.168.1.1")]
- asns = scanner.scan(input_data)
+ asns = transform.scan(input_data)
assert isinstance(asns, list)
assert len(asns) == 1
@@ -138,7 +138,7 @@ def test_scan_handles_subprocess_exception(monkeypatch):
monkeypatch.setattr("subprocess.run", mock_subprocess_run)
input_data = [Ip(address="8.8.8.8")]
- asns = scanner.scan(input_data)
+ asns = transform.scan(input_data)
assert isinstance(asns, list)
assert len(asns) == 1
@@ -181,7 +181,7 @@ def test_scan_multiple_ips(monkeypatch):
monkeypatch.setattr("subprocess.run", mock_subprocess_run)
input_data = [Ip(address="8.8.8.8"), Ip(address="1.1.1.1")]
- asns = scanner.scan(input_data)
+ asns = transform.scan(input_data)
assert len(asns) == 2
@@ -195,8 +195,8 @@ def test_scan_multiple_ips(monkeypatch):
def test_schemas():
- input_schema = scanner.input_schema()
- output_schema = scanner.output_schema()
+ input_schema = transform.input_schema()
+ output_schema = transform.output_schema()
# Input schema should have address field
assert "properties" in input_schema
@@ -218,7 +218,7 @@ def test_schemas():
def test_postprocess_creates_neo4j_relationships(monkeypatch):
# Mock Neo4j connection
mock_neo4j = Mock()
- scanner.neo4j_conn = mock_neo4j
+ transform.neo4j_conn = mock_neo4j
input_data = [Ip(address="8.8.8.8")]
asn_results = [
@@ -230,7 +230,7 @@ def test_postprocess_creates_neo4j_relationships(monkeypatch):
)
]
- result = scanner.postprocess(asn_results, input_data)
+ result = transform.postprocess(asn_results, input_data)
# Verify Neo4j query was called
mock_neo4j.query.assert_called_once()
@@ -251,14 +251,14 @@ def test_postprocess_creates_neo4j_relationships(monkeypatch):
def test_postprocess_skips_unknown_asns(monkeypatch):
# Mock Neo4j connection
mock_neo4j = Mock()
- scanner.neo4j_conn = mock_neo4j
+ transform.neo4j_conn = mock_neo4j
input_data = [Ip(address="192.168.1.1")]
asn_results = [
ASN(number=0, name="Unknown", country="Unknown", cidrs=[]) # Unknown ASN
]
- result = scanner.postprocess(asn_results, input_data)
+ result = transform.postprocess(asn_results, input_data)
# Verify Neo4j query was NOT called for unknown ASN
mock_neo4j.query.assert_not_called()
diff --git a/flowsint-transforms/tests/scanners/ip/ip_to_infos.py b/flowsint-transforms/tests/transforms/ip/ip_to_infos.py
similarity index 84%
rename from flowsint-transforms/tests/scanners/ip/ip_to_infos.py
rename to flowsint-transforms/tests/transforms/ip/ip_to_infos.py
index cc44e2bd..8e137377 100644
--- a/flowsint-transforms/tests/scanners/ip/ip_to_infos.py
+++ b/flowsint-transforms/tests/transforms/ip/ip_to_infos.py
@@ -1,7 +1,7 @@
-from flowsint_transforms.ips.geolocation import IpToInfosScanner
+from flowsint_transforms.ips.geolocation import IpToInfosTransform
from flowsint_types.ip import Ip, Ip
-scanner = IpToInfosScanner("sketch_123", "scan_123")
+transform = IpToInfosTransform("sketch_123", "scan_123")
def test_preprocess_valid_ips():
@@ -9,7 +9,7 @@ def test_preprocess_valid_ips():
Ip(address="8.8.8.8"),
Ip(address="1.1.1.1"),
]
- result = scanner.preprocess(ips)
+ result = transform.preprocess(ips)
result_ips = [d.address for d in result]
expected_ips = [d.address for d in ips]
assert result_ips == expected_ips
@@ -20,7 +20,7 @@ def test_preprocess_string_ips():
"8.8.8.8",
"1.1.1.1",
]
- result = scanner.preprocess(ips)
+ result = transform.preprocess(ips)
result_ips = [d.address for d in result]
expected_ips = [d for d in ips]
assert [ip.address for ip in result] == expected_ips
@@ -32,7 +32,7 @@ def test_preprocess_invalid_ips():
Ip(address="invalid_ip"),
Ip(address="1.1.1.1"),
]
- result = scanner.preprocess(ips)
+ result = transform.preprocess(ips)
result_ips = [d.address for d in result]
assert "8.8.8.8" in result_ips
assert "1.1.1.1" in result_ips
@@ -46,7 +46,7 @@ def test_preprocess_multiple_formats():
Ip(address="1.1.1.1"),
"1.1.1.1",
]
- result = scanner.preprocess(ips)
+ result = transform.preprocess(ips)
result_ips = [d.address for d in result]
assert "8.8.8.8" in result_ips
assert "1.1.1.1" in result_ips
@@ -64,10 +64,10 @@ def test_scan_returns_ip(monkeypatch):
"isp": "Google LLC",
}
- monkeypatch.setattr(scanner, "get_location_data", mock_get_location_data)
+ monkeypatch.setattr(transform, "get_location_data", mock_get_location_data)
input_data = [Ip(address="8.8.8.8")]
- output = scanner.execute(input_data)
+ output = transform.execute(input_data)
assert isinstance(output, list)
assert isinstance(output[0], Ip)
assert output[0].address == "8.8.8.8"
@@ -77,8 +77,8 @@ def test_scan_returns_ip(monkeypatch):
def test_schemas():
- input_schema = scanner.input_schema()
- output_schema = scanner.output_schema()
+ input_schema = transform.input_schema()
+ output_schema = transform.output_schema()
assert input_schema == {
"type": "Ip",
"properties": [
diff --git a/flowsint-transforms/tests/scanners/organization/__init__.py b/flowsint-transforms/tests/transforms/organization/__init__.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/organization/__init__.py
rename to flowsint-transforms/tests/transforms/organization/__init__.py
diff --git a/flowsint-transforms/tests/scanners/organization/to_infos.py b/flowsint-transforms/tests/transforms/organization/to_infos.py
similarity index 74%
rename from flowsint-transforms/tests/scanners/organization/to_infos.py
rename to flowsint-transforms/tests/transforms/organization/to_infos.py
index 26eb180f..9c3cca16 100644
--- a/flowsint-transforms/tests/scanners/organization/to_infos.py
+++ b/flowsint-transforms/tests/transforms/organization/to_infos.py
@@ -1,12 +1,12 @@
-from flowsint_transforms.organizations.to_infos import OrgToInfosScanner
+from flowsint_transforms.organizations.to_infos import OrgToInfosTransform
from flowsint_types.organization import Organization
-scanner = OrgToInfosScanner("sketch_123", "scan_123")
+transform = OrgToInfosTransform("sketch_123", "scan_123")
def test_preprocess_valid_names():
data = [Organization(name="OpenAI"), {"name": "Inria"}, "OVH"]
- result = scanner.preprocess(data)
+ result = transform.preprocess(data)
result_names = [org.name for org in result]
assert result_names == ["OpenAI", "Inria", "OVH"]
@@ -20,10 +20,10 @@ def test_preprocess_valid_names():
# "",
# {"name": ""},
# ]
-# result = scanner.preprocess(data)
+# result = transform.preprocess(data)
# assert result == []
def test_execute():
- scanner.execute(["Karim Terrache"])
+ transform.execute(["Karim Terrache"])
assert True
diff --git a/flowsint-transforms/tests/scanners/social/__init__.py b/flowsint-transforms/tests/transforms/social/__init__.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/social/__init__.py
rename to flowsint-transforms/tests/transforms/social/__init__.py
diff --git a/flowsint-transforms/tests/scanners/social/maigret.py b/flowsint-transforms/tests/transforms/social/maigret.py
similarity index 76%
rename from flowsint-transforms/tests/scanners/social/maigret.py
rename to flowsint-transforms/tests/transforms/social/maigret.py
index e0c368a7..3a47aef9 100644
--- a/flowsint-transforms/tests/scanners/social/maigret.py
+++ b/flowsint-transforms/tests/transforms/social/maigret.py
@@ -1,8 +1,8 @@
from pathlib import Path
-from flowsint_transforms.socials.maigret import MaigretScanner
+from flowsint_transforms.socials.maigret import MaigretTransform
from flowsint_types.social import SocialProfile
-scanner = MaigretScanner("sketch_123", "scan_123")
+transform = MaigretTransform("sketch_123", "scan_123")
def test_unprocessed_valid_usernames():
@@ -10,7 +10,7 @@ def test_unprocessed_valid_usernames():
"toto123",
"DorianXd78",
]
- result = scanner.preprocess(usernames)
+ result = transform.preprocess(usernames)
result_usernames = [d for d in result]
expected_usernames = [SocialProfile(username=d) for d in usernames]
assert result_usernames == expected_usernames
@@ -22,7 +22,7 @@ def test_preprocess_invalid_usernames():
SocialProfile(username="DorianXd78_Official"),
SocialProfile(username="This is not a username"),
]
- result = scanner.preprocess(usernames)
+ result = transform.preprocess(usernames)
result_usernames = [d.username for d in result]
assert "toto123" in result_usernames
@@ -37,7 +37,7 @@ def test_preprocess_multiple_formats():
SocialProfile(username="DorianXd78_Official"),
"MySimpleUsername",
]
- result = scanner.preprocess(usernames)
+ result = transform.preprocess(usernames)
result_usernames = [d.username for d in result]
assert "toto123" in result_usernames
@@ -47,11 +47,11 @@ def test_preprocess_multiple_formats():
def test_parsing_invalid_output_file():
- results = scanner.parse_maigret_output("toto123", Path("/this/path/does/not/exist"))
+ results = transform.parse_maigret_output("toto123", Path("/this/path/does/not/exist"))
assert results == []
def test_parsing():
- results = scanner.parse_maigret_output("toto123", Path("/tmp/maigret_test.json"))
+ results = transform.parse_maigret_output("toto123", Path("/tmp/maigret_test.json"))
print(results)
assert len(results) == 2
diff --git a/flowsint-transforms/tests/scanners/website/__init__.py b/flowsint-transforms/tests/transforms/website/__init__.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/website/__init__.py
rename to flowsint-transforms/tests/transforms/website/__init__.py
diff --git a/flowsint-transforms/tests/scanners/website/to_links.py b/flowsint-transforms/tests/transforms/website/to_links.py
similarity index 80%
rename from flowsint-transforms/tests/scanners/website/to_links.py
rename to flowsint-transforms/tests/transforms/website/to_links.py
index 02c668ab..a2fdb270 100644
--- a/flowsint-transforms/tests/scanners/website/to_links.py
+++ b/flowsint-transforms/tests/transforms/website/to_links.py
@@ -39,34 +39,34 @@ class MockCrawler:
@pytest.mark.asyncio
async def test_website_to_links_real_time_neo4j_creation():
"""Test that Neo4j nodes are created in real-time during the callback."""
- scanner = WebsiteToLinks(sketch_id="test", scan_id="test")
+ transform = WebsiteToLinks(sketch_id="test", scan_id="test")
# Mock neo4j connection and methods
- scanner.neo4j_conn = Mock()
- scanner.create_node = Mock()
- scanner.create_relationship = Mock()
- scanner.log_graph_message = Mock()
+ transform.neo4j_conn = Mock()
+ transform.create_node = Mock()
+ transform.create_relationship = Mock()
+ transform.log_graph_message = Mock()
# Test input
websites = [Website(url="https://example.com")]
with patch("src.transforms.websites.to_links.Crawler", MockCrawler):
- results = await scanner.scan(websites)
+ results = await transform.scan(websites)
# Verify main website and domain nodes were created upfront
- scanner.create_node.assert_any_call(
+ transform.create_node.assert_any_call(
"website",
"url",
"https://example.com",
caption="https://example.com",
type="website",
)
- scanner.create_node.assert_any_call(
+ transform.create_node.assert_any_call(
"domain", "name", "example.com", caption="example.com", type="domain"
)
# Verify main website to domain relationship
- scanner.create_relationship.assert_any_call(
+ transform.create_relationship.assert_any_call(
"website",
"url",
"https://example.com",
@@ -77,14 +77,14 @@ async def test_website_to_links_real_time_neo4j_creation():
)
# Verify internal website nodes were created in callback
- scanner.create_node.assert_any_call(
+ transform.create_node.assert_any_call(
"website",
"url",
"https://example.com/page1",
caption="https://example.com/page1",
type="website",
)
- scanner.create_node.assert_any_call(
+ transform.create_node.assert_any_call(
"website",
"url",
"https://example.com/page2",
@@ -93,7 +93,7 @@ async def test_website_to_links_real_time_neo4j_creation():
)
# Verify internal website relationships
- scanner.create_relationship.assert_any_call(
+ transform.create_relationship.assert_any_call(
"website",
"url",
"https://example.com",
@@ -102,7 +102,7 @@ async def test_website_to_links_real_time_neo4j_creation():
"https://example.com/page1",
"LINKS_TO",
)
- scanner.create_relationship.assert_any_call(
+ transform.create_relationship.assert_any_call(
"website",
"url",
"https://example.com",
@@ -113,14 +113,14 @@ async def test_website_to_links_real_time_neo4j_creation():
)
# Verify external website nodes were created in callback
- scanner.create_node.assert_any_call(
+ transform.create_node.assert_any_call(
"website",
"url",
"https://external.com/page",
caption="https://external.com/page",
type="website",
)
- scanner.create_node.assert_any_call(
+ transform.create_node.assert_any_call(
"website",
"url",
"https://another-external.org/resource",
@@ -129,10 +129,10 @@ async def test_website_to_links_real_time_neo4j_creation():
)
# Verify external domain nodes were created in callback
- scanner.create_node.assert_any_call(
+ transform.create_node.assert_any_call(
"domain", "name", "external.com", caption="external.com", type="domain"
)
- scanner.create_node.assert_any_call(
+ transform.create_node.assert_any_call(
"domain",
"name",
"another-external.org",
@@ -141,7 +141,7 @@ async def test_website_to_links_real_time_neo4j_creation():
)
# Verify main website to external website relationships
- scanner.create_relationship.assert_any_call(
+ transform.create_relationship.assert_any_call(
"website",
"url",
"https://example.com",
@@ -150,7 +150,7 @@ async def test_website_to_links_real_time_neo4j_creation():
"https://external.com/page",
"LINKS_TO",
)
- scanner.create_relationship.assert_any_call(
+ transform.create_relationship.assert_any_call(
"website",
"url",
"https://example.com",
@@ -161,7 +161,7 @@ async def test_website_to_links_real_time_neo4j_creation():
)
# Verify external website to domain relationships
- scanner.create_relationship.assert_any_call(
+ transform.create_relationship.assert_any_call(
"website",
"url",
"https://external.com/page",
@@ -170,7 +170,7 @@ async def test_website_to_links_real_time_neo4j_creation():
"external.com",
"BELONGS_TO_DOMAIN",
)
- scanner.create_relationship.assert_any_call(
+ transform.create_relationship.assert_any_call(
"website",
"url",
"https://another-external.org/resource",
@@ -181,7 +181,7 @@ async def test_website_to_links_real_time_neo4j_creation():
)
# Verify main website to external domain relationships
- scanner.create_relationship.assert_any_call(
+ transform.create_relationship.assert_any_call(
"website",
"url",
"https://example.com",
@@ -190,7 +190,7 @@ async def test_website_to_links_real_time_neo4j_creation():
"external.com",
"LINKS_TO_DOMAIN",
)
- scanner.create_relationship.assert_any_call(
+ transform.create_relationship.assert_any_call(
"website",
"url",
"https://example.com",
@@ -204,13 +204,13 @@ async def test_website_to_links_real_time_neo4j_creation():
@pytest.mark.asyncio
async def test_website_to_links_error_handling_with_neo4j():
"""Test that main nodes are still created even when crawling fails."""
- scanner = WebsiteToLinks(sketch_id="test", scan_id="test")
+ transform = WebsiteToLinks(sketch_id="test", scan_id="test")
# Mock neo4j connection and methods
- scanner.neo4j_conn = Mock()
- scanner.create_node = Mock()
- scanner.create_relationship = Mock()
- scanner.log_graph_message = Mock()
+ transform.neo4j_conn = Mock()
+ transform.create_node = Mock()
+ transform.create_relationship = Mock()
+ transform.log_graph_message = Mock()
# Mock crawler that raises an exception
def mock_crawler_error(*args, **kwargs):
@@ -219,22 +219,22 @@ async def test_website_to_links_error_handling_with_neo4j():
websites = [Website(url="https://example.com")]
with patch("src.transforms.websites.to_links.Crawler", mock_crawler_error):
- results = await scanner.scan(websites)
+ results = await transform.scan(websites)
# Verify main website and domain nodes were still created despite error
- scanner.create_node.assert_any_call(
+ transform.create_node.assert_any_call(
"website",
"url",
"https://example.com",
caption="https://example.com",
type="website",
)
- scanner.create_node.assert_any_call(
+ transform.create_node.assert_any_call(
"domain", "name", "example.com", caption="example.com", type="domain"
)
# Verify main website to domain relationship was created
- scanner.create_relationship.assert_any_call(
+ transform.create_relationship.assert_any_call(
"website",
"url",
"https://example.com",
@@ -256,7 +256,7 @@ async def test_website_to_links_error_handling_with_neo4j():
def test_postprocess_simplified():
"""Test that postprocess now just returns results as-is."""
- scanner = WebsiteToLinks(sketch_id="test", scan_id="test")
+ transform = WebsiteToLinks(sketch_id="test", scan_id="test")
original_input = [Website(url="https://example.com")]
results = [
@@ -269,7 +269,7 @@ def test_postprocess_simplified():
}
]
- processed_results = scanner.postprocess(results, original_input)
+ processed_results = transform.postprocess(results, original_input)
# Should just return the same results since Neo4j work is done in real-time
assert processed_results == results
diff --git a/flowsint-transforms/tests/utils.py b/flowsint-transforms/tests/utils.py
deleted file mode 100644
index d9affb08..00000000
--- a/flowsint-transforms/tests/utils.py
+++ /dev/null
@@ -1,40 +0,0 @@
-from flowsint_core.core.scanner_base import build_params_model
-
-
-def test_build_params_model_valid():
- param_schema = [
- {
- "name": "ETHERSCAN_API_KEY",
- "type": "string",
- "description": "The Etherscan API key to use for the transaction lookup.",
- "required": True,
- },
- {
- "name": "url",
- "type": "string",
- "description": "Base URL for API",
- "required": False,
- "default": "https://api.etherscan.io/api",
- },
- ]
- ParamsModel = build_params_model(param_schema)
- validated_params = ParamsModel(ETHERSCAN_API_KEY="clef-123")
- assert validated_params.ETHERSCAN_API_KEY == "clef-123"
- assert validated_params.url == "https://api.etherscan.io/api"
-
-
-def test_build_params_model_invalid():
- param_schema = [
- {},
- {
- "name": "url",
- "type": "string",
- "description": "Base URL for API",
- "required": False,
- "default": "https://api.etherscan.io/api",
- },
- ]
- ParamsModel = build_params_model(param_schema)
- validated_params = ParamsModel(ETHERSCAN_API_KEY="clef-123")
- assert validated_params.ETHERSCAN_API_KEY == "clef-123"
- assert validated_params.url == "https://api.etherscan.io/api"