Skip to content

A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.

feat: whoxy scanners

dextmorgn59ecd842071d
diff --git a/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx b/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx
index 109bba80..65dae61a 100644
--- a/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx
@@ -17,7 +17,7 @@ export default function DetailsPanel({ node }: { node: GraphNode | null }) {
const { id: sketchId } = useParams({ strict: false })
if (!node) {
return (
- <div className="flex flex-col items-center justify-center h-full text-center p-8">
+ <div className="flex p-12 rounded-xl border flex-col items-center justify-center h-full text-center p-8">
<div className="w-16 h-16 bg-muted rounded-full flex items-center justify-center mb-4">
<MousePointer className="h-8 w-8 text-muted-foreground" />
</div>
diff --git a/flowsint-app/src/renderer/src/components/graphs/details-panel/neighbors.tsx b/flowsint-app/src/renderer/src/components/graphs/details-panel/neighbors.tsx
index 5c047cd7..5d2a6f40 100644
--- a/flowsint-app/src/renderer/src/components/graphs/details-panel/neighbors.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/details-panel/neighbors.tsx
@@ -27,6 +27,7 @@ const NeighborsGraph = ({ sketchId, nodeId }: { sketchId: string, nodeId: string
showIcons={true}
backgroundColor="transparent"
onGraphRef={() => { }}
+ instanceId="neighbors"
/>
</>}
</div>
diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx
index dab6878a..e6c0f039 100644
--- a/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx
@@ -1,5 +1,5 @@
import { useGraphStore } from '@/stores/graph-store'
-import React, { useEffect, useRef, useCallback } from 'react'
+import React, { useRef, useCallback } from 'react'
import GraphViewer from './graph-viewer'
import ContextMenu from './context-menu'
@@ -15,13 +15,6 @@ const GraphMain = () => {
const [menu, setMenu] = React.useState<any>(null)
- // // Handle current node centering
- // useEffect(() => {
- // if (!currentNode || !graphRef.current) return
- // graphRef.current.centerAt(currentNode.position.x, currentNode.position.x, 500)
- // graphRef.current.zoom(5, 500)
- // }, [currentNode])
-
const handleNodeClick = useCallback((node: any) => {
toggleNodeSelection(node, false)
}, [toggleNodeSelection])
diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx
index 945be343..0485090d 100644
--- a/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx
@@ -1,5 +1,5 @@
import { useLoaderData } from '@tanstack/react-router'
-import { useEffect, useRef, memo, useState, lazy, Suspense, useMemo } from 'react'
+import { useEffect, useRef, memo, useState, lazy, Suspense } from 'react'
import { useGraphStore, type GraphNode, type GraphEdge } from '@/stores/graph-store'
import { Toolbar } from './toolbar'
import { cn } from '@/lib/utils'
@@ -56,48 +56,6 @@ const GraphPanel = ({ graphData, isLoading }: GraphPanelProps) => {
})
const [isDraggingOver, setIsDraggingOver] = useState(false)
- // Determine which graph component to use based on node count with debouncing
- const [stableGraphType, setStableGraphType] = useState<'cosmograph' | 'force-graph' | null>(null)
-
- const shouldUseCosmograph = useMemo(() => {
- if (!nodes || nodes.length === 0) return false
- return nodes.length > NODE_COUNT_THRESHOLD
- }, [nodes?.length])
-
- // Debounce graph type changes to prevent rapid switching
- useEffect(() => {
- if (!nodes || nodes.length === 0) {
- setStableGraphType(null)
- return
- }
-
- const newType = shouldUseCosmograph ? 'cosmograph' : 'force-graph'
-
- // Only update if the type actually changed
- if (stableGraphType !== newType) {
- const timer = setTimeout(() => {
- setStableGraphType(newType)
- }, 500) // 500ms delay to prevent rapid switching
-
- return () => clearTimeout(timer)
- }
-
- // Return empty cleanup function for the case when type hasn't changed
- return () => { }
- }, [shouldUseCosmograph, stableGraphType, nodes?.length])
-
- const graphComponentKey = stableGraphType || 'loading'
-
- // Memoize the graph component to prevent unnecessary re-renders
- const graphComponent = useMemo(() => {
- if (stableGraphType === 'cosmograph') {
- return <Graph />
- } else if (stableGraphType === 'force-graph') {
- return <GraphMain />
- }
- return null
- }, [stableGraphType])
-
useEffect(() => {
if (graphData?.nds && graphData?.rls) {
updateGraphData(graphData.nds, graphData.rls)
@@ -170,28 +128,21 @@ const GraphPanel = ({ graphData, isLoading }: GraphPanelProps) => {
</div>
</div>
}>
- {/* Only render graph components when we have actual data */}
- {nodes && nodes.length > 0 && stableGraphType ? (
- <div key={graphComponentKey} className="h-full w-full flex relative outline-2 outline-transparent bg-background"
- >
- {view === "table" && <NodesTable />}
+ {nodes?.length > NODE_COUNT_THRESHOLD ? (
+ <>{view === "table" && <NodesTable />}
+ {["force", "hierarchy"].includes(view) && <Graph />}
{view === "map" && <MapPanel />}
{view === "relationships" && <RelationshipsTable />}
- {["force", "hierarchy"].includes(view) && graphComponent}
- </div>
+ </>
) : (
- // Show loading state while waiting for data or graph type to stabilize
- <div className="h-full w-full flex items-center justify-center">
- <div className="text-center flex flex-col items-center gap-2">
- <Loader />
- <span className="text-muted-foreground">
- {!nodes || nodes.length === 0
- ? 'Loading data...'
- : 'Preparing data...'
- }
- </span>
- </div>
- </div>
+ <>
+ {/* {view === "hierarchy" && <Wall isLoading={false} isRefetching={false} />} */}
+ {/* {view === "force" && <GraphMain />} */}
+ {["force", "hierarchy"].includes(view) && <GraphMain />}
+ {view === "table" && <NodesTable />}
+ {view === "map" && <MapPanel />}
+ {view === "relationships" && <RelationshipsTable />}
+ </>
)}
</Suspense>
<DragOverlay isDragging={isDraggingOver} />
diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx
index 7714afca..710fc746 100644
--- a/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx
@@ -8,6 +8,7 @@ import ForceGraph2D from 'react-force-graph-2d';
import { Button } from '../ui/button';
import { useTheme } from '@/components/theme-provider'
import { GRAPH_COLORS } from '../flows/scanner-data';
+import { Share2 } from 'lucide-react';
function truncateText(text: string, limit: number = 16) {
if (text.length <= limit)
@@ -31,6 +32,7 @@ interface GraphViewerProps {
className?: string;
style?: React.CSSProperties;
onGraphRef?: (ref: any) => void;
+ instanceId?: string; // Add instanceId prop for instance-specific actions
}
const CONSTANTS = {
@@ -47,7 +49,15 @@ const CONSTANTS = {
MIN_FONT_SIZE: 0.5,
LINK_WIDTH: 1,
ARROW_SIZE: 8,
- ARROW_ANGLE: Math.PI / 6
+ ARROW_ANGLE: Math.PI / 6,
+ // New constants for layered label display
+ LABEL_LAYERS: [
+ { minZoom: 1.5, minWeight: 15, maxNodes: 20 }, // High-weight nodes first
+ { minZoom: 2.0, minWeight: 10, maxNodes: 50 }, // Medium-weight nodes
+ { minZoom: 2.5, minWeight: 5, maxNodes: 100 }, // Lower-weight nodes
+ { minZoom: 3.0, minWeight: 1, maxNodes: 200 }, // All remaining nodes
+ { minZoom: 4.0, minWeight: 0, maxNodes: Infinity } // All nodes at high zoom
+ ]
};
// Pre-computed constants
@@ -107,7 +117,8 @@ const GraphViewer: React.FC<GraphViewerProps> = ({
backgroundColor = 'transparent',
className = '',
style,
- onGraphRef
+ onGraphRef,
+ instanceId
}) => {
const [currentZoom, setCurrentZoom] = useState(1);
const [dimensions, setDimensions] = useState({ width: 0, height: 0 });
@@ -147,35 +158,50 @@ const GraphViewer: React.FC<GraphViewerProps> = ({
const initializeGraph = useCallback((graphInstance: any) => {
if (!graphInstance || isGraphReadyRef.current) return;
isGraphReadyRef.current = true;
- // Set up graph actions
- setActions({
- zoomIn: () => {
- const zoom = graphInstance.zoom();
- graphInstance.zoom(zoom * 1.5);
- },
- zoomOut: () => {
- const zoom = graphInstance.zoom();
- graphInstance.zoom(zoom * 0.75);
- },
- zoomToFit: () => {
- graphInstance.zoomToFit(400);
- }
- });
+
+ // Only set global actions if no instanceId is provided (for main graph)
+ if (!instanceId) {
+ setActions({
+ zoomIn: () => {
+ const zoom = graphInstance.zoom();
+ graphInstance.zoom(zoom * 1.5);
+ },
+ zoomOut: () => {
+ const zoom = graphInstance.zoom();
+ graphInstance.zoom(zoom * 0.75);
+ },
+ zoomToFit: () => {
+ graphInstance.zoomToFit(400);
+ }
+ });
+ }
// Call external ref callback
onGraphRef?.(graphInstance);
- }, [setActions, onGraphRef]);
+ }, [setActions, onGraphRef, instanceId]);
// Handle graph ref changes
useEffect(() => {
if (graphRef.current) {
initializeGraph(graphRef.current);
}
+
+ // Cleanup: reset actions when component unmounts (only for main graph)
+ return () => {
+ if (!instanceId && isGraphReadyRef.current) {
+ setActions({
+ zoomIn: () => { },
+ zoomOut: () => { },
+ zoomToFit: () => { },
+ });
+ isGraphReadyRef.current = false;
+ }
+ };
}, [initializeGraph]);
// Memoized rendering check
const shouldUseSimpleRendering = useMemo(() =>
- nodes.length > CONSTANTS.NODE_COUNT_THRESHOLD || currentZoom < 2.5
+ nodes.length > CONSTANTS.NODE_COUNT_THRESHOLD || currentZoom < 1.5
, [nodes.length, currentZoom]);
// Optimized graph data transformation with proper memoization dependencies
@@ -189,7 +215,7 @@ const GraphViewer: React.FC<GraphViewerProps> = ({
nodeColor: nodeColors[type] || GRAPH_COLORS.NODE_DEFAULT,
nodeSize: CONSTANTS.NODE_DEFAULT_SIZE,
nodeType: type,
- val: getSize(type),
+ val: CONSTANTS.NODE_DEFAULT_SIZE,
neighbors: [] as any[],
links: [] as any[]
};
@@ -248,17 +274,72 @@ const GraphViewer: React.FC<GraphViewerProps> = ({
if (view === "hierarchy") {
const { nodes: nds, edges: eds } = getDagreLayoutedElements(transformedNodes, transformedEdges);
return {
- nodes: nds.map((nd) => ({ ...nd, x: nd.position.x, y: nd.position.y })),
+ nodes: nds.map((nd) => ({
+ ...nd,
+ x: nd.position.x,
+ y: nd.position.y,
+ // Preserve the neighbors and links from the original transformed node
+ neighbors: nodeMap.get(nd.id)?.neighbors || [],
+ links: nodeMap.get(nd.id)?.links || []
+ })),
links: eds,
};
}
-
return {
nodes: transformedNodes,
links: transformedEdges,
};
}, [nodes, edges, nodeColors, getSize, view]);
+ // New function to determine which labels should be visible based on zoom and weight
+ const getVisibleLabels = useMemo(() => {
+ if (!showLabels) return new Set<string>();
+
+ // Find the appropriate layer for current zoom
+ const currentLayer = CONSTANTS.LABEL_LAYERS.find(layer => currentZoom >= layer.minZoom);
+ if (!currentLayer) return new Set<string>();
+
+ // Sort nodes by weight (number of connections) in descending order
+ const sortedNodes = [...graphData.nodes].sort((a, b) => {
+ const aWeight = a.neighbors?.length || 0;
+ const bWeight = b.neighbors?.length || 0;
+ return bWeight - aWeight; // Higher weight first
+ });
+
+ // Take only the nodes that meet the criteria for this layer
+ const visibleNodes = sortedNodes
+ .filter(node => {
+ const weight = node.neighbors?.length || 0;
+ return weight >= currentLayer.minWeight;
+ })
+ .slice(0, currentLayer.maxNodes);
+
+ return new Set(visibleNodes.map(node => node.id));
+ }, [graphData.nodes, currentZoom, showLabels]);
+
+ // Get current label layer info for display
+ const currentLabelLayer = useMemo(() => {
+ if (!showLabels) return null;
+ return CONSTANTS.LABEL_LAYERS.find(layer => currentZoom >= layer.minZoom);
+ }, [currentZoom, showLabels]);
+
+ // Get label count info for current layer
+ const labelInfo = useMemo(() => {
+ if (!currentLabelLayer || !showLabels) return null;
+
+ const totalNodes = graphData.nodes.length;
+ const visibleCount = getVisibleLabels.size;
+ const weightThreshold = currentLabelLayer.minWeight;
+
+ return {
+ visibleCount,
+ totalNodes,
+ weightThreshold,
+ zoomLevel: currentZoom,
+ layer: currentLabelLayer
+ };
+ }, [currentLabelLayer, showLabels, getVisibleLabels.size, graphData.nodes.length, currentZoom]);
+
// Event handlers with proper memoization
const handleNodeClick = useCallback((node: any) => {
onNodeClick?.(node);
@@ -320,6 +401,80 @@ const GraphViewer: React.FC<GraphViewerProps> = ({
setHighlightLinks(newHighlightLinks);
}, []);
+ // Add state for tooltip
+ const [tooltip, setTooltip] = useState<{
+ x: number;
+ y: number;
+ data: {
+ label: string,
+ connections: string
+ } | null;
+ visible: boolean;
+ }>({
+ x: 0,
+ y: 0,
+ data: null,
+ visible: false
+ });
+
+ // Enhanced node hover with tooltip
+ const handleNodeHoverWithTooltip = useCallback((node: any) => {
+ if (node) {
+ const weight = node.neighbors?.length || 0;
+ const label = node.nodeLabel || node.label || node.id;
+
+ // Position tooltip above the node using the graph's coordinate conversion
+ if (graphRef.current) {
+ try {
+ // Use the graph's built-in method to convert graph coordinates to screen coordinates
+ const screenCoords = graphRef.current.graph2ScreenCoords(node.x, node.y);
+
+ // Ensure tooltip stays within viewport bounds
+ const tooltipWidth = 120; // Approximate tooltip width
+ const tooltipHeight = 60; // Approximate tooltip height
+
+ let x = screenCoords.x;
+ let y = screenCoords.y - 30; // Position above the node
+
+ // Adjust X position if tooltip would go off-screen
+ if (x + tooltipWidth > window.innerWidth) {
+ x = window.innerWidth - tooltipWidth - 10;
+ }
+ if (x < 10) {
+ x = 10;
+ }
+
+ // Adjust Y position if tooltip would go off-screen
+ if (y < tooltipHeight + 10) {
+ y = screenCoords.y + 30; // Position below the node instead
+ }
+
+ setTooltip({
+ x,
+ y,
+ data: {
+ label,
+ connections: weight
+ },
+ visible: true
+ });
+ } catch (error) {
+ // Fallback: hide tooltip if coordinate conversion fails
+ setTooltip(prev => ({ ...prev, visible: false }));
+ }
+ }
+ } else {
+ setTooltip(prev => ({ ...prev, visible: false }));
+ }
+ handleNodeHover(node);
+ }, [handleNodeHover]);
+
+ // Remove mouse tracking since we don't need it anymore
+ // const [mousePosition, setMousePosition] = useState({ x: 0, y: 0 });
+ // const handleMouseMove = useCallback((event: React.MouseEvent) => {
+ // setMousePosition({ x: event.clientX, y: event.clientY });
+ // }, []);
+
const handleLinkHover = useCallback((link: any) => {
// Throttle hover updates to max 60fps
const now = Date.now();
@@ -353,7 +508,8 @@ const GraphViewer: React.FC<GraphViewerProps> = ({
// Optimized node rendering with proper icon caching
const renderNode = useCallback((node: any, ctx: CanvasRenderingContext2D, globalScale: number) => {
- const size = node.nodeSize * (settings.nodeSize.value / 100 + .4);
+ const size = Math.min((node.nodeSize + node.neighbors.length / 5), 20) * (settings.nodeSize.value / 100 + .4);
+ node.val = size / 5
const isHighlighted = highlightNodes.has(node.id);
const hasAnyHighlight = highlightNodes.size > 0 || highlightLinks.size > 0;
const isHovered = hoverNode === node.id;
@@ -392,13 +548,15 @@ const GraphViewer: React.FC<GraphViewerProps> = ({
}
}
- // Optimized label rendering
+ // Optimized label rendering with layered display
if (showLabels && globalScale > 3) {
const label = truncateText(node.nodeLabel || node.label || node.id, 58);
if (label) {
- // Only show labels for highlighted nodes when there's any highlighting
- // or show all labels when there's no highlighting
- if (hasAnyHighlight && !isHighlighted) {
+ // Check if this node's label should be visible in current layer
+ const shouldShowLabel = getVisibleLabels.has(node.id);
+
+ // Always show labels for highlighted nodes
+ if (!shouldShowLabel && !isHighlighted) {
return;
}
@@ -411,14 +569,22 @@ const GraphViewer: React.FC<GraphViewerProps> = ({
ctx.textAlign = 'center';
ctx.textBaseline = 'middle';
- ctx.fillStyle = isHighlighted ? color : `${color}2D`;
+
+ // Adjust opacity based on whether it's a highlighted node or just visible in current layer
+ if (isHighlighted) {
+ ctx.fillStyle = color;
+ } else {
+ // Fade out labels that are just visible due to layer, not highlighting
+ ctx.fillStyle = `${color}4D`; // 30% opacity
+ }
+
ctx.fillText(label, node.x, bgY + bgHeight / 2);
}
}
- }, [shouldUseSimpleRendering, showLabels, showIcons, settings.nodeSize.value, theme, highlightNodes, highlightLinks, hoverNode]);
+ }, [shouldUseSimpleRendering, showLabels, showIcons, settings.nodeSize.value, theme, highlightNodes, highlightLinks, hoverNode, getVisibleLabels]);
// Optimized link rendering with reduced canvas state changes
- const linkCanvasObject = useCallback((link: any, ctx: CanvasRenderingContext2D) => {
+ const renderLink = useCallback((link: any, ctx: CanvasRenderingContext2D) => {
const { source: start, target: end } = link;
// Early exit for unbound links
if (typeof start !== 'object' || typeof end !== 'object') return;
@@ -505,50 +671,50 @@ const GraphViewer: React.FC<GraphViewerProps> = ({
if (shouldUseSimpleRendering || !link.label) return;
// Only show labels for highlighted links when there's any highlighting
- if (hasAnyHighlight && !isHighlighted) {
- return;
- }
+ if (isHighlighted) {
- // Calculate label position and angle
- tempPos.x = (start.x + end.x) * 0.5;
- tempPos.y = (start.y + end.y) * 0.5;
- const dx = end.x - start.x;
- const dy = end.y - start.y;
- let textAngle = Math.atan2(dy, dx);
+ // Calculate label position and angle
+ tempPos.x = (start.x + end.x) * 0.5;
+ tempPos.y = (start.y + end.y) * 0.5;
- // Flip text for readability
- if (textAngle > CONSTANTS.HALF_PI || textAngle < -CONSTANTS.HALF_PI) {
- textAngle += textAngle > 0 ? -CONSTANTS.PI : CONSTANTS.PI;
- }
+ const dx = end.x - start.x;
+ const dy = end.y - start.y;
+ let textAngle = Math.atan2(dy, dx);
- // Measure and draw label
- ctx.font = LABEL_FONT_STRING;
- const textWidth = ctx.measureText(link.label).width;
- const padding = CONSTANTS.LABEL_FONT_SIZE * CONSTANTS.PADDING_RATIO;
+ // Flip text for readability
+ if (textAngle > CONSTANTS.HALF_PI || textAngle < -CONSTANTS.HALF_PI) {
+ textAngle += textAngle > 0 ? -CONSTANTS.PI : CONSTANTS.PI;
+ }
- tempDimensions[0] = textWidth + padding;
- tempDimensions[1] = CONSTANTS.LABEL_FONT_SIZE + padding;
+ // Measure and draw label
+ ctx.font = LABEL_FONT_STRING;
+ const textWidth = ctx.measureText(link.label).width;
+ const padding = CONSTANTS.LABEL_FONT_SIZE * CONSTANTS.PADDING_RATIO;
- const halfWidth = tempDimensions[0] * 0.5;
- const halfHeight = tempDimensions[1] * 0.5;
+ tempDimensions[0] = textWidth + padding;
+ tempDimensions[1] = CONSTANTS.LABEL_FONT_SIZE + padding;
- // Batch canvas operations
- ctx.save();
- ctx.translate(tempPos.x, tempPos.y);
- ctx.rotate(textAngle);
+ const halfWidth = tempDimensions[0] * 0.5;
+ const halfHeight = tempDimensions[1] * 0.5;
- // Background
- ctx.fillStyle = theme === "light" ? GRAPH_COLORS.BACKGROUND_LIGHT : GRAPH_COLORS.BACKGROUND_DARK;
- ctx.fillRect(-halfWidth, -halfHeight, tempDimensions[0], tempDimensions[1]);
+ // Batch canvas operations
+ ctx.save();
+ ctx.translate(tempPos.x, tempPos.y);
+ ctx.rotate(textAngle);
+
+ // Background
+ ctx.fillStyle = theme === "light" ? GRAPH_COLORS.BACKGROUND_LIGHT : GRAPH_COLORS.BACKGROUND_DARK;
+ ctx.fillRect(-halfWidth, -halfHeight, tempDimensions[0], tempDimensions[1]);
- // Text - follow same highlighting behavior as links
- ctx.fillStyle = isHighlighted ? GRAPH_COLORS.LINK_LABEL_HIGHLIGHTED : GRAPH_COLORS.LINK_LABEL_DEFAULT;
- ctx.textAlign = 'center';
- ctx.textBaseline = 'middle';
- ctx.fillText(link.label, 0, 0);
+ // Text - follow same highlighting behavior as links
+ ctx.fillStyle = isHighlighted ? GRAPH_COLORS.LINK_LABEL_HIGHLIGHTED : GRAPH_COLORS.LINK_LABEL_DEFAULT;
+ ctx.textAlign = 'center';
+ ctx.textBaseline = 'middle';
+ ctx.fillText(link.label, 0, 0);
- ctx.restore();
+ ctx.restore();
+ }
}, [shouldUseSimpleRendering, settings.linkWidth.value, settings.linkDirectionalArrowLength?.value, settings.linkDirectionalArrowRelPos?.value, settings.nodeSize.value, theme, highlightLinks, highlightNodes]);
// Container resize observer with debouncing
@@ -631,6 +797,7 @@ const GraphViewer: React.FC<GraphViewerProps> = ({
<div className="space-y-2 text-xs text-muted-foreground mb-6">
<p><strong>Tip:</strong> Use the search bar to find entities or import data to get started</p>
<p><strong>Explore:</strong> Try searching for domains, emails, or other entities</p>
+ <p><strong>Labels:</strong> Zoom in to see node labels progressively by connection weight</p>
</div>
<Button onClick={handleOpenNewAddItemDialog}>
<svg className="w-4 h-4 mr-2" fill="none" stroke="currentColor" viewBox="0 0 24 24">
@@ -653,16 +820,31 @@ const GraphViewer: React.FC<GraphViewerProps> = ({
height: height || '100%',
minHeight: 300,
minWidth: 300,
+ position: 'relative',
...style
}}
>
+ {tooltip.visible && (
+ <div
+ className="absolute z-20 bg-background border rounded-lg p-2 shadow-lg text-xs pointer-events-none"
+ style={{
+ left: tooltip.x,
+ top: tooltip.y,
+ transform: 'translate(-50%, -100%)' // Center horizontally on the node
+ }}
+ >
+ <div className="whitespace-pre-line truncate flex flex-col gap-1">
+ <span className='text-md font-semibold'>{tooltip.data?.label}</span>
+ <span className='flex items-center gap-1'><Share2 className='h-3 w-3 opacity-60' /> connections: <span className='font-semibold'>{tooltip.data?.connections}</span></span>
+ </div>
+ </div>
+ )}
<ForceGraph2D
ref={graphRef}
width={dimensions.width || width}
height={dimensions.height || height}
graphData={graphData}
- nodeLabel="label"
- nodeColor={node => shouldUseSimpleRendering ? node.nodeColor : GRAPH_COLORS.TRANSPARENT}
+ nodeLabel={() => ''} nodeColor={node => shouldUseSimpleRendering ? node.nodeColor : GRAPH_COLORS.TRANSPARENT}
nodeRelSize={6}
onNodeRightClick={handleNodeRightClick}
onNodeClick={handleNodeClick}
@@ -682,10 +864,10 @@ const GraphViewer: React.FC<GraphViewerProps> = ({
dagLevelDistance={settings.dagLevelDistance?.value}
backgroundColor={backgroundColor}
onZoom={(zoom) => setCurrentZoom(zoom.k)}
- linkCanvasObject={linkCanvasObject}
+ linkCanvasObject={renderLink}
enableNodeDrag={!shouldUseSimpleRendering}
autoPauseRedraw={false}
- onNodeHover={handleNodeHover}
+ onNodeHover={handleNodeHoverWithTooltip}
onLinkHover={handleLinkHover}
/>
</div>
diff --git a/flowsint-app/src/renderer/src/components/graphs/graph.tsx b/flowsint-app/src/renderer/src/components/graphs/graph.tsx
index 11da00da..35bb46e4 100644
--- a/flowsint-app/src/renderer/src/components/graphs/graph.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/graph.tsx
@@ -97,11 +97,12 @@ const GraphContent = memo(() => {
const cosmograph = useCosmograph()
const actionsSetRef = useRef(false)
+ const instanceId = useRef('cosmograph-main') // Add instance-specific ID
const handleLabelClick = useCallback((node: any, event: MouseEvent) => {
const multiSelect = event.ctrlKey || event.shiftKey || event.altKey
toggleNodeSelection(node, multiSelect)
- cosmograph?.cosmograph?.zoomToNode(node)
+ // cosmograph?.cosmograph?.zoomToNode(node)
}, [toggleNodeSelection, cosmograph?.cosmograph])
//@ts-ignore
@@ -113,9 +114,9 @@ const GraphContent = memo(() => {
if (event) {
const multiSelect = event.ctrlKey || event.shiftKey || event.altKey
toggleNodeSelection(clickedNode, multiSelect)
- cosmograph?.cosmograph?.zoomToNode(clickedNode)
+ // cosmograph?.cosmograph?.zoomToNode(clickedNode)
} else {
- cosmograph?.cosmograph?.zoomToNode(clickedNode)
+ // cosmograph?.cosmograph?.zoomToNode(clickedNode)
}
}, [toggleNodeSelection, clearSelectedNodes, cosmograph?.cosmograph])
@@ -162,19 +163,31 @@ const GraphContent = memo(() => {
cosmograph.cosmograph.restart()
actionsSetRef.current = true
}
- }, [cosmograph?.cosmograph, setActions])
+
+ // Cleanup: reset actions when component unmounts
+ return () => {
+ if (actionsSetRef.current) {
+ setActions({
+ zoomIn: () => {},
+ zoomOut: () => {},
+ zoomToFit: () => {},
+ })
+ actionsSetRef.current = false
+ }
+ }
+ }, [cosmograph?.cosmograph, setActions, instanceId.current])
//@ts-ignore
- useEffect(() => {
- if (currentNode && cosmograph?.cosmograph && isCosmographReady) {
- const timeoutId = setTimeout(() => {
- cosmograph.cosmograph?.zoomToNode(currentNode)
- cosmograph.cosmograph?.focusNode(currentNode)
- }, 100)
+ // useEffect(() => {
+ // if (currentNode && cosmograph?.cosmograph && isCosmographReady) {
+ // const timeoutId = setTimeout(() => {
+ // cosmograph.cosmograph?.zoomToNode(currentNode)
+ // cosmograph.cosmograph?.focusNode(currentNode)
+ // }, 100)
- return () => clearTimeout(timeoutId)
- }
- }, [currentNode, cosmograph?.cosmograph, isCosmographReady])
+ // return () => clearTimeout(timeoutId)
+ // }
+ // }, [currentNode, cosmograph?.cosmograph, isCosmographReady])
return (
<div className="relative h-full w-full">
diff --git a/flowsint-app/src/renderer/src/hooks/use-launch-flow.ts b/flowsint-app/src/renderer/src/hooks/use-launch-flow.ts
index d8c58188..05649b03 100644
--- a/flowsint-app/src/renderer/src/hooks/use-launch-flow.ts
+++ b/flowsint-app/src/renderer/src/hooks/use-launch-flow.ts
@@ -14,9 +14,11 @@ export function useLaunchFlow(askUser: boolean = false) {
if (!confirmed) return
}
const body = JSON.stringify({ values, sketch_id })
+ const sliced = values.slice(0, 2)
+ const left = values.length - sliced.length
toast.promise(flowService.launch(flow_id, body), {
loading: "Loading...",
- success: () => `Scan on "${values.join(",")}" has been launched.`,
+ success: () => `Flow ${flow_id} has been launched on "${sliced.join(",")}" ${left > 0 && ` and ${left} others`}.`,
error: () => `An error occurred launching flow.`,
})
return
diff --git a/flowsint-app/src/renderer/src/hooks/use-launch-transform.ts b/flowsint-app/src/renderer/src/hooks/use-launch-transform.ts
index 08c845c6..6152867c 100644
--- a/flowsint-app/src/renderer/src/hooks/use-launch-transform.ts
+++ b/flowsint-app/src/renderer/src/hooks/use-launch-transform.ts
@@ -14,9 +14,11 @@ export function useLaunchTransform(askUser: boolean = false) {
if (!confirmed) return
}
const body = JSON.stringify({ values, sketch_id })
+ const sliced = values.slice(0, 2)
+ const left = values.length - sliced.length
toast.promise(transformService.launch(transformName, body), {
loading: "Loading...",
- success: () => `Scan on "${values.join(",")}" has been launched.`,
+ success: () => `Transform ${transformName} has been launched on ${sliced.join(", ")}${left > 0 ? ` and ${left} others` : ""}.`,
error: () => `An error occurred launching transform.`,
})
return
diff --git a/flowsint-app/src/renderer/src/stores/node-display-settings.ts b/flowsint-app/src/renderer/src/stores/node-display-settings.ts
index f73c884d..5ae2f299 100644
--- a/flowsint-app/src/renderer/src/stores/node-display-settings.ts
+++ b/flowsint-app/src/renderer/src/stores/node-display-settings.ts
@@ -109,66 +109,67 @@ export const ITEM_TYPES: ItemType[] = [
"creditcard",
"alias",
"affiliation",
- "phrase"
+ "phrase",
]
const DEFAULT_COLORS: Record<ItemType, string> = {
- individual: "#E07A7A", // stronger pastel blue
- phone: "#5FC9B5", // teal-green
- location: "#E57373", // warm rose
- email: "#8E7CC3", // lavender purple
- ip: "#F4A261", // orange
- socialprofile: "#B569B9", // purple pink
- organization: "#BCA18A", // taupe
- vehicle: "#E1B84D", // golden wheat
+ individual: "#4C8EDA", // bleu moyen
+ phone: "#4CB5AE", // teal doux
+ location: "#E57373", // rouge rosé
+ email: "#8E7CC3", // violet pastel
+ ip: "#F4A261", // orange chaud
+ socialprofile: "#A76DAA", // mauve
+ organization: "#BCA18A", // taupe clair
+ vehicle: "#E1B84D", // jaune doux
car: "#BFAF7A", // olive beige
- motorcycle: "#A78B6C", // warm taupe
- boat: "#6FB1C5", // blue teal
- plane: "#C1A78E", // light brown
- website: "#D279A6", // dusty rose
- domain: "#66A892", // sage green
- subdomain: "#5AA1C8", // sky blue
- document: "#8F9CA3", // cool gray
- financial: "#E98973", // coral
- event: "#6DBBA2", // mint green
- device: "#E3A857", // peach orange
+ motorcycle: "#A78B6C", // brun taupe
+ boat: "#6FB1C5", // bleu teal
+ plane: "#C1A78E", // brun clair
+ website: "#D279A6", // rose pastel
+ domain: "#66A892", // vert sauge
+ subdomain: "#5AA1C8", // bleu ciel
+ document: "#8F9CA3", // gris bleuté
+ financial: "#E98973", // corail doux
+ event: "#6DBBA2", // vert menthe
+ device: "#E3A857", // orange sable
media: "#C97C73", // terracotta
- education: "#6C8CBF", // steel blue
- relationship: "#B18B84", // mauve brown
- online_activity: "#7EAD6F", // sage green
- digital_footprint: "#B97777", // brick
- username: "#8B83C1", // periwinkle purple
- credential: "#D4B030", // gold
- biometric: "#7F868D", // slate gray
- siret: "#7D8B99", // blue-gray
- siren: "#687684", // dark slate
- cryptowallet: "#D4B030", // gold yellow
- cryptotransaction: "#BFA750", // warm yellow
- cryptonft: "#A5BF50", // lime green
- asn: "#D97474", // warm peach
- cidr: "#80BF80", // mint green
- whois: "#9B6F9B", // lavender violet
- gravatar: "#6CB7CA", // pale cyan
- breach: "#CC7A7A", // warm rose
- webtracker: "#C7BF50", // warm yellow
- session: "#A8BF50", // lime green
- dns: "#80BF9F", // mint teal
- ssl: "#BFAF80", // warm sand
- message: "#897FC9", // lavender
- malware: "#4AA29E", // teal
- weapon: "#E98973", // coral brown
- script: "#A36FA3", // dusty violet
- reputation: "#6FA8DC", // steel blue
- risk: "#D97474", // dusty rose
- file: "#8F9CA3", // cool gray
- bank: "#D4B030", // gold
- creditcard: "#285E8E", // deep blue
+ education: "#6C8CBF", // bleu acier
+ relationship: "#B18B84", // brun rosé
+ online_activity: "#7EAD6F", // vert sauge
+ digital_footprint: "#B97777", // brique
+ username: "#8B83C1", // pervenche
+ credential: "#D4B030", // doré doux
+ biometric: "#7F868D", // gris ardoise
+ siret: "#7D8B99", // gris bleu
+ siren: "#687684", // gris foncé
+ cryptowallet: "#D4B030", // or clair
+ cryptotransaction: "#BFA750", // or chaud
+ cryptonft: "#A5BF50", // vert lime doux
+ asn: "#D97474", // pêche rosée
+ cidr: "#80BF80", // vert menthe
+ whois: "#9B6F9B", // violet doux
+ gravatar: "#6CB7CA", // cyan clair
+ breach: "#CC7A7A", // rose chaud
+ webtracker: "#C7BF50", // jaune doux
+ session: "#A8BF50", // lime atténué
+ dns: "#80BF9F", // vert teal clair
+ ssl: "#BFAF80", // sable chaud
+ message: "#897FC9", // violet lavande
+ malware: "#4AA29E", // teal saturé
+ weapon: "#E98973", // corail brun
+ script: "#A36FA3", // violet doux
+ reputation: "#6FA8DC", // bleu clair
+ risk: "#D97474", // rouge doux
+ file: "#8F9CA3", // gris bleuté
+ bank: "#D4B030", // or clair
+ creditcard: "#285E8E", // bleu profond
alias: "#A36FA3", // violet
- affiliation: "#66A892", // sage
- phrase: "#BFA77A" // warm beige
+ affiliation: "#66A892", // vert sauge
+ phrase: "#BFA77A", // beige chaud
};
+
// Définition des icônes par défaut pour chaque type d'élément
const DEFAULT_ICONS: Record<ItemType, string> = {
individual: "👤",
@@ -223,7 +224,7 @@ const DEFAULT_ICONS: Record<ItemType, string> = {
creditcard: "💳",
alias: "👤",
affiliation: "🤝",
- phrase: "��"
+ phrase: "��",
};
const DEFAULT_SIZES: Record<ItemType, number> = {
@@ -279,7 +280,7 @@ const DEFAULT_SIZES: Record<ItemType, number> = {
creditcard: 18, // Medium-large
alias: 12, // Small-medium
affiliation: 14, // Medium
- phrase: 10 // Small
+ phrase: 10, // Small
}
interface NodesDisplaySettingsState {
diff --git a/flowsint-core/pyproject.toml b/flowsint-core/pyproject.toml
index 7c97ae66..ad7fce9e 100644
--- a/flowsint-core/pyproject.toml
+++ b/flowsint-core/pyproject.toml
@@ -17,7 +17,6 @@ redis = "^5.0"
celery = "^5.3"
python-dotenv = "^1.0"
requests = "^2.31"
-docker = "^6.1"
networkx = "^2.6.3"
passlib = {extras = ["bcrypt"], version = "^1.7"}
python-jose = {extras = ["cryptography"], version = "^3.3"}
diff --git a/flowsint-core/src/flowsint_core/core/registry.py b/flowsint-core/src/flowsint_core/core/registry.py
index 156b8e4c..9d3a4eb7 100644
--- a/flowsint-core/src/flowsint_core/core/registry.py
+++ b/flowsint-core/src/flowsint_core/core/registry.py
@@ -11,6 +11,8 @@ from flowsint_transforms.domain.to_root_domain import DomainToRootDomain
from flowsint_transforms.domain.to_asn import DomainToAsnScanner
# IP-related scanners
+from flowsint_transforms.email.to_domains import EmailToDomainsScanner
+from flowsint_transforms.individual.to_domains import IndividualToDomainsScanner
from flowsint_transforms.ip.to_domain import ReverseResolveScanner
from flowsint_transforms.ip.to_geolocation import GeolocationScanner
@@ -24,6 +26,7 @@ from flowsint_transforms.asn.to_cidrs import AsnToCidrsScanner
from flowsint_transforms.cidr.to_ips import CidrToIpsScanner
# Social media scanners
+from flowsint_transforms.organization.to_domains import OrgToDomainsScanner
from flowsint_transforms.social.to_maigret import MaigretScanner
# Organization-related scanners
@@ -155,7 +158,7 @@ TransformRegistry.register(MaigretScanner)
# Organization-related scanners
TransformRegistry.register(OrgToAsnScanner)
TransformRegistry.register(OrgToInfosScanner)
-
+TransformRegistry.register(OrgToDomainsScanner)
# Cryptocurrency scanners
TransformRegistry.register(CryptoWalletAddressToTransactions)
TransformRegistry.register(CryptoWalletAddressToNFTs)
@@ -170,9 +173,11 @@ TransformRegistry.register(WebsiteToText)
# Email-related scanners
TransformRegistry.register(EmailToGravatarScanner)
TransformRegistry.register(EmailToBreachesScanner)
+TransformRegistry.register(EmailToDomainsScanner)
# Individual-related scanners
TransformRegistry.register(IndividualToOrgScanner)
+TransformRegistry.register(IndividualToDomainsScanner)
# Integration scanners
TransformRegistry.register(N8nConnector)
diff --git a/flowsint-transforms/pyproject.toml b/flowsint-transforms/pyproject.toml
index 55363b9a..d1679188 100644
--- a/flowsint-transforms/pyproject.toml
+++ b/flowsint-transforms/pyproject.toml
@@ -22,7 +22,6 @@ pydantic = "^2.11.7"
requests-random-user-agent = "^2023.10.25"
spacy = "^3.7"
-docker = "^6.1"
maigret = { git = "https://github.com/soxoj/maigret" }
recontrack = { git = "https://github.com/reconurge/recontrack.git" }
reconcrawl = { git = "https://github.com/reconurge/reconcrawl.git" }
diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py
index 242dbbe9..597a4180 100644
--- a/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py
+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py
@@ -5,6 +5,9 @@ from flowsint_core.core.scanner_base import Scanner
from flowsint_types.wallet import CryptoWallet, CryptoNFT
from flowsint_core.core.logger import Logger
from flowsint_core.core.graph_db import Neo4jConnection
+from dotenv import load_dotenv
+
+load_dotenv()
ETHERSCAN_API_URL = os.getenv("ETHERSCAN_API_URL")
diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py
index 6e2e31d0..59ddb2da 100644
--- a/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py
+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py
@@ -3,11 +3,14 @@ from typing import List, Dict, Any, Optional, Union
import requests
import requests.exceptions
from datetime import datetime
+from dotenv import load_dotenv
from flowsint_core.core.scanner_base import Scanner
from flowsint_types.wallet import CryptoWallet, CryptoWalletTransaction
from flowsint_core.core.graph_db import Neo4jConnection
from flowsint_core.core.logger import Logger
+load_dotenv()
+
ETHERSCAN_API_URL = os.getenv("ETHERSCAN_API_URL")
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py b/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py
index 9010729d..4cb590d5 100644
--- a/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py
@@ -45,7 +45,7 @@ class SubdomainScanner(Scanner):
async def scan(self, data: InputType) -> OutputType:
"""Find subdomains using subfinder (if available) or fallback to crt.sh."""
domains: OutputType = []
- use_subfinder = not self.__is_subfinder_installed()
+ use_subfinder = self.__is_subfinder_installed()
for md in data:
d = Domain(domain=md.domain)
@@ -54,11 +54,16 @@ class SubdomainScanner(Scanner):
if not subdomains:
Logger.warn(
self.sketch_id,
- f"subfinder failed for {d.domain}, falling back to crt.sh",
+ {
+ "message": f"subfinder failed for {d.domain}, falling back to crt.sh"
+ },
)
subdomains = self.__get_subdomains_from_crtsh(d.domain)
else:
- Logger.info(self.sketch_id, "subfinder not found, using crt.sh only")
+ Logger.info(
+ self.sketch_id,
+ {"message": "subfinder not found, using crt.sh only"},
+ )
subdomains = self.__get_subdomains_from_crtsh(d.domain)
domains.append({"domain": d.domain, "subdomains": sorted(subdomains)})
@@ -90,7 +95,9 @@ class SubdomainScanner(Scanner):
elif "*" in sub:
continue
except Exception as e:
- Logger.error(self.sketch_id, f"crt.sh failed for {domain}: {e}")
+ Logger.error(
+ self.sketch_id, {"message": f"crt.sh failed for {domain}: {e}"}
+ )
return subdomains
def __get_subdomains_from_subfinder(self, domain: str) -> set[str]:
@@ -99,7 +106,9 @@ class SubdomainScanner(Scanner):
try:
subdomains = subfinder.launch(domain)
except Exception as e:
- Logger.error(self.sketch_id, f"subfinder exception for {domain}: {e}")
+ Logger.error(
+ self.sketch_id, {"message": f"subfinder exception for {domain}: {e}"}
+ )
return subdomains
def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
diff --git a/flowsint-transforms/src/flowsint_transforms/email/to_domains.py b/flowsint-transforms/src/flowsint_transforms/email/to_domains.py
new file mode 100644
index 00000000..46dde35c
--- /dev/null
+++ b/flowsint-transforms/src/flowsint_transforms/email/to_domains.py
@@ -0,0 +1,409 @@
+import os
+import re
+from typing import Any, List, Union, Dict, Set
+from flowsint_core.core.scanner_base import Scanner
+from flowsint_types.domain import Domain
+from flowsint_types.individual import Individual
+from flowsint_types.email import Email
+from flowsint_types.address import PhysicalAddress
+from flowsint_core.core.logger import Logger
+from tools.network.whoxy import WhoxyTool
+from dotenv import load_dotenv
+
+load_dotenv()
+
+WHOXY_API_KEY = os.getenv("WHOXY_API_KEY")
+
+
+class EmailToDomainsScanner(Scanner):
+ """[WHOXY] Takes an email and returns the domains it registered."""
+
+ # Define types as class attributes - base class handles schema generation automatically
+ InputType = List[Email]
+ OutputType = List[Domain]
+
+ @classmethod
+ def name(cls) -> str:
+ return "email_to_domains_scanner"
+
+ @classmethod
+ def category(cls) -> str:
+ return "Email"
+
+ @classmethod
+ def key(cls) -> str:
+ return "email"
+
+ def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:
+ cleaned: InputType = []
+ for item in data:
+ email_obj = None
+ if isinstance(item, str):
+ email_obj = Email(email=item)
+ elif isinstance(item, dict) and "email" in item:
+ email_obj = Email(email=item["email"])
+ elif isinstance(item, Email):
+ email_obj = item
+ if email_obj:
+ cleaned.append(email_obj)
+ return cleaned
+
+ async def scan(self, data: InputType) -> OutputType:
+ """Find domains related to emails using whoxy api."""
+ domains: OutputType = []
+ self._extracted_data = [] # Store all extracted data for postprocess
+
+ for email in data:
+ infos_data = self.__get_infos_from_whoxy(email.email)
+ if infos_data and "search_result" in infos_data:
+ # Process each domain result
+ for result in infos_data["search_result"]:
+ if self.__is_valid_domain_result(result):
+ domain_name = result.get("domain_name")
+ if domain_name:
+ domain = Domain(domain=domain_name, root=True)
+ domains.append(domain)
+
+ # Store extracted data for postprocess
+ extracted_info = {
+ "email": email,
+ "domain": domain,
+ "domain_data": result,
+ "contacts": {
+ "registrant": result.get("registrant_contact", {}),
+ "administrative": result.get(
+ "administrative_contact", {}
+ ),
+ "technical": result.get("technical_contact", {}),
+ "billing": result.get("billing_contact", {}),
+ },
+ }
+ self._extracted_data.append(extracted_info)
+ else:
+ Logger.info(
+ self.sketch_id,
+ {"message": f"[WHOXY] No domain found for email {email.email}."},
+ )
+ return domains
+
+ def __get_infos_from_whoxy(self, email: str) -> Dict[str, Any]:
+ infos: Dict[str, Any] = {}
+ whoxy = WhoxyTool()
+ try:
+ params = {
+ "key": WHOXY_API_KEY,
+ "reverse": "whois",
+ "email": email,
+ }
+ infos = whoxy.launch(params=params)
+ except Exception as e:
+ Logger.error(
+ self.sketch_id, {"message": f"[WHOXY] Whoxy exception for {email}: {e}"}
+ )
+ return infos
+
+ def __is_valid_domain_result(self, result: Dict[str, Any]) -> bool:
+ """Check if a domain result is valid."""
+ domain_name = result.get("domain_name")
+ if not domain_name:
+ return False
+ return True
+
+ def __extract_individual_from_contact(
+ self, contact: Dict[str, Any], contact_type: str
+ ) -> Individual:
+ """Extract individual information from contact data."""
+ full_name = contact.get("full_name", "")
+
+ # Skip if name is redacted
+ if "REDACTED" in full_name or "REDACTED FOR PRIVACY" in full_name:
+ return None
+
+ # Parse full name into first and last name
+ name_parts = full_name.strip().split()
+ first_name = name_parts[0] if name_parts else ""
+ last_name = " ".join(name_parts[1:]) if len(name_parts) > 1 else ""
+
+ # Extract email and phone
+ email = contact.get("email_address", "")
+ phone = contact.get("phone_number", "")
+
+ # Skip if email is redacted or invalid
+ if (
+ "REDACTED" in email
+ or "REDACTED FOR PRIVACY" in email
+ or not self.__is_valid_email(email)
+ ):
+ email = ""
+
+ # Skip if phone is redacted
+ if "REDACTED" in phone or "REDACTED FOR PRIVACY" in phone:
+ phone = ""
+
+ # Extract address information
+ address = contact.get("mailing_address", "")
+ city = contact.get("city_name", "")
+ zip_code = contact.get("zip_code", "")
+ country = contact.get("country_name", "")
+
+ # Skip if address is redacted
+ if "REDACTED" in address or "REDACTED FOR PRIVACY" in address:
+ address = ""
+ if "REDACTED" in city or "REDACTED FOR PRIVACY" in city:
+ city = ""
+ if "REDACTED" in zip_code or "REDACTED FOR PRIVACY" in zip_code:
+ zip_code = ""
+ if "REDACTED" in country or "REDACTED FOR PRIVACY" in country:
+ country = ""
+
+ # Create individual object
+ individual = Individual(
+ first_name=first_name,
+ last_name=last_name,
+ full_name=full_name,
+ email_addresses=[email] if email else None,
+ phone_numbers=[phone] if phone else None,
+ )
+
+ return individual
+
+ def __is_valid_email(self, email: str) -> bool:
+ """Check if email is valid."""
+ if not email:
+ return False
+ # Basic email validation
+ pattern = r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$"
+ return bool(re.match(pattern, email))
+
+ def __extract_physical_address(self, contact: Dict[str, Any]) -> PhysicalAddress:
+ """Extract physical address from contact data."""
+ address = contact.get("mailing_address", "")
+ city = contact.get("city_name", "")
+ zip_code = contact.get("zip_code", "")
+ country = contact.get("country_name", "")
+
+ # Skip if any part is redacted
+ if any("REDACTED" in field for field in [address, city, zip_code, country]):
+ return None
+
+ if not all([address, city, zip_code, country]):
+ return None
+
+ return PhysicalAddress(
+ address=address, city=city, zip=zip_code, country=country
+ )
+
+ def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ """Create Neo4j nodes and relationships from extracted data."""
+ if not self.neo4j_conn:
+ return results
+
+ # Track processed entities to avoid duplicates
+ processed_domains: Set[str] = set()
+ processed_individuals: Set[str] = set()
+ processed_emails: Set[str] = set()
+ processed_phones: Set[str] = set()
+ processed_addresses: Set[str] = set()
+
+ for extracted_info in self._extracted_data:
+ email = extracted_info["email"]
+ domain = extracted_info["domain"]
+ domain_data = extracted_info["domain_data"]
+ contacts = extracted_info["contacts"]
+
+ domain_name = domain.domain
+ if domain_name in processed_domains:
+ continue
+ processed_domains.add(domain_name)
+
+ # Create email node
+ self.create_node(
+ "email",
+ "email",
+ email.email,
+ caption=email.email,
+ type="email",
+ )
+
+ # Create domain node
+ self.create_node(
+ "domain",
+ "domain",
+ domain_name,
+ label=domain_name,
+ caption=domain_name,
+ type="domain",
+ )
+
+ # Create relationship between email and domain
+ self.create_relationship(
+ "email",
+ "email",
+ email.email,
+ "domain",
+ "domain",
+ domain_name,
+ "HAS_REGISTERED_DOMAIN",
+ )
+
+ # Process all contact types
+ for contact_type, contact in contacts.items():
+ if contact:
+ self.__process_contact(
+ contact,
+ contact_type.upper(),
+ domain_name,
+ email.email,
+ processed_individuals,
+ processed_emails,
+ processed_phones,
+ processed_addresses,
+ )
+
+ self.log_graph_message(
+ f"Processed domain {domain_name} for email {email.email}"
+ )
+
+ return results
+
+ def __process_contact(
+ self,
+ contact: Dict[str, Any],
+ contact_type: str,
+ domain_name: str,
+ email_address: str,
+ processed_individuals: Set[str],
+ processed_emails: Set[str],
+ processed_phones: Set[str],
+ processed_addresses: Set[str],
+ ):
+ """Process a contact and create all related entities and relationships."""
+
+ # Extract individual
+ individual = self.__extract_individual_from_contact(contact, contact_type)
+ if not individual:
+ return
+
+ individual_id = (
+ f"{individual.first_name}_{individual.last_name}_{individual.full_name}"
+ )
+ if individual_id in processed_individuals:
+ return
+
+ processed_individuals.add(individual_id)
+
+ # Create individual node
+ self.create_node(
+ "individual",
+ "full_name",
+ individual.full_name,
+ caption=individual.full_name,
+ type="individual",
+ )
+
+ # Create relationship between individual and domain
+ self.create_relationship(
+ "individual",
+ "full_name",
+ individual.full_name,
+ "domain",
+ "domain",
+ domain_name,
+ f"IS_{contact_type}_CONTACT",
+ )
+
+ # Create relationship between individual and email
+ self.create_relationship(
+ "individual",
+ "full_name",
+ individual.full_name,
+ "email",
+ "email",
+ email_address,
+ f"WORKS_FOR",
+ )
+
+ # Process email addresses
+ if individual.email_addresses:
+ for email in individual.email_addresses:
+ if email and email not in processed_emails:
+ processed_emails.add(email)
+
+ # Create email node
+ self.create_node(
+ "email",
+ "email",
+ email,
+ caption=email,
+ type="email",
+ )
+
+ # Create relationship between individual and email
+ self.create_relationship(
+ "individual",
+ "full_name",
+ individual.full_name,
+ "email",
+ "email",
+ email,
+ "HAS_EMAIL",
+ )
+
+ # Process phone numbers
+ if individual.phone_numbers:
+ for phone in individual.phone_numbers:
+ if phone and phone not in processed_phones:
+ processed_phones.add(phone)
+
+ # Create phone node
+ self.create_node(
+ "phone",
+ "number",
+ phone,
+ caption=phone,
+ type="phone",
+ )
+
+ # Create relationship between individual and phone
+ self.create_relationship(
+ "individual",
+ "full_name",
+ individual.full_name,
+ "phone",
+ "number",
+ phone,
+ "HAS_PHONE",
+ )
+
+ # Process physical address
+ address = self.__extract_physical_address(contact)
+ if address:
+ address_id = (
+ f"{address.address}_{address.city}_{address.zip}_{address.country}"
+ )
+ if address_id not in processed_addresses:
+ processed_addresses.add(address_id)
+
+ # Create address node
+ self.create_node(
+ "location",
+ "address",
+ address.address,
+ caption=f"{address.address}, {address.city}",
+ type="location",
+ )
+
+ # Create relationship between individual and address
+ self.create_relationship(
+ "individual",
+ "full_name",
+ individual.full_name,
+ "location",
+ "address",
+ address.address,
+ "LIVES_AT",
+ )
+
+
+InputType = EmailToDomainsScanner.InputType
+OutputType = EmailToDomainsScanner.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py b/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py
index c8d418bc..77ff2de3 100644
--- a/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py
+++ b/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py
@@ -15,7 +15,7 @@ class EmailToGravatarScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "to_gravatar"
+ return "email_to_gravatar"
@classmethod
def category(cls) -> str:
diff --git a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py
index cc04f33d..9741805a 100644
--- a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py
+++ b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py
@@ -40,7 +40,7 @@ class EmailToBreachesScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "to_breaches"
+ return "email_to_breaches"
@classmethod
def category(cls) -> str:
diff --git a/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py b/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py
new file mode 100644
index 00000000..27bfb0d0
--- /dev/null
+++ b/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py
@@ -0,0 +1,427 @@
+import os
+import re
+from typing import Any, List, Union, Dict, Set
+from flowsint_core.core.scanner_base import Scanner
+from flowsint_types.domain import Domain
+from flowsint_types.organization import Organization
+from flowsint_types.individual import Individual
+from flowsint_types.address import PhysicalAddress
+from flowsint_core.core.logger import Logger
+from tools.network.whoxy import WhoxyTool
+from dotenv import load_dotenv
+
+load_dotenv()
+
+WHOXY_API_KEY = os.getenv("WHOXY_API_KEY")
+
+
+class IndividualToDomainsScanner(Scanner):
+ """[WHOXY] Takes an individual and returns the domains it registered."""
+
+ # Define types as class attributes - base class handles schema generation automatically
+ InputType = List[Individual]
+ OutputType = List[Domain]
+
+ @classmethod
+ def name(cls) -> str:
+ return "individual_to_domains_scanner"
+
+ @classmethod
+ def category(cls) -> str:
+ return "Individual"
+
+ @classmethod
+ def key(cls) -> str:
+ return "full_name"
+
+ def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:
+ if not isinstance(data, list):
+ raise ValueError(f"Expected list input, got {type(data).__name__}")
+ cleaned: InputType = []
+ for item in data:
+ if isinstance(item, str):
+ parts = item.strip().split()
+ if len(parts) >= 2:
+ firstname = parts[0]
+ lastname = " ".join(parts[1:])
+ cleaned.append(
+ Individual(
+ first_name=firstname, last_name=lastname, full_name=item
+ )
+ )
+ elif (
+ isinstance(item, dict)
+ and item.get("first_name")
+ and item.get("last_name")
+ ):
+ cleaned.append(Individual(**item))
+ elif isinstance(item, Individual):
+ cleaned.append(item)
+ if len(cleaned) == 0:
+ Logger.error(
+ self.sketch_id,
+ {"message": "[INDIVIDUAL_TO_DOMAINS] The input type did not match."},
+ )
+ return cleaned
+
+ async def scan(self, data: InputType) -> OutputType:
+ """Find domains related to individuals using whoxy api."""
+ domains: OutputType = []
+ self._extracted_data = [] # Store all extracted data for postprocess
+
+ for individual in data:
+ infos_data = self.__get_infos_from_whoxy(individual.full_name)
+ if infos_data and "search_result" in infos_data:
+ # Process each domain result
+ for result in infos_data["search_result"]:
+ if self.__is_valid_domain_result(result):
+ domain_name = result.get("domain_name")
+ if domain_name:
+ domain = Domain(domain=domain_name, root=True)
+ domains.append(domain)
+
+ # Store extracted data for postprocess
+ extracted_info = {
+ "individual": individual,
+ "domain": domain,
+ "domain_data": result,
+ "contacts": {
+ "registrant": result.get("registrant_contact", {}),
+ "administrative": result.get(
+ "administrative_contact", {}
+ ),
+ "technical": result.get("technical_contact", {}),
+ "billing": result.get("billing_contact", {}),
+ },
+ }
+ self._extracted_data.append(extracted_info)
+ else:
+ Logger.info(
+ self.sketch_id,
+ {"message": f"[WHOXY] No domain found for individual {individual.full_name}."},
+ )
+ return domains
+
+ def __get_infos_from_whoxy(self, individual_name: str) -> Dict[str, Any]:
+ infos: Dict[str, Any] = {}
+ whoxy = WhoxyTool()
+ try:
+ params = {
+ "key": WHOXY_API_KEY,
+ "reverse": "whois",
+ "name": individual_name,
+ }
+ infos = whoxy.launch(params=params)
+ except Exception as e:
+ Logger.error(
+ self.sketch_id,
+ {"message": f"[WHOXY] Whoxy exception for {individual_name}: {e}"},
+ )
+ return infos
+
+ def __is_valid_domain_result(self, result: Dict[str, Any]) -> bool:
+ """Check if a domain result is valid."""
+ domain_name = result.get("domain_name")
+ if not domain_name:
+ return False
+ return True
+
+ def __extract_individual_from_contact(
+ self, contact: Dict[str, Any], contact_type: str
+ ) -> Individual:
+ """Extract individual information from contact data."""
+ full_name = contact.get("full_name", "")
+
+ # Skip if name is redacted
+ if "REDACTED" in full_name or "REDACTED FOR PRIVACY" in full_name:
+ return None
+
+ # Parse full name into first and last name
+ name_parts = full_name.strip().split()
+ first_name = name_parts[0] if name_parts else ""
+ last_name = " ".join(name_parts[1:]) if len(name_parts) > 1 else ""
+
+ # Extract email and phone
+ email = contact.get("email_address", "")
+ phone = contact.get("phone_number", "")
+
+ # Skip if email is redacted or invalid
+ if (
+ "REDACTED" in email
+ or "REDACTED FOR PRIVACY" in email
+ or not self.__is_valid_email(email)
+ ):
+ email = ""
+
+ # Skip if phone is redacted
+ if "REDACTED" in phone or "REDACTED FOR PRIVACY" in phone:
+ phone = ""
+
+ # Extract address information
+ address = contact.get("mailing_address", "")
+ city = contact.get("city_name", "")
+ zip_code = contact.get("zip_code", "")
+ country = contact.get("country_name", "")
+
+ # Skip if address is redacted
+ if "REDACTED" in address or "REDACTED FOR PRIVACY" in address:
+ address = ""
+ if "REDACTED" in city or "REDACTED FOR PRIVACY" in city:
+ city = ""
+ if "REDACTED" in zip_code or "REDACTED FOR PRIVACY" in zip_code:
+ zip_code = ""
+ if "REDACTED" in country or "REDACTED FOR PRIVACY" in country:
+ country = ""
+
+ # Create individual object
+ individual = Individual(
+ first_name=first_name,
+ last_name=last_name,
+ full_name=full_name,
+ email_addresses=[email] if email else None,
+ phone_numbers=[phone] if phone else None,
+ )
+
+ return individual
+
+ def __is_valid_email(self, email: str) -> bool:
+ """Check if email is valid."""
+ if not email:
+ return False
+ # Basic email validation
+ pattern = r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$"
+ return bool(re.match(pattern, email))
+
+ def __extract_physical_address(self, contact: Dict[str, Any]) -> PhysicalAddress:
+ """Extract physical address from contact data."""
+ address = contact.get("mailing_address", "")
+ city = contact.get("city_name", "")
+ zip_code = contact.get("zip_code", "")
+ country = contact.get("country_name", "")
+
+ # Skip if any part is redacted
+ if any("REDACTED" in field for field in [address, city, zip_code, country]):
+ return None
+
+ if not all([address, city, zip_code, country]):
+ return None
+
+ return PhysicalAddress(
+ address=address, city=city, zip=zip_code, country=country
+ )
+
+ def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ """Create Neo4j nodes and relationships from extracted data."""
+ if not self.neo4j_conn:
+ return results
+
+ # Track processed entities to avoid duplicates
+ processed_domains: Set[str] = set()
+ processed_individuals: Set[str] = set()
+ processed_emails: Set[str] = set()
+ processed_phones: Set[str] = set()
+ processed_addresses: Set[str] = set()
+
+ for extracted_info in self._extracted_data:
+ individual = extracted_info["individual"]
+ domain = extracted_info["domain"]
+ domain_data = extracted_info["domain_data"]
+ contacts = extracted_info["contacts"]
+
+ domain_name = domain.domain
+ if domain_name in processed_domains:
+ continue
+ processed_domains.add(domain_name)
+
+ # Create individual node
+ self.create_node(
+ "individual",
+ "full_name",
+ individual.full_name,
+ caption=individual.full_name,
+ type="individual",
+ )
+
+ # Create domain node
+ self.create_node(
+ "domain",
+ "domain",
+ domain_name,
+ label=domain_name,
+ caption=domain_name,
+ type="domain",
+ )
+
+ # Create relationship between individual and domain
+ self.create_relationship(
+ "individual",
+ "full_name",
+ individual.full_name,
+ "domain",
+ "domain",
+ domain_name,
+ "HAS_REGISTERED_DOMAIN",
+ )
+
+ # Process all contact types
+ for contact_type, contact in contacts.items():
+ if contact:
+ self.__process_contact(
+ contact,
+ contact_type.upper(),
+ domain_name,
+ individual.full_name,
+ processed_individuals,
+ processed_emails,
+ processed_phones,
+ processed_addresses,
+ )
+
+ self.log_graph_message(
+ f"Processed domain {domain_name} for individual {individual.full_name}"
+ )
+
+ return results
+
+ def __process_contact(
+ self,
+ contact: Dict[str, Any],
+ contact_type: str,
+ domain_name: str,
+ individual_name: str,
+ processed_individuals: Set[str],
+ processed_emails: Set[str],
+ processed_phones: Set[str],
+ processed_addresses: Set[str],
+ ):
+ """Process a contact and create all related entities and relationships."""
+
+ # Extract individual
+ contact_individual = self.__extract_individual_from_contact(contact, contact_type)
+ if not contact_individual:
+ return
+
+ individual_id = (
+ f"{contact_individual.first_name}_{contact_individual.last_name}_{contact_individual.full_name}"
+ )
+ if individual_id in processed_individuals:
+ return
+
+ processed_individuals.add(individual_id)
+
+ # Create individual node
+ self.create_node(
+ "individual",
+ "full_name",
+ contact_individual.full_name,
+ caption=contact_individual.full_name,
+ type="individual",
+ )
+
+ # Create relationship between individual and domain
+ self.create_relationship(
+ "individual",
+ "full_name",
+ contact_individual.full_name,
+ "domain",
+ "domain",
+ domain_name,
+ f"IS_{contact_type}_CONTACT",
+ )
+
+ # Create relationship between contact individual and main individual
+ self.create_relationship(
+ "individual",
+ "full_name",
+ contact_individual.full_name,
+ "individual",
+ "full_name",
+ individual_name,
+ f"WORKS_FOR",
+ )
+
+ # Process email addresses
+ if contact_individual.email_addresses:
+ for email in contact_individual.email_addresses:
+ if email and email not in processed_emails:
+ processed_emails.add(email)
+
+ # Create email node
+ self.create_node(
+ "email",
+ "email",
+ email,
+ caption=email,
+ type="email",
+ )
+
+ # Create relationship between individual and email
+ self.create_relationship(
+ "individual",
+ "full_name",
+ contact_individual.full_name,
+ "email",
+ "email",
+ email,
+ "HAS_EMAIL",
+ )
+
+ # Process phone numbers
+ if contact_individual.phone_numbers:
+ for phone in contact_individual.phone_numbers:
+ if phone and phone not in processed_phones:
+ processed_phones.add(phone)
+
+ # Create phone node
+ self.create_node(
+ "phone",
+ "number",
+ phone,
+ caption=phone,
+ type="phone",
+ )
+
+ # Create relationship between individual and phone
+ self.create_relationship(
+ "individual",
+ "full_name",
+ contact_individual.full_name,
+ "phone",
+ "number",
+ phone,
+ "HAS_PHONE",
+ )
+
+ # Process physical address
+ address = self.__extract_physical_address(contact)
+ if address:
+ address_id = (
+ f"{address.address}_{address.city}_{address.zip}_{address.country}"
+ )
+ if address_id not in processed_addresses:
+ processed_addresses.add(address_id)
+
+ # Create address node
+ self.create_node(
+ "location",
+ "address",
+ address.address,
+ caption=f"{address.address}, {address.city}",
+ type="location",
+ )
+
+ # Create relationship between individual and address
+ self.create_relationship(
+ "individual",
+ "full_name",
+ contact_individual.full_name,
+ "location",
+ "address",
+ address.address,
+ "LIVES_AT",
+ )
+
+
+# Make types available at module level for easy access
+InputType = IndividualToDomainsScanner.InputType
+OutputType = IndividualToDomainsScanner.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/individual/to_org.py b/flowsint-transforms/src/flowsint_transforms/individual/to_org.py
index b4f3351c..085a09aa 100644
--- a/flowsint-transforms/src/flowsint_transforms/individual/to_org.py
+++ b/flowsint-transforms/src/flowsint_transforms/individual/to_org.py
@@ -16,7 +16,7 @@ class IndividualToOrgScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "to_org"
+ return "individual_to_org"
@classmethod
def category(cls) -> str:
diff --git a/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py b/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py
new file mode 100644
index 00000000..64c1a60e
--- /dev/null
+++ b/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py
@@ -0,0 +1,410 @@
+import os
+import re
+from typing import Any, List, Union, Dict, Set
+from flowsint_core.core.scanner_base import Scanner
+from flowsint_types.domain import Domain
+from flowsint_types.organization import Organization
+from flowsint_types.individual import Individual
+from flowsint_types.email import Email
+from flowsint_types.phone import Phone
+from flowsint_types.address import PhysicalAddress
+from flowsint_core.core.logger import Logger
+from tools.network.whoxy import WhoxyTool
+from dotenv import load_dotenv
+
+load_dotenv()
+
+WHOXY_API_KEY = os.getenv("WHOXY_API_KEY")
+
+
+class OrgToDomainsScanner(Scanner):
+ """[WHOXY] Takes an organization and returns the domains it registered."""
+
+ # Define types as class attributes - base class handles schema generation automatically
+ InputType = List[Organization]
+ OutputType = List[Domain]
+
+ @classmethod
+ def name(cls) -> str:
+ return "org_to_domains_scanner"
+
+ @classmethod
+ def category(cls) -> str:
+ return "Organization"
+
+ @classmethod
+ def key(cls) -> str:
+ return "name"
+
+ def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:
+ cleaned: InputType = []
+ for item in data:
+ org_obj = None
+ if isinstance(item, str):
+ org_obj = Organization(name=item)
+ elif isinstance(item, dict) and "name" in item:
+ org_obj = Organization(name=item["name"])
+ elif isinstance(item, Organization):
+ org_obj = item
+ if org_obj:
+ cleaned.append(org_obj)
+ return cleaned
+
+ async def scan(self, data: InputType) -> OutputType:
+ """Find domains related to organizations using whoxy api."""
+ domains: OutputType = []
+ self._extracted_data = [] # Store all extracted data for postprocess
+
+ for org in data:
+ infos_data = self.__get_infos_from_whoxy(org.name)
+ if infos_data and "search_result" in infos_data:
+ # Process each domain result
+ for result in infos_data["search_result"]:
+ if self.__is_valid_domain_result(result):
+ domain_name = result.get("domain_name")
+ if domain_name:
+ domain = Domain(domain=domain_name, root=True)
+ domains.append(domain)
+
+ # Store extracted data for postprocess
+ extracted_info = {
+ 'org': org,
+ 'domain': domain,
+ 'domain_data': result,
+ 'contacts': {
+ 'registrant': result.get("registrant_contact", {}),
+ 'administrative': result.get("administrative_contact", {}),
+ 'technical': result.get("technical_contact", {}),
+ 'billing': result.get("billing_contact", {})
+ }
+ }
+ self._extracted_data.append(extracted_info)
+ else:
+ Logger.info(
+ self.sketch_id,
+ {"message": f"[WHOXY] No domain found for org {org.name}."},
+ )
+ return domains
+
+ def __get_infos_from_whoxy(self, org_name: str) -> Dict[str, Any]:
+ infos: Dict[str, Any] = {}
+ whoxy = WhoxyTool()
+ try:
+ params = {
+ "key": WHOXY_API_KEY,
+ "reverse": "whois",
+ "company": org_name,
+ }
+ infos = whoxy.launch(params=params)
+ except Exception as e:
+ Logger.error(
+ self.sketch_id,
+ {"message": f"[WHOXY] Whoxy exception for {org_name}: {e}"},
+ )
+ return infos
+
+ def __is_valid_domain_result(self, result: Dict[str, Any]) -> bool:
+ """Check if a domain result is valid."""
+ domain_name = result.get("domain_name")
+ if not domain_name:
+ return False
+ return True
+
+ def __extract_individual_from_contact(
+ self, contact: Dict[str, Any], contact_type: str
+ ) -> Individual:
+ """Extract individual information from contact data."""
+ full_name = contact.get("full_name", "")
+
+ # Skip if name is redacted
+ if "REDACTED" in full_name or "REDACTED FOR PRIVACY" in full_name:
+ return None
+
+ # Parse full name into first and last name
+ name_parts = full_name.strip().split()
+ first_name = name_parts[0] if name_parts else ""
+ last_name = " ".join(name_parts[1:]) if len(name_parts) > 1 else ""
+
+ # Extract email and phone
+ email = contact.get("email_address", "")
+ phone = contact.get("phone_number", "")
+
+ # Skip if email is redacted or invalid
+ if (
+ "REDACTED" in email
+ or "REDACTED FOR PRIVACY" in email
+ or not self.__is_valid_email(email)
+ ):
+ email = ""
+
+ # Skip if phone is redacted
+ if "REDACTED" in phone or "REDACTED FOR PRIVACY" in phone:
+ phone = ""
+
+ # Extract address information
+ address = contact.get("mailing_address", "")
+ city = contact.get("city_name", "")
+ zip_code = contact.get("zip_code", "")
+ country = contact.get("country_name", "")
+
+ # Skip if address is redacted
+ if "REDACTED" in address or "REDACTED FOR PRIVACY" in address:
+ address = ""
+ if "REDACTED" in city or "REDACTED FOR PRIVACY" in city:
+ city = ""
+ if "REDACTED" in zip_code or "REDACTED FOR PRIVACY" in zip_code:
+ zip_code = ""
+ if "REDACTED" in country or "REDACTED FOR PRIVACY" in country:
+ country = ""
+
+ # Create individual object
+ individual = Individual(
+ first_name=first_name,
+ last_name=last_name,
+ full_name=full_name,
+ email_addresses=[email] if email else None,
+ phone_numbers=[phone] if phone else None,
+ )
+
+ return individual
+
+ def __is_valid_email(self, email: str) -> bool:
+ """Check if email is valid."""
+ if not email:
+ return False
+ # Basic email validation
+ pattern = r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$"
+ return bool(re.match(pattern, email))
+
+ def __extract_physical_address(self, contact: Dict[str, Any]) -> PhysicalAddress:
+ """Extract physical address from contact data."""
+ address = contact.get("mailing_address", "")
+ city = contact.get("city_name", "")
+ zip_code = contact.get("zip_code", "")
+ country = contact.get("country_name", "")
+
+ # Skip if any part is redacted
+ if any("REDACTED" in field for field in [address, city, zip_code, country]):
+ return None
+
+ if not all([address, city, zip_code, country]):
+ return None
+
+ return PhysicalAddress(
+ address=address, city=city, zip=zip_code, country=country
+ )
+
+ def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ """Create Neo4j nodes and relationships from extracted data."""
+ if not self.neo4j_conn:
+ return results
+
+ # Track processed entities to avoid duplicates
+ processed_domains: Set[str] = set()
+ processed_individuals: Set[str] = set()
+ processed_emails: Set[str] = set()
+ processed_phones: Set[str] = set()
+ processed_addresses: Set[str] = set()
+
+ for extracted_info in self._extracted_data:
+ org = extracted_info['org']
+ domain = extracted_info['domain']
+ domain_data = extracted_info['domain_data']
+ contacts = extracted_info['contacts']
+
+ domain_name = domain.domain
+ if domain_name in processed_domains:
+ continue
+ processed_domains.add(domain_name)
+
+ # Create organization node
+ self.create_node(
+ "organization",
+ "name",
+ org.name,
+ caption=org.name,
+ type="organization",
+ )
+
+ # Create domain node
+ self.create_node(
+ "domain",
+ "domain",
+ domain_name,
+ label=domain_name,
+ caption=domain_name,
+ type="domain",
+ )
+
+ # Create relationship between organization and domain
+ self.create_relationship(
+ "organization",
+ "name",
+ org.name,
+ "domain",
+ "domain",
+ domain_name,
+ "HAS_REGISTERED_DOMAIN",
+ )
+
+ # Process all contact types
+ for contact_type, contact in contacts.items():
+ if contact:
+ self.__process_contact(
+ contact,
+ contact_type.upper(),
+ domain_name,
+ org.name,
+ processed_individuals,
+ processed_emails,
+ processed_phones,
+ processed_addresses,
+ )
+
+ self.log_graph_message(
+ f"Processed domain {domain_name} for organization {org.name}"
+ )
+
+ return results
+
+ def __process_contact(
+ self,
+ contact: Dict[str, Any],
+ contact_type: str,
+ domain_name: str,
+ org_name: str,
+ processed_individuals: Set[str],
+ processed_emails: Set[str],
+ processed_phones: Set[str],
+ processed_addresses: Set[str],
+ ):
+ """Process a contact and create all related entities and relationships."""
+
+ # Extract individual
+ individual = self.__extract_individual_from_contact(contact, contact_type)
+ if not individual:
+ return
+
+ individual_id = (
+ f"{individual.first_name}_{individual.last_name}_{individual.full_name}"
+ )
+ if individual_id in processed_individuals:
+ return
+
+ processed_individuals.add(individual_id)
+
+ # Create individual node
+ self.create_node(
+ "individual",
+ "full_name",
+ individual.full_name,
+ caption=individual.full_name,
+ type="individual",
+ )
+
+ # Create relationship between individual and domain
+ self.create_relationship(
+ "individual",
+ "full_name",
+ individual.full_name,
+ "domain",
+ "domain",
+ domain_name,
+ f"IS_{contact_type}_CONTACT",
+ )
+
+ # Create relationship between individual and organization
+ self.create_relationship(
+ "individual",
+ "full_name",
+ individual.full_name,
+ "organization",
+ "name",
+ org_name,
+ f"WORKS_FOR",
+ )
+
+ # Process email addresses
+ if individual.email_addresses:
+ for email in individual.email_addresses:
+ if email and email not in processed_emails:
+ processed_emails.add(email)
+
+ # Create email node
+ self.create_node(
+ "email",
+ "email",
+ email,
+ caption=email,
+ type="email",
+ )
+
+ # Create relationship between individual and email
+ self.create_relationship(
+ "individual",
+ "full_name",
+ individual.full_name,
+ "email",
+ "email",
+ email,
+ "HAS_EMAIL",
+ )
+
+ # Process phone numbers
+ if individual.phone_numbers:
+ for phone in individual.phone_numbers:
+ if phone and phone not in processed_phones:
+ processed_phones.add(phone)
+
+ # Create phone node
+ self.create_node(
+ "phone",
+ "number",
+ phone,
+ caption=phone,
+ type="phone",
+ )
+
+ # Create relationship between individual and phone
+ self.create_relationship(
+ "individual",
+ "full_name",
+ individual.full_name,
+ "phone",
+ "number",
+ phone,
+ "HAS_PHONE",
+ )
+
+ # Process physical address
+ address = self.__extract_physical_address(contact)
+ if address:
+ address_id = (
+ f"{address.address}_{address.city}_{address.zip}_{address.country}"
+ )
+ if address_id not in processed_addresses:
+ processed_addresses.add(address_id)
+
+ # Create address node
+ self.create_node(
+ "location",
+ "address",
+ address.address,
+ caption=f"{address.address}, {address.city}",
+ type="location",
+ )
+
+ # Create relationship between individual and address
+ self.create_relationship(
+ "individual",
+ "full_name",
+ individual.full_name,
+ "location",
+ "address",
+ address.address,
+ "LIVES_AT",
+ )
+
+
+InputType = OrgToDomainsScanner.InputType
+OutputType = OrgToDomainsScanner.OutputType
diff --git a/flowsint-transforms/src/tools/network/whoxy.py b/flowsint-transforms/src/tools/network/whoxy.py
new file mode 100644
index 00000000..3faabe0b
--- /dev/null
+++ b/flowsint-transforms/src/tools/network/whoxy.py
@@ -0,0 +1,42 @@
+from typing import Dict
+
+import requests
+from ..base import Tool
+
+
+class WhoxyTool(Tool):
+
+ @classmethod
+ def name(cls) -> str:
+ return "whoxy"
+
+ @classmethod
+ def version(cls) -> str:
+ return "1.0.0"
+
+ @classmethod
+ def description(cls) -> str:
+ return "The WHOIS API returns consistent and well-structured WHOIS data in XML & JSON format. Returned data contain parsed WHOIS fields that can be easily understood. Along with WHOIS API, Whoxy also offer WHOIS History API and Reverse WHOIS API."
+
+ @classmethod
+ def category(cls) -> str:
+ return "Network intelligence"
+
+ def launch(self, params: Dict[str, str] = {}) -> list[Dict]:
+ try:
+ resp = requests.get(
+ "https://api.whoxy.com/",
+ params=params,
+ timeout=10,
+ )
+ resp.raise_for_status()
+ data = resp.json()
+ if data.get("status") != 1:
+ raise RuntimeError(
+ f"Error querying Whoxy API: {str(data.get("status_reason") )}"
+ )
+ if data.get("total_results") == 0:
+ raise ValueError(f"No match found for Whoxy search.")
+ return data
+ except Exception as e:
+ raise RuntimeError(f"{str(e)}")
diff --git a/flowsint-transforms/tests/tools/network/whoxy.py b/flowsint-transforms/tests/tools/network/whoxy.py
new file mode 100644
index 00000000..f3545b24
--- /dev/null
+++ b/flowsint-transforms/tests/tools/network/whoxy.py
@@ -0,0 +1,18 @@
+from tools.network.whoxy import WhoxyTool
+
+tool = WhoxyTool()
+
+
+def test_name():
+ assert tool.name() == "whoxy"
+
+
+def test_description():
+ assert (
+ tool.description()
+ == "The WHOIS API returns consistent and well-structured WHOIS data in XML & JSON format. Returned data contain parsed WHOIS fields that can be easily understood. Along with WHOIS API, Whoxy also offer WHOIS History API and Reverse WHOIS API."
+ )
+
+
+def test_category():
+ assert tool.category() == "Network intelligence"
diff --git a/flowsint-types/src/flowsint_types/document.py b/flowsint-types/src/flowsint_types/document.py
index f8e5f0d6..7be38dee 100644
--- a/flowsint-types/src/flowsint_types/document.py
+++ b/flowsint-types/src/flowsint_types/document.py
@@ -6,7 +6,7 @@ class Document(BaseModel):
"""Represents a document with metadata, security, and content information."""
title: str = Field(..., description="Document title", title="Title")
- type: Optional[str] = Field(
+ doc_type: Optional[str] = Field(
None, description="Type of document (PDF, DOC, etc.)", title="Document Type"
)
file_size: Optional[int] = Field(
diff --git a/poetry.lock b/poetry.lock
index 49ff50b1..62e72597 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1152,25 +1152,26 @@ wmi = ["wmi (>=1.5.1)"]
[[package]]
name = "docker"
-version = "6.1.3"
+version = "7.1.0"
description = "A Python library for the Docker Engine API."
optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
groups = ["main"]
files = [
- {file = "docker-6.1.3-py3-none-any.whl", hash = "sha256:aecd2277b8bf8e506e484f6ab7aec39abe0038e29fa4a6d3ba86c3fe01844ed9"},
- {file = "docker-6.1.3.tar.gz", hash = "sha256:aa6d17830045ba5ef0168d5eaa34d37beeb113948c413affe1d5991fc11f9a20"},
+ {file = "docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0"},
+ {file = "docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c"},
]
[package.dependencies]
-packaging = ">=14.0"
pywin32 = {version = ">=304", markers = "sys_platform == \"win32\""}
requests = ">=2.26.0"
urllib3 = ">=1.26.0"
-websocket-client = ">=0.32.0"
[package.extras]
+dev = ["coverage (==7.2.7)", "pytest (==7.4.2)", "pytest-cov (==4.1.0)", "pytest-timeout (==2.1.0)", "ruff (==0.1.8)"]
+docs = ["myst-parser (==0.18.0)", "sphinx (==5.1.1)"]
ssh = ["paramiko (>=2.4.3)"]
+websockets = ["websocket-client (>=1.3.0)"]
[[package]]
name = "ecdsa"
@@ -1331,7 +1332,6 @@ develop = true
alembic = "1.13.0"
asyncpg = "^0.29"
celery = "^5.3"
-docker = "^6.1"
flowsint-transforms = {path = "../flowsint-transforms", develop = true}
neo4j = "^5.0"
networkx = "^2.6.3"
@@ -1363,7 +1363,6 @@ develop = true
[package.dependencies]
dnspython = "^2.4"
-docker = "^6.1"
flowsint-core = {path = "../flowsint-core", develop = true}
flowsint-types = {path = "../flowsint-types", develop = true}
hibpwned = "^1.3.9"
@@ -5290,23 +5289,6 @@ files = [
{file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"},
]
-[[package]]
-name = "websocket-client"
-version = "1.8.0"
-description = "WebSocket client for Python with low level API options"
-optional = false
-python-versions = ">=3.8"
-groups = ["main"]
-files = [
- {file = "websocket_client-1.8.0-py3-none-any.whl", hash = "sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526"},
- {file = "websocket_client-1.8.0.tar.gz", hash = "sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da"},
-]
-
-[package.extras]
-docs = ["Sphinx (>=6.0)", "myst-parser (>=2.0.0)", "sphinx-rtd-theme (>=1.1.0)"]
-optional = ["python-socks", "wsaccel"]
-test = ["websockets"]
-
[[package]]
name = "werkzeug"
version = "3.1.3"
@@ -5580,4 +5562,4 @@ propcache = ">=0.2.1"
[metadata]
lock-version = "2.1"
python-versions = ">=3.11,<3.13"
-content-hash = "17855cf5181cb53298fb8d9ae7d27342d9ffda473d2205f3493f22f943e6ad67"
+content-hash = "9a8a3f39280330fdf9eff55a050f6809081947173c8eac811da2ef13373c59f7"
diff --git a/pyproject.toml b/pyproject.toml
index ef42a57e..ff8fbf67 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -13,6 +13,7 @@ flowsint-transforms = { path = "./flowsint-transforms", develop = true }
flowsint-api = { path = "./flowsint-api", develop = true }
pydantic = {extras = ["email"], version = "^2.11.7"}
python-multipart = "^0.0.20"
+docker = "^7.1.0"
[build-system]
requires = ["poetry-core"]