A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat: whoxy scanners
dextmorgn59ecd842071d
diff --git a/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx b/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsxindex 109bba80..65dae61a 100644--- a/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx+++ b/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx@@ -17,7 +17,7 @@ export default function DetailsPanel({ node }: { node: GraphNode | null }) {const { id: sketchId } = useParams({ strict: false })if (!node) {return (- <div className="flex flex-col items-center justify-center h-full text-center p-8">+ <div className="flex p-12 rounded-xl border flex-col items-center justify-center h-full text-center p-8"><div className="w-16 h-16 bg-muted rounded-full flex items-center justify-center mb-4"><MousePointer className="h-8 w-8 text-muted-foreground" /></div>diff --git a/flowsint-app/src/renderer/src/components/graphs/details-panel/neighbors.tsx b/flowsint-app/src/renderer/src/components/graphs/details-panel/neighbors.tsxindex 5c047cd7..5d2a6f40 100644--- a/flowsint-app/src/renderer/src/components/graphs/details-panel/neighbors.tsx+++ b/flowsint-app/src/renderer/src/components/graphs/details-panel/neighbors.tsx@@ -27,6 +27,7 @@ const NeighborsGraph = ({ sketchId, nodeId }: { sketchId: string, nodeId: stringshowIcons={true}backgroundColor="transparent"onGraphRef={() => { }}+ instanceId="neighbors"/></>}</div>diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-main.tsxindex dab6878a..e6c0f039 100644--- a/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx+++ b/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx@@ -1,5 +1,5 @@import { useGraphStore } from '@/stores/graph-store'-import React, { useEffect, useRef, useCallback } from 'react'+import React, { useRef, useCallback } from 'react'import GraphViewer from './graph-viewer'import ContextMenu from './context-menu'@@ -15,13 +15,6 @@ const GraphMain = () => {const [menu, setMenu] = React.useState<any>(null)- // // Handle current node centering- // useEffect(() => {- // if (!currentNode || !graphRef.current) return- // graphRef.current.centerAt(currentNode.position.x, currentNode.position.x, 500)- // graphRef.current.zoom(5, 500)- // }, [currentNode])-const handleNodeClick = useCallback((node: any) => {toggleNodeSelection(node, false)}, [toggleNodeSelection])diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsxindex 945be343..0485090d 100644--- a/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx+++ b/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx@@ -1,5 +1,5 @@import { useLoaderData } from '@tanstack/react-router'-import { useEffect, useRef, memo, useState, lazy, Suspense, useMemo } from 'react'+import { useEffect, useRef, memo, useState, lazy, Suspense } from 'react'import { useGraphStore, type GraphNode, type GraphEdge } from '@/stores/graph-store'import { Toolbar } from './toolbar'import { cn } from '@/lib/utils'@@ -56,48 +56,6 @@ const GraphPanel = ({ graphData, isLoading }: GraphPanelProps) => {})const [isDraggingOver, setIsDraggingOver] = useState(false)- // Determine which graph component to use based on node count with debouncing- const [stableGraphType, setStableGraphType] = useState<'cosmograph' | 'force-graph' | null>(null)-- const shouldUseCosmograph = useMemo(() => {- if (!nodes || nodes.length === 0) return false- return nodes.length > NODE_COUNT_THRESHOLD- }, [nodes?.length])-- // Debounce graph type changes to prevent rapid switching- useEffect(() => {- if (!nodes || nodes.length === 0) {- setStableGraphType(null)- return- }-- const newType = shouldUseCosmograph ? 'cosmograph' : 'force-graph'-- // Only update if the type actually changed- if (stableGraphType !== newType) {- const timer = setTimeout(() => {- setStableGraphType(newType)- }, 500) // 500ms delay to prevent rapid switching-- return () => clearTimeout(timer)- }-- // Return empty cleanup function for the case when type hasn't changed- return () => { }- }, [shouldUseCosmograph, stableGraphType, nodes?.length])-- const graphComponentKey = stableGraphType || 'loading'-- // Memoize the graph component to prevent unnecessary re-renders- const graphComponent = useMemo(() => {- if (stableGraphType === 'cosmograph') {- return <Graph />- } else if (stableGraphType === 'force-graph') {- return <GraphMain />- }- return null- }, [stableGraphType])-useEffect(() => {if (graphData?.nds && graphData?.rls) {updateGraphData(graphData.nds, graphData.rls)@@ -170,28 +128,21 @@ const GraphPanel = ({ graphData, isLoading }: GraphPanelProps) => {</div></div>}>- {/* Only render graph components when we have actual data */}- {nodes && nodes.length > 0 && stableGraphType ? (- <div key={graphComponentKey} className="h-full w-full flex relative outline-2 outline-transparent bg-background"- >- {view === "table" && <NodesTable />}+ {nodes?.length > NODE_COUNT_THRESHOLD ? (+ <>{view === "table" && <NodesTable />}+ {["force", "hierarchy"].includes(view) && <Graph />}{view === "map" && <MapPanel />}{view === "relationships" && <RelationshipsTable />}- {["force", "hierarchy"].includes(view) && graphComponent}- </div>+ </>) : (- // Show loading state while waiting for data or graph type to stabilize- <div className="h-full w-full flex items-center justify-center">- <div className="text-center flex flex-col items-center gap-2">- <Loader />- <span className="text-muted-foreground">- {!nodes || nodes.length === 0- ? 'Loading data...'- : 'Preparing data...'- }- </span>- </div>- </div>+ <>+ {/* {view === "hierarchy" && <Wall isLoading={false} isRefetching={false} />} */}+ {/* {view === "force" && <GraphMain />} */}+ {["force", "hierarchy"].includes(view) && <GraphMain />}+ {view === "table" && <NodesTable />}+ {view === "map" && <MapPanel />}+ {view === "relationships" && <RelationshipsTable />}+ </>)}</Suspense><DragOverlay isDragging={isDraggingOver} />diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsxindex 7714afca..710fc746 100644--- a/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx+++ b/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx@@ -8,6 +8,7 @@ import ForceGraph2D from 'react-force-graph-2d';import { Button } from '../ui/button';import { useTheme } from '@/components/theme-provider'import { GRAPH_COLORS } from '../flows/scanner-data';+import { Share2 } from 'lucide-react';function truncateText(text: string, limit: number = 16) {if (text.length <= limit)@@ -31,6 +32,7 @@ interface GraphViewerProps {className?: string;style?: React.CSSProperties;onGraphRef?: (ref: any) => void;+ instanceId?: string; // Add instanceId prop for instance-specific actions}const CONSTANTS = {@@ -47,7 +49,15 @@ const CONSTANTS = {MIN_FONT_SIZE: 0.5,LINK_WIDTH: 1,ARROW_SIZE: 8,- ARROW_ANGLE: Math.PI / 6+ ARROW_ANGLE: Math.PI / 6,+ // New constants for layered label display+ LABEL_LAYERS: [+ { minZoom: 1.5, minWeight: 15, maxNodes: 20 }, // High-weight nodes first+ { minZoom: 2.0, minWeight: 10, maxNodes: 50 }, // Medium-weight nodes+ { minZoom: 2.5, minWeight: 5, maxNodes: 100 }, // Lower-weight nodes+ { minZoom: 3.0, minWeight: 1, maxNodes: 200 }, // All remaining nodes+ { minZoom: 4.0, minWeight: 0, maxNodes: Infinity } // All nodes at high zoom+ ]};// Pre-computed constants@@ -107,7 +117,8 @@ const GraphViewer: React.FC<GraphViewerProps> = ({backgroundColor = 'transparent',className = '',style,- onGraphRef+ onGraphRef,+ instanceId}) => {const [currentZoom, setCurrentZoom] = useState(1);const [dimensions, setDimensions] = useState({ width: 0, height: 0 });@@ -147,35 +158,50 @@ const GraphViewer: React.FC<GraphViewerProps> = ({const initializeGraph = useCallback((graphInstance: any) => {if (!graphInstance || isGraphReadyRef.current) return;isGraphReadyRef.current = true;- // Set up graph actions- setActions({- zoomIn: () => {- const zoom = graphInstance.zoom();- graphInstance.zoom(zoom * 1.5);- },- zoomOut: () => {- const zoom = graphInstance.zoom();- graphInstance.zoom(zoom * 0.75);- },- zoomToFit: () => {- graphInstance.zoomToFit(400);- }- });++ // Only set global actions if no instanceId is provided (for main graph)+ if (!instanceId) {+ setActions({+ zoomIn: () => {+ const zoom = graphInstance.zoom();+ graphInstance.zoom(zoom * 1.5);+ },+ zoomOut: () => {+ const zoom = graphInstance.zoom();+ graphInstance.zoom(zoom * 0.75);+ },+ zoomToFit: () => {+ graphInstance.zoomToFit(400);+ }+ });+ }// Call external ref callbackonGraphRef?.(graphInstance);- }, [setActions, onGraphRef]);+ }, [setActions, onGraphRef, instanceId]);// Handle graph ref changesuseEffect(() => {if (graphRef.current) {initializeGraph(graphRef.current);}++ // Cleanup: reset actions when component unmounts (only for main graph)+ return () => {+ if (!instanceId && isGraphReadyRef.current) {+ setActions({+ zoomIn: () => { },+ zoomOut: () => { },+ zoomToFit: () => { },+ });+ isGraphReadyRef.current = false;+ }+ };}, [initializeGraph]);// Memoized rendering checkconst shouldUseSimpleRendering = useMemo(() =>- nodes.length > CONSTANTS.NODE_COUNT_THRESHOLD || currentZoom < 2.5+ nodes.length > CONSTANTS.NODE_COUNT_THRESHOLD || currentZoom < 1.5, [nodes.length, currentZoom]);// Optimized graph data transformation with proper memoization dependencies@@ -189,7 +215,7 @@ const GraphViewer: React.FC<GraphViewerProps> = ({nodeColor: nodeColors[type] || GRAPH_COLORS.NODE_DEFAULT,nodeSize: CONSTANTS.NODE_DEFAULT_SIZE,nodeType: type,- val: getSize(type),+ val: CONSTANTS.NODE_DEFAULT_SIZE,neighbors: [] as any[],links: [] as any[]};@@ -248,17 +274,72 @@ const GraphViewer: React.FC<GraphViewerProps> = ({if (view === "hierarchy") {const { nodes: nds, edges: eds } = getDagreLayoutedElements(transformedNodes, transformedEdges);return {- nodes: nds.map((nd) => ({ ...nd, x: nd.position.x, y: nd.position.y })),+ nodes: nds.map((nd) => ({+ ...nd,+ x: nd.position.x,+ y: nd.position.y,+ // Preserve the neighbors and links from the original transformed node+ neighbors: nodeMap.get(nd.id)?.neighbors || [],+ links: nodeMap.get(nd.id)?.links || []+ })),links: eds,};}-return {nodes: transformedNodes,links: transformedEdges,};}, [nodes, edges, nodeColors, getSize, view]);+ // New function to determine which labels should be visible based on zoom and weight+ const getVisibleLabels = useMemo(() => {+ if (!showLabels) return new Set<string>();++ // Find the appropriate layer for current zoom+ const currentLayer = CONSTANTS.LABEL_LAYERS.find(layer => currentZoom >= layer.minZoom);+ if (!currentLayer) return new Set<string>();++ // Sort nodes by weight (number of connections) in descending order+ const sortedNodes = [...graphData.nodes].sort((a, b) => {+ const aWeight = a.neighbors?.length || 0;+ const bWeight = b.neighbors?.length || 0;+ return bWeight - aWeight; // Higher weight first+ });++ // Take only the nodes that meet the criteria for this layer+ const visibleNodes = sortedNodes+ .filter(node => {+ const weight = node.neighbors?.length || 0;+ return weight >= currentLayer.minWeight;+ })+ .slice(0, currentLayer.maxNodes);++ return new Set(visibleNodes.map(node => node.id));+ }, [graphData.nodes, currentZoom, showLabels]);++ // Get current label layer info for display+ const currentLabelLayer = useMemo(() => {+ if (!showLabels) return null;+ return CONSTANTS.LABEL_LAYERS.find(layer => currentZoom >= layer.minZoom);+ }, [currentZoom, showLabels]);++ // Get label count info for current layer+ const labelInfo = useMemo(() => {+ if (!currentLabelLayer || !showLabels) return null;++ const totalNodes = graphData.nodes.length;+ const visibleCount = getVisibleLabels.size;+ const weightThreshold = currentLabelLayer.minWeight;++ return {+ visibleCount,+ totalNodes,+ weightThreshold,+ zoomLevel: currentZoom,+ layer: currentLabelLayer+ };+ }, [currentLabelLayer, showLabels, getVisibleLabels.size, graphData.nodes.length, currentZoom]);+// Event handlers with proper memoizationconst handleNodeClick = useCallback((node: any) => {onNodeClick?.(node);@@ -320,6 +401,80 @@ const GraphViewer: React.FC<GraphViewerProps> = ({setHighlightLinks(newHighlightLinks);}, []);+ // Add state for tooltip+ const [tooltip, setTooltip] = useState<{+ x: number;+ y: number;+ data: {+ label: string,+ connections: string+ } | null;+ visible: boolean;+ }>({+ x: 0,+ y: 0,+ data: null,+ visible: false+ });++ // Enhanced node hover with tooltip+ const handleNodeHoverWithTooltip = useCallback((node: any) => {+ if (node) {+ const weight = node.neighbors?.length || 0;+ const label = node.nodeLabel || node.label || node.id;++ // Position tooltip above the node using the graph's coordinate conversion+ if (graphRef.current) {+ try {+ // Use the graph's built-in method to convert graph coordinates to screen coordinates+ const screenCoords = graphRef.current.graph2ScreenCoords(node.x, node.y);++ // Ensure tooltip stays within viewport bounds+ const tooltipWidth = 120; // Approximate tooltip width+ const tooltipHeight = 60; // Approximate tooltip height++ let x = screenCoords.x;+ let y = screenCoords.y - 30; // Position above the node++ // Adjust X position if tooltip would go off-screen+ if (x + tooltipWidth > window.innerWidth) {+ x = window.innerWidth - tooltipWidth - 10;+ }+ if (x < 10) {+ x = 10;+ }++ // Adjust Y position if tooltip would go off-screen+ if (y < tooltipHeight + 10) {+ y = screenCoords.y + 30; // Position below the node instead+ }++ setTooltip({+ x,+ y,+ data: {+ label,+ connections: weight+ },+ visible: true+ });+ } catch (error) {+ // Fallback: hide tooltip if coordinate conversion fails+ setTooltip(prev => ({ ...prev, visible: false }));+ }+ }+ } else {+ setTooltip(prev => ({ ...prev, visible: false }));+ }+ handleNodeHover(node);+ }, [handleNodeHover]);++ // Remove mouse tracking since we don't need it anymore+ // const [mousePosition, setMousePosition] = useState({ x: 0, y: 0 });+ // const handleMouseMove = useCallback((event: React.MouseEvent) => {+ // setMousePosition({ x: event.clientX, y: event.clientY });+ // }, []);+const handleLinkHover = useCallback((link: any) => {// Throttle hover updates to max 60fpsconst now = Date.now();@@ -353,7 +508,8 @@ const GraphViewer: React.FC<GraphViewerProps> = ({// Optimized node rendering with proper icon cachingconst renderNode = useCallback((node: any, ctx: CanvasRenderingContext2D, globalScale: number) => {- const size = node.nodeSize * (settings.nodeSize.value / 100 + .4);+ const size = Math.min((node.nodeSize + node.neighbors.length / 5), 20) * (settings.nodeSize.value / 100 + .4);+ node.val = size / 5const isHighlighted = highlightNodes.has(node.id);const hasAnyHighlight = highlightNodes.size > 0 || highlightLinks.size > 0;const isHovered = hoverNode === node.id;@@ -392,13 +548,15 @@ const GraphViewer: React.FC<GraphViewerProps> = ({}}- // Optimized label rendering+ // Optimized label rendering with layered displayif (showLabels && globalScale > 3) {const label = truncateText(node.nodeLabel || node.label || node.id, 58);if (label) {- // Only show labels for highlighted nodes when there's any highlighting- // or show all labels when there's no highlighting- if (hasAnyHighlight && !isHighlighted) {+ // Check if this node's label should be visible in current layer+ const shouldShowLabel = getVisibleLabels.has(node.id);++ // Always show labels for highlighted nodes+ if (!shouldShowLabel && !isHighlighted) {return;}@@ -411,14 +569,22 @@ const GraphViewer: React.FC<GraphViewerProps> = ({ctx.textAlign = 'center';ctx.textBaseline = 'middle';- ctx.fillStyle = isHighlighted ? color : `${color}2D`;++ // Adjust opacity based on whether it's a highlighted node or just visible in current layer+ if (isHighlighted) {+ ctx.fillStyle = color;+ } else {+ // Fade out labels that are just visible due to layer, not highlighting+ ctx.fillStyle = `${color}4D`; // 30% opacity+ }+ctx.fillText(label, node.x, bgY + bgHeight / 2);}}- }, [shouldUseSimpleRendering, showLabels, showIcons, settings.nodeSize.value, theme, highlightNodes, highlightLinks, hoverNode]);+ }, [shouldUseSimpleRendering, showLabels, showIcons, settings.nodeSize.value, theme, highlightNodes, highlightLinks, hoverNode, getVisibleLabels]);// Optimized link rendering with reduced canvas state changes- const linkCanvasObject = useCallback((link: any, ctx: CanvasRenderingContext2D) => {+ const renderLink = useCallback((link: any, ctx: CanvasRenderingContext2D) => {const { source: start, target: end } = link;// Early exit for unbound linksif (typeof start !== 'object' || typeof end !== 'object') return;@@ -505,50 +671,50 @@ const GraphViewer: React.FC<GraphViewerProps> = ({if (shouldUseSimpleRendering || !link.label) return;// Only show labels for highlighted links when there's any highlighting- if (hasAnyHighlight && !isHighlighted) {- return;- }+ if (isHighlighted) {- // Calculate label position and angle- tempPos.x = (start.x + end.x) * 0.5;- tempPos.y = (start.y + end.y) * 0.5;- const dx = end.x - start.x;- const dy = end.y - start.y;- let textAngle = Math.atan2(dy, dx);+ // Calculate label position and angle+ tempPos.x = (start.x + end.x) * 0.5;+ tempPos.y = (start.y + end.y) * 0.5;- // Flip text for readability- if (textAngle > CONSTANTS.HALF_PI || textAngle < -CONSTANTS.HALF_PI) {- textAngle += textAngle > 0 ? -CONSTANTS.PI : CONSTANTS.PI;- }+ const dx = end.x - start.x;+ const dy = end.y - start.y;+ let textAngle = Math.atan2(dy, dx);- // Measure and draw label- ctx.font = LABEL_FONT_STRING;- const textWidth = ctx.measureText(link.label).width;- const padding = CONSTANTS.LABEL_FONT_SIZE * CONSTANTS.PADDING_RATIO;+ // Flip text for readability+ if (textAngle > CONSTANTS.HALF_PI || textAngle < -CONSTANTS.HALF_PI) {+ textAngle += textAngle > 0 ? -CONSTANTS.PI : CONSTANTS.PI;+ }- tempDimensions[0] = textWidth + padding;- tempDimensions[1] = CONSTANTS.LABEL_FONT_SIZE + padding;+ // Measure and draw label+ ctx.font = LABEL_FONT_STRING;+ const textWidth = ctx.measureText(link.label).width;+ const padding = CONSTANTS.LABEL_FONT_SIZE * CONSTANTS.PADDING_RATIO;- const halfWidth = tempDimensions[0] * 0.5;- const halfHeight = tempDimensions[1] * 0.5;+ tempDimensions[0] = textWidth + padding;+ tempDimensions[1] = CONSTANTS.LABEL_FONT_SIZE + padding;- // Batch canvas operations- ctx.save();- ctx.translate(tempPos.x, tempPos.y);- ctx.rotate(textAngle);+ const halfWidth = tempDimensions[0] * 0.5;+ const halfHeight = tempDimensions[1] * 0.5;- // Background- ctx.fillStyle = theme === "light" ? GRAPH_COLORS.BACKGROUND_LIGHT : GRAPH_COLORS.BACKGROUND_DARK;- ctx.fillRect(-halfWidth, -halfHeight, tempDimensions[0], tempDimensions[1]);+ // Batch canvas operations+ ctx.save();+ ctx.translate(tempPos.x, tempPos.y);+ ctx.rotate(textAngle);++ // Background+ ctx.fillStyle = theme === "light" ? GRAPH_COLORS.BACKGROUND_LIGHT : GRAPH_COLORS.BACKGROUND_DARK;+ ctx.fillRect(-halfWidth, -halfHeight, tempDimensions[0], tempDimensions[1]);- // Text - follow same highlighting behavior as links- ctx.fillStyle = isHighlighted ? GRAPH_COLORS.LINK_LABEL_HIGHLIGHTED : GRAPH_COLORS.LINK_LABEL_DEFAULT;- ctx.textAlign = 'center';- ctx.textBaseline = 'middle';- ctx.fillText(link.label, 0, 0);+ // Text - follow same highlighting behavior as links+ ctx.fillStyle = isHighlighted ? GRAPH_COLORS.LINK_LABEL_HIGHLIGHTED : GRAPH_COLORS.LINK_LABEL_DEFAULT;+ ctx.textAlign = 'center';+ ctx.textBaseline = 'middle';+ ctx.fillText(link.label, 0, 0);- ctx.restore();+ ctx.restore();+ }}, [shouldUseSimpleRendering, settings.linkWidth.value, settings.linkDirectionalArrowLength?.value, settings.linkDirectionalArrowRelPos?.value, settings.nodeSize.value, theme, highlightLinks, highlightNodes]);// Container resize observer with debouncing@@ -631,6 +797,7 @@ const GraphViewer: React.FC<GraphViewerProps> = ({<div className="space-y-2 text-xs text-muted-foreground mb-6"><p><strong>Tip:</strong> Use the search bar to find entities or import data to get started</p><p><strong>Explore:</strong> Try searching for domains, emails, or other entities</p>+ <p><strong>Labels:</strong> Zoom in to see node labels progressively by connection weight</p></div><Button onClick={handleOpenNewAddItemDialog}><svg className="w-4 h-4 mr-2" fill="none" stroke="currentColor" viewBox="0 0 24 24">@@ -653,16 +820,31 @@ const GraphViewer: React.FC<GraphViewerProps> = ({height: height || '100%',minHeight: 300,minWidth: 300,+ position: 'relative',...style}}>+ {tooltip.visible && (+ <div+ className="absolute z-20 bg-background border rounded-lg p-2 shadow-lg text-xs pointer-events-none"+ style={{+ left: tooltip.x,+ top: tooltip.y,+ transform: 'translate(-50%, -100%)' // Center horizontally on the node+ }}+ >+ <div className="whitespace-pre-line truncate flex flex-col gap-1">+ <span className='text-md font-semibold'>{tooltip.data?.label}</span>+ <span className='flex items-center gap-1'><Share2 className='h-3 w-3 opacity-60' /> connections: <span className='font-semibold'>{tooltip.data?.connections}</span></span>+ </div>+ </div>+ )}<ForceGraph2Dref={graphRef}width={dimensions.width || width}height={dimensions.height || height}graphData={graphData}- nodeLabel="label"- nodeColor={node => shouldUseSimpleRendering ? node.nodeColor : GRAPH_COLORS.TRANSPARENT}+ nodeLabel={() => ''} nodeColor={node => shouldUseSimpleRendering ? node.nodeColor : GRAPH_COLORS.TRANSPARENT}nodeRelSize={6}onNodeRightClick={handleNodeRightClick}onNodeClick={handleNodeClick}@@ -682,10 +864,10 @@ const GraphViewer: React.FC<GraphViewerProps> = ({dagLevelDistance={settings.dagLevelDistance?.value}backgroundColor={backgroundColor}onZoom={(zoom) => setCurrentZoom(zoom.k)}- linkCanvasObject={linkCanvasObject}+ linkCanvasObject={renderLink}enableNodeDrag={!shouldUseSimpleRendering}autoPauseRedraw={false}- onNodeHover={handleNodeHover}+ onNodeHover={handleNodeHoverWithTooltip}onLinkHover={handleLinkHover}/></div>diff --git a/flowsint-app/src/renderer/src/components/graphs/graph.tsx b/flowsint-app/src/renderer/src/components/graphs/graph.tsxindex 11da00da..35bb46e4 100644--- a/flowsint-app/src/renderer/src/components/graphs/graph.tsx+++ b/flowsint-app/src/renderer/src/components/graphs/graph.tsx@@ -97,11 +97,12 @@ const GraphContent = memo(() => {const cosmograph = useCosmograph()const actionsSetRef = useRef(false)+ const instanceId = useRef('cosmograph-main') // Add instance-specific IDconst handleLabelClick = useCallback((node: any, event: MouseEvent) => {const multiSelect = event.ctrlKey || event.shiftKey || event.altKeytoggleNodeSelection(node, multiSelect)- cosmograph?.cosmograph?.zoomToNode(node)+ // cosmograph?.cosmograph?.zoomToNode(node)}, [toggleNodeSelection, cosmograph?.cosmograph])//@ts-ignore@@ -113,9 +114,9 @@ const GraphContent = memo(() => {if (event) {const multiSelect = event.ctrlKey || event.shiftKey || event.altKeytoggleNodeSelection(clickedNode, multiSelect)- cosmograph?.cosmograph?.zoomToNode(clickedNode)+ // cosmograph?.cosmograph?.zoomToNode(clickedNode)} else {- cosmograph?.cosmograph?.zoomToNode(clickedNode)+ // cosmograph?.cosmograph?.zoomToNode(clickedNode)}}, [toggleNodeSelection, clearSelectedNodes, cosmograph?.cosmograph])@@ -162,19 +163,31 @@ const GraphContent = memo(() => {cosmograph.cosmograph.restart()actionsSetRef.current = true}- }, [cosmograph?.cosmograph, setActions])++ // Cleanup: reset actions when component unmounts+ return () => {+ if (actionsSetRef.current) {+ setActions({+ zoomIn: () => {},+ zoomOut: () => {},+ zoomToFit: () => {},+ })+ actionsSetRef.current = false+ }+ }+ }, [cosmograph?.cosmograph, setActions, instanceId.current])//@ts-ignore- useEffect(() => {- if (currentNode && cosmograph?.cosmograph && isCosmographReady) {- const timeoutId = setTimeout(() => {- cosmograph.cosmograph?.zoomToNode(currentNode)- cosmograph.cosmograph?.focusNode(currentNode)- }, 100)+ // useEffect(() => {+ // if (currentNode && cosmograph?.cosmograph && isCosmographReady) {+ // const timeoutId = setTimeout(() => {+ // cosmograph.cosmograph?.zoomToNode(currentNode)+ // cosmograph.cosmograph?.focusNode(currentNode)+ // }, 100)- return () => clearTimeout(timeoutId)- }- }, [currentNode, cosmograph?.cosmograph, isCosmographReady])+ // return () => clearTimeout(timeoutId)+ // }+ // }, [currentNode, cosmograph?.cosmograph, isCosmographReady])return (<div className="relative h-full w-full">diff --git a/flowsint-app/src/renderer/src/hooks/use-launch-flow.ts b/flowsint-app/src/renderer/src/hooks/use-launch-flow.tsindex d8c58188..05649b03 100644--- a/flowsint-app/src/renderer/src/hooks/use-launch-flow.ts+++ b/flowsint-app/src/renderer/src/hooks/use-launch-flow.ts@@ -14,9 +14,11 @@ export function useLaunchFlow(askUser: boolean = false) {if (!confirmed) return}const body = JSON.stringify({ values, sketch_id })+ const sliced = values.slice(0, 2)+ const left = values.length - sliced.lengthtoast.promise(flowService.launch(flow_id, body), {loading: "Loading...",- success: () => `Scan on "${values.join(",")}" has been launched.`,+ success: () => `Flow ${flow_id} has been launched on "${sliced.join(",")}" ${left > 0 && ` and ${left} others`}.`,error: () => `An error occurred launching flow.`,})returndiff --git a/flowsint-app/src/renderer/src/hooks/use-launch-transform.ts b/flowsint-app/src/renderer/src/hooks/use-launch-transform.tsindex 08c845c6..6152867c 100644--- a/flowsint-app/src/renderer/src/hooks/use-launch-transform.ts+++ b/flowsint-app/src/renderer/src/hooks/use-launch-transform.ts@@ -14,9 +14,11 @@ export function useLaunchTransform(askUser: boolean = false) {if (!confirmed) return}const body = JSON.stringify({ values, sketch_id })+ const sliced = values.slice(0, 2)+ const left = values.length - sliced.lengthtoast.promise(transformService.launch(transformName, body), {loading: "Loading...",- success: () => `Scan on "${values.join(",")}" has been launched.`,+ success: () => `Transform ${transformName} has been launched on ${sliced.join(", ")}${left > 0 ? ` and ${left} others` : ""}.`,error: () => `An error occurred launching transform.`,})returndiff --git a/flowsint-app/src/renderer/src/stores/node-display-settings.ts b/flowsint-app/src/renderer/src/stores/node-display-settings.tsindex f73c884d..5ae2f299 100644--- a/flowsint-app/src/renderer/src/stores/node-display-settings.ts+++ b/flowsint-app/src/renderer/src/stores/node-display-settings.ts@@ -109,66 +109,67 @@ export const ITEM_TYPES: ItemType[] = ["creditcard","alias","affiliation",- "phrase"+ "phrase",]const DEFAULT_COLORS: Record<ItemType, string> = {- individual: "#E07A7A", // stronger pastel blue- phone: "#5FC9B5", // teal-green- location: "#E57373", // warm rose- email: "#8E7CC3", // lavender purple- ip: "#F4A261", // orange- socialprofile: "#B569B9", // purple pink- organization: "#BCA18A", // taupe- vehicle: "#E1B84D", // golden wheat+ individual: "#4C8EDA", // bleu moyen+ phone: "#4CB5AE", // teal doux+ location: "#E57373", // rouge rosé+ email: "#8E7CC3", // violet pastel+ ip: "#F4A261", // orange chaud+ socialprofile: "#A76DAA", // mauve+ organization: "#BCA18A", // taupe clair+ vehicle: "#E1B84D", // jaune douxcar: "#BFAF7A", // olive beige- motorcycle: "#A78B6C", // warm taupe- boat: "#6FB1C5", // blue teal- plane: "#C1A78E", // light brown- website: "#D279A6", // dusty rose- domain: "#66A892", // sage green- subdomain: "#5AA1C8", // sky blue- document: "#8F9CA3", // cool gray- financial: "#E98973", // coral- event: "#6DBBA2", // mint green- device: "#E3A857", // peach orange+ motorcycle: "#A78B6C", // brun taupe+ boat: "#6FB1C5", // bleu teal+ plane: "#C1A78E", // brun clair+ website: "#D279A6", // rose pastel+ domain: "#66A892", // vert sauge+ subdomain: "#5AA1C8", // bleu ciel+ document: "#8F9CA3", // gris bleuté+ financial: "#E98973", // corail doux+ event: "#6DBBA2", // vert menthe+ device: "#E3A857", // orange sablemedia: "#C97C73", // terracotta- education: "#6C8CBF", // steel blue- relationship: "#B18B84", // mauve brown- online_activity: "#7EAD6F", // sage green- digital_footprint: "#B97777", // brick- username: "#8B83C1", // periwinkle purple- credential: "#D4B030", // gold- biometric: "#7F868D", // slate gray- siret: "#7D8B99", // blue-gray- siren: "#687684", // dark slate- cryptowallet: "#D4B030", // gold yellow- cryptotransaction: "#BFA750", // warm yellow- cryptonft: "#A5BF50", // lime green- asn: "#D97474", // warm peach- cidr: "#80BF80", // mint green- whois: "#9B6F9B", // lavender violet- gravatar: "#6CB7CA", // pale cyan- breach: "#CC7A7A", // warm rose- webtracker: "#C7BF50", // warm yellow- session: "#A8BF50", // lime green- dns: "#80BF9F", // mint teal- ssl: "#BFAF80", // warm sand- message: "#897FC9", // lavender- malware: "#4AA29E", // teal- weapon: "#E98973", // coral brown- script: "#A36FA3", // dusty violet- reputation: "#6FA8DC", // steel blue- risk: "#D97474", // dusty rose- file: "#8F9CA3", // cool gray- bank: "#D4B030", // gold- creditcard: "#285E8E", // deep blue+ education: "#6C8CBF", // bleu acier+ relationship: "#B18B84", // brun rosé+ online_activity: "#7EAD6F", // vert sauge+ digital_footprint: "#B97777", // brique+ username: "#8B83C1", // pervenche+ credential: "#D4B030", // doré doux+ biometric: "#7F868D", // gris ardoise+ siret: "#7D8B99", // gris bleu+ siren: "#687684", // gris foncé+ cryptowallet: "#D4B030", // or clair+ cryptotransaction: "#BFA750", // or chaud+ cryptonft: "#A5BF50", // vert lime doux+ asn: "#D97474", // pêche rosée+ cidr: "#80BF80", // vert menthe+ whois: "#9B6F9B", // violet doux+ gravatar: "#6CB7CA", // cyan clair+ breach: "#CC7A7A", // rose chaud+ webtracker: "#C7BF50", // jaune doux+ session: "#A8BF50", // lime atténué+ dns: "#80BF9F", // vert teal clair+ ssl: "#BFAF80", // sable chaud+ message: "#897FC9", // violet lavande+ malware: "#4AA29E", // teal saturé+ weapon: "#E98973", // corail brun+ script: "#A36FA3", // violet doux+ reputation: "#6FA8DC", // bleu clair+ risk: "#D97474", // rouge doux+ file: "#8F9CA3", // gris bleuté+ bank: "#D4B030", // or clair+ creditcard: "#285E8E", // bleu profondalias: "#A36FA3", // violet- affiliation: "#66A892", // sage- phrase: "#BFA77A" // warm beige+ affiliation: "#66A892", // vert sauge+ phrase: "#BFA77A", // beige chaud};+// Définition des icônes par défaut pour chaque type d'élémentconst DEFAULT_ICONS: Record<ItemType, string> = {individual: "👤",@@ -223,7 +224,7 @@ const DEFAULT_ICONS: Record<ItemType, string> = {creditcard: "💳",alias: "👤",affiliation: "🤝",- phrase: "��"+ phrase: "��",};const DEFAULT_SIZES: Record<ItemType, number> = {@@ -279,7 +280,7 @@ const DEFAULT_SIZES: Record<ItemType, number> = {creditcard: 18, // Medium-largealias: 12, // Small-mediumaffiliation: 14, // Medium- phrase: 10 // Small+ phrase: 10, // Small}interface NodesDisplaySettingsState {diff --git a/flowsint-core/pyproject.toml b/flowsint-core/pyproject.tomlindex 7c97ae66..ad7fce9e 100644--- a/flowsint-core/pyproject.toml+++ b/flowsint-core/pyproject.toml@@ -17,7 +17,6 @@ redis = "^5.0"celery = "^5.3"python-dotenv = "^1.0"requests = "^2.31"-docker = "^6.1"networkx = "^2.6.3"passlib = {extras = ["bcrypt"], version = "^1.7"}python-jose = {extras = ["cryptography"], version = "^3.3"}diff --git a/flowsint-core/src/flowsint_core/core/registry.py b/flowsint-core/src/flowsint_core/core/registry.pyindex 156b8e4c..9d3a4eb7 100644--- a/flowsint-core/src/flowsint_core/core/registry.py+++ b/flowsint-core/src/flowsint_core/core/registry.py@@ -11,6 +11,8 @@ from flowsint_transforms.domain.to_root_domain import DomainToRootDomainfrom flowsint_transforms.domain.to_asn import DomainToAsnScanner# IP-related scanners+from flowsint_transforms.email.to_domains import EmailToDomainsScanner+from flowsint_transforms.individual.to_domains import IndividualToDomainsScannerfrom flowsint_transforms.ip.to_domain import ReverseResolveScannerfrom flowsint_transforms.ip.to_geolocation import GeolocationScanner@@ -24,6 +26,7 @@ from flowsint_transforms.asn.to_cidrs import AsnToCidrsScannerfrom flowsint_transforms.cidr.to_ips import CidrToIpsScanner# Social media scanners+from flowsint_transforms.organization.to_domains import OrgToDomainsScannerfrom flowsint_transforms.social.to_maigret import MaigretScanner# Organization-related scanners@@ -155,7 +158,7 @@ TransformRegistry.register(MaigretScanner)# Organization-related scannersTransformRegistry.register(OrgToAsnScanner)TransformRegistry.register(OrgToInfosScanner)-+TransformRegistry.register(OrgToDomainsScanner)# Cryptocurrency scannersTransformRegistry.register(CryptoWalletAddressToTransactions)TransformRegistry.register(CryptoWalletAddressToNFTs)@@ -170,9 +173,11 @@ TransformRegistry.register(WebsiteToText)# Email-related scannersTransformRegistry.register(EmailToGravatarScanner)TransformRegistry.register(EmailToBreachesScanner)+TransformRegistry.register(EmailToDomainsScanner)# Individual-related scannersTransformRegistry.register(IndividualToOrgScanner)+TransformRegistry.register(IndividualToDomainsScanner)# Integration scannersTransformRegistry.register(N8nConnector)diff --git a/flowsint-transforms/pyproject.toml b/flowsint-transforms/pyproject.tomlindex 55363b9a..d1679188 100644--- a/flowsint-transforms/pyproject.toml+++ b/flowsint-transforms/pyproject.toml@@ -22,7 +22,6 @@ pydantic = "^2.11.7"requests-random-user-agent = "^2023.10.25"spacy = "^3.7"-docker = "^6.1"maigret = { git = "https://github.com/soxoj/maigret" }recontrack = { git = "https://github.com/reconurge/recontrack.git" }reconcrawl = { git = "https://github.com/reconurge/reconcrawl.git" }diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.pyindex 242dbbe9..597a4180 100644--- a/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py@@ -5,6 +5,9 @@ from flowsint_core.core.scanner_base import Scannerfrom flowsint_types.wallet import CryptoWallet, CryptoNFTfrom flowsint_core.core.logger import Loggerfrom flowsint_core.core.graph_db import Neo4jConnection+from dotenv import load_dotenv++load_dotenv()ETHERSCAN_API_URL = os.getenv("ETHERSCAN_API_URL")diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.pyindex 6e2e31d0..59ddb2da 100644--- a/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py@@ -3,11 +3,14 @@ from typing import List, Dict, Any, Optional, Unionimport requestsimport requests.exceptionsfrom datetime import datetime+from dotenv import load_dotenvfrom flowsint_core.core.scanner_base import Scannerfrom flowsint_types.wallet import CryptoWallet, CryptoWalletTransactionfrom flowsint_core.core.graph_db import Neo4jConnectionfrom flowsint_core.core.logger import Logger+load_dotenv()+ETHERSCAN_API_URL = os.getenv("ETHERSCAN_API_URL")diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py b/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.pyindex 9010729d..4cb590d5 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py@@ -45,7 +45,7 @@ class SubdomainScanner(Scanner):async def scan(self, data: InputType) -> OutputType:"""Find subdomains using subfinder (if available) or fallback to crt.sh."""domains: OutputType = []- use_subfinder = not self.__is_subfinder_installed()+ use_subfinder = self.__is_subfinder_installed()for md in data:d = Domain(domain=md.domain)@@ -54,11 +54,16 @@ class SubdomainScanner(Scanner):if not subdomains:Logger.warn(self.sketch_id,- f"subfinder failed for {d.domain}, falling back to crt.sh",+ {+ "message": f"subfinder failed for {d.domain}, falling back to crt.sh"+ },)subdomains = self.__get_subdomains_from_crtsh(d.domain)else:- Logger.info(self.sketch_id, "subfinder not found, using crt.sh only")+ Logger.info(+ self.sketch_id,+ {"message": "subfinder not found, using crt.sh only"},+ )subdomains = self.__get_subdomains_from_crtsh(d.domain)domains.append({"domain": d.domain, "subdomains": sorted(subdomains)})@@ -90,7 +95,9 @@ class SubdomainScanner(Scanner):elif "*" in sub:continueexcept Exception as e:- Logger.error(self.sketch_id, f"crt.sh failed for {domain}: {e}")+ Logger.error(+ self.sketch_id, {"message": f"crt.sh failed for {domain}: {e}"}+ )return subdomainsdef __get_subdomains_from_subfinder(self, domain: str) -> set[str]:@@ -99,7 +106,9 @@ class SubdomainScanner(Scanner):try:subdomains = subfinder.launch(domain)except Exception as e:- Logger.error(self.sketch_id, f"subfinder exception for {domain}: {e}")+ Logger.error(+ self.sketch_id, {"message": f"subfinder exception for {domain}: {e}"}+ )return subdomainsdef postprocess(self, results: OutputType, original_input: InputType) -> OutputType:diff --git a/flowsint-transforms/src/flowsint_transforms/email/to_domains.py b/flowsint-transforms/src/flowsint_transforms/email/to_domains.pynew file mode 100644index 00000000..46dde35c--- /dev/null+++ b/flowsint-transforms/src/flowsint_transforms/email/to_domains.py@@ -0,0 +1,409 @@+import os+import re+from typing import Any, List, Union, Dict, Set+from flowsint_core.core.scanner_base import Scanner+from flowsint_types.domain import Domain+from flowsint_types.individual import Individual+from flowsint_types.email import Email+from flowsint_types.address import PhysicalAddress+from flowsint_core.core.logger import Logger+from tools.network.whoxy import WhoxyTool+from dotenv import load_dotenv++load_dotenv()++WHOXY_API_KEY = os.getenv("WHOXY_API_KEY")+++class EmailToDomainsScanner(Scanner):+ """[WHOXY] Takes an email and returns the domains it registered."""++ # Define types as class attributes - base class handles schema generation automatically+ InputType = List[Email]+ OutputType = List[Domain]++ @classmethod+ def name(cls) -> str:+ return "email_to_domains_scanner"++ @classmethod+ def category(cls) -> str:+ return "Email"++ @classmethod+ def key(cls) -> str:+ return "email"++ def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:+ cleaned: InputType = []+ for item in data:+ email_obj = None+ if isinstance(item, str):+ email_obj = Email(email=item)+ elif isinstance(item, dict) and "email" in item:+ email_obj = Email(email=item["email"])+ elif isinstance(item, Email):+ email_obj = item+ if email_obj:+ cleaned.append(email_obj)+ return cleaned++ async def scan(self, data: InputType) -> OutputType:+ """Find domains related to emails using whoxy api."""+ domains: OutputType = []+ self._extracted_data = [] # Store all extracted data for postprocess++ for email in data:+ infos_data = self.__get_infos_from_whoxy(email.email)+ if infos_data and "search_result" in infos_data:+ # Process each domain result+ for result in infos_data["search_result"]:+ if self.__is_valid_domain_result(result):+ domain_name = result.get("domain_name")+ if domain_name:+ domain = Domain(domain=domain_name, root=True)+ domains.append(domain)++ # Store extracted data for postprocess+ extracted_info = {+ "email": email,+ "domain": domain,+ "domain_data": result,+ "contacts": {+ "registrant": result.get("registrant_contact", {}),+ "administrative": result.get(+ "administrative_contact", {}+ ),+ "technical": result.get("technical_contact", {}),+ "billing": result.get("billing_contact", {}),+ },+ }+ self._extracted_data.append(extracted_info)+ else:+ Logger.info(+ self.sketch_id,+ {"message": f"[WHOXY] No domain found for email {email.email}."},+ )+ return domains++ def __get_infos_from_whoxy(self, email: str) -> Dict[str, Any]:+ infos: Dict[str, Any] = {}+ whoxy = WhoxyTool()+ try:+ params = {+ "key": WHOXY_API_KEY,+ "reverse": "whois",+ "email": email,+ }+ infos = whoxy.launch(params=params)+ except Exception as e:+ Logger.error(+ self.sketch_id, {"message": f"[WHOXY] Whoxy exception for {email}: {e}"}+ )+ return infos++ def __is_valid_domain_result(self, result: Dict[str, Any]) -> bool:+ """Check if a domain result is valid."""+ domain_name = result.get("domain_name")+ if not domain_name:+ return False+ return True++ def __extract_individual_from_contact(+ self, contact: Dict[str, Any], contact_type: str+ ) -> Individual:+ """Extract individual information from contact data."""+ full_name = contact.get("full_name", "")++ # Skip if name is redacted+ if "REDACTED" in full_name or "REDACTED FOR PRIVACY" in full_name:+ return None++ # Parse full name into first and last name+ name_parts = full_name.strip().split()+ first_name = name_parts[0] if name_parts else ""+ last_name = " ".join(name_parts[1:]) if len(name_parts) > 1 else ""++ # Extract email and phone+ email = contact.get("email_address", "")+ phone = contact.get("phone_number", "")++ # Skip if email is redacted or invalid+ if (+ "REDACTED" in email+ or "REDACTED FOR PRIVACY" in email+ or not self.__is_valid_email(email)+ ):+ email = ""++ # Skip if phone is redacted+ if "REDACTED" in phone or "REDACTED FOR PRIVACY" in phone:+ phone = ""++ # Extract address information+ address = contact.get("mailing_address", "")+ city = contact.get("city_name", "")+ zip_code = contact.get("zip_code", "")+ country = contact.get("country_name", "")++ # Skip if address is redacted+ if "REDACTED" in address or "REDACTED FOR PRIVACY" in address:+ address = ""+ if "REDACTED" in city or "REDACTED FOR PRIVACY" in city:+ city = ""+ if "REDACTED" in zip_code or "REDACTED FOR PRIVACY" in zip_code:+ zip_code = ""+ if "REDACTED" in country or "REDACTED FOR PRIVACY" in country:+ country = ""++ # Create individual object+ individual = Individual(+ first_name=first_name,+ last_name=last_name,+ full_name=full_name,+ email_addresses=[email] if email else None,+ phone_numbers=[phone] if phone else None,+ )++ return individual++ def __is_valid_email(self, email: str) -> bool:+ """Check if email is valid."""+ if not email:+ return False+ # Basic email validation+ pattern = r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$"+ return bool(re.match(pattern, email))++ def __extract_physical_address(self, contact: Dict[str, Any]) -> PhysicalAddress:+ """Extract physical address from contact data."""+ address = contact.get("mailing_address", "")+ city = contact.get("city_name", "")+ zip_code = contact.get("zip_code", "")+ country = contact.get("country_name", "")++ # Skip if any part is redacted+ if any("REDACTED" in field for field in [address, city, zip_code, country]):+ return None++ if not all([address, city, zip_code, country]):+ return None++ return PhysicalAddress(+ address=address, city=city, zip=zip_code, country=country+ )++ def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ """Create Neo4j nodes and relationships from extracted data."""+ if not self.neo4j_conn:+ return results++ # Track processed entities to avoid duplicates+ processed_domains: Set[str] = set()+ processed_individuals: Set[str] = set()+ processed_emails: Set[str] = set()+ processed_phones: Set[str] = set()+ processed_addresses: Set[str] = set()++ for extracted_info in self._extracted_data:+ email = extracted_info["email"]+ domain = extracted_info["domain"]+ domain_data = extracted_info["domain_data"]+ contacts = extracted_info["contacts"]++ domain_name = domain.domain+ if domain_name in processed_domains:+ continue+ processed_domains.add(domain_name)++ # Create email node+ self.create_node(+ "email",+ "email",+ email.email,+ caption=email.email,+ type="email",+ )++ # Create domain node+ self.create_node(+ "domain",+ "domain",+ domain_name,+ label=domain_name,+ caption=domain_name,+ type="domain",+ )++ # Create relationship between email and domain+ self.create_relationship(+ "email",+ "email",+ email.email,+ "domain",+ "domain",+ domain_name,+ "HAS_REGISTERED_DOMAIN",+ )++ # Process all contact types+ for contact_type, contact in contacts.items():+ if contact:+ self.__process_contact(+ contact,+ contact_type.upper(),+ domain_name,+ email.email,+ processed_individuals,+ processed_emails,+ processed_phones,+ processed_addresses,+ )++ self.log_graph_message(+ f"Processed domain {domain_name} for email {email.email}"+ )++ return results++ def __process_contact(+ self,+ contact: Dict[str, Any],+ contact_type: str,+ domain_name: str,+ email_address: str,+ processed_individuals: Set[str],+ processed_emails: Set[str],+ processed_phones: Set[str],+ processed_addresses: Set[str],+ ):+ """Process a contact and create all related entities and relationships."""++ # Extract individual+ individual = self.__extract_individual_from_contact(contact, contact_type)+ if not individual:+ return++ individual_id = (+ f"{individual.first_name}_{individual.last_name}_{individual.full_name}"+ )+ if individual_id in processed_individuals:+ return++ processed_individuals.add(individual_id)++ # Create individual node+ self.create_node(+ "individual",+ "full_name",+ individual.full_name,+ caption=individual.full_name,+ type="individual",+ )++ # Create relationship between individual and domain+ self.create_relationship(+ "individual",+ "full_name",+ individual.full_name,+ "domain",+ "domain",+ domain_name,+ f"IS_{contact_type}_CONTACT",+ )++ # Create relationship between individual and email+ self.create_relationship(+ "individual",+ "full_name",+ individual.full_name,+ "email",+ "email",+ email_address,+ f"WORKS_FOR",+ )++ # Process email addresses+ if individual.email_addresses:+ for email in individual.email_addresses:+ if email and email not in processed_emails:+ processed_emails.add(email)++ # Create email node+ self.create_node(+ "email",+ "email",+ email,+ caption=email,+ type="email",+ )++ # Create relationship between individual and email+ self.create_relationship(+ "individual",+ "full_name",+ individual.full_name,+ "email",+ "email",+ email,+ "HAS_EMAIL",+ )++ # Process phone numbers+ if individual.phone_numbers:+ for phone in individual.phone_numbers:+ if phone and phone not in processed_phones:+ processed_phones.add(phone)++ # Create phone node+ self.create_node(+ "phone",+ "number",+ phone,+ caption=phone,+ type="phone",+ )++ # Create relationship between individual and phone+ self.create_relationship(+ "individual",+ "full_name",+ individual.full_name,+ "phone",+ "number",+ phone,+ "HAS_PHONE",+ )++ # Process physical address+ address = self.__extract_physical_address(contact)+ if address:+ address_id = (+ f"{address.address}_{address.city}_{address.zip}_{address.country}"+ )+ if address_id not in processed_addresses:+ processed_addresses.add(address_id)++ # Create address node+ self.create_node(+ "location",+ "address",+ address.address,+ caption=f"{address.address}, {address.city}",+ type="location",+ )++ # Create relationship between individual and address+ self.create_relationship(+ "individual",+ "full_name",+ individual.full_name,+ "location",+ "address",+ address.address,+ "LIVES_AT",+ )+++InputType = EmailToDomainsScanner.InputType+OutputType = EmailToDomainsScanner.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py b/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.pyindex c8d418bc..77ff2de3 100644--- a/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py+++ b/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py@@ -15,7 +15,7 @@ class EmailToGravatarScanner(Scanner):@classmethoddef name(cls) -> str:- return "to_gravatar"+ return "email_to_gravatar"@classmethoddef category(cls) -> str:diff --git a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.pyindex cc04f33d..9741805a 100644--- a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py+++ b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py@@ -40,7 +40,7 @@ class EmailToBreachesScanner(Scanner):@classmethoddef name(cls) -> str:- return "to_breaches"+ return "email_to_breaches"@classmethoddef category(cls) -> str:diff --git a/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py b/flowsint-transforms/src/flowsint_transforms/individual/to_domains.pynew file mode 100644index 00000000..27bfb0d0--- /dev/null+++ b/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py@@ -0,0 +1,427 @@+import os+import re+from typing import Any, List, Union, Dict, Set+from flowsint_core.core.scanner_base import Scanner+from flowsint_types.domain import Domain+from flowsint_types.organization import Organization+from flowsint_types.individual import Individual+from flowsint_types.address import PhysicalAddress+from flowsint_core.core.logger import Logger+from tools.network.whoxy import WhoxyTool+from dotenv import load_dotenv++load_dotenv()++WHOXY_API_KEY = os.getenv("WHOXY_API_KEY")+++class IndividualToDomainsScanner(Scanner):+ """[WHOXY] Takes an individual and returns the domains it registered."""++ # Define types as class attributes - base class handles schema generation automatically+ InputType = List[Individual]+ OutputType = List[Domain]++ @classmethod+ def name(cls) -> str:+ return "individual_to_domains_scanner"++ @classmethod+ def category(cls) -> str:+ return "Individual"++ @classmethod+ def key(cls) -> str:+ return "full_name"++ def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:+ if not isinstance(data, list):+ raise ValueError(f"Expected list input, got {type(data).__name__}")+ cleaned: InputType = []+ for item in data:+ if isinstance(item, str):+ parts = item.strip().split()+ if len(parts) >= 2:+ firstname = parts[0]+ lastname = " ".join(parts[1:])+ cleaned.append(+ Individual(+ first_name=firstname, last_name=lastname, full_name=item+ )+ )+ elif (+ isinstance(item, dict)+ and item.get("first_name")+ and item.get("last_name")+ ):+ cleaned.append(Individual(**item))+ elif isinstance(item, Individual):+ cleaned.append(item)+ if len(cleaned) == 0:+ Logger.error(+ self.sketch_id,+ {"message": "[INDIVIDUAL_TO_DOMAINS] The input type did not match."},+ )+ return cleaned++ async def scan(self, data: InputType) -> OutputType:+ """Find domains related to individuals using whoxy api."""+ domains: OutputType = []+ self._extracted_data = [] # Store all extracted data for postprocess++ for individual in data:+ infos_data = self.__get_infos_from_whoxy(individual.full_name)+ if infos_data and "search_result" in infos_data:+ # Process each domain result+ for result in infos_data["search_result"]:+ if self.__is_valid_domain_result(result):+ domain_name = result.get("domain_name")+ if domain_name:+ domain = Domain(domain=domain_name, root=True)+ domains.append(domain)++ # Store extracted data for postprocess+ extracted_info = {+ "individual": individual,+ "domain": domain,+ "domain_data": result,+ "contacts": {+ "registrant": result.get("registrant_contact", {}),+ "administrative": result.get(+ "administrative_contact", {}+ ),+ "technical": result.get("technical_contact", {}),+ "billing": result.get("billing_contact", {}),+ },+ }+ self._extracted_data.append(extracted_info)+ else:+ Logger.info(+ self.sketch_id,+ {"message": f"[WHOXY] No domain found for individual {individual.full_name}."},+ )+ return domains++ def __get_infos_from_whoxy(self, individual_name: str) -> Dict[str, Any]:+ infos: Dict[str, Any] = {}+ whoxy = WhoxyTool()+ try:+ params = {+ "key": WHOXY_API_KEY,+ "reverse": "whois",+ "name": individual_name,+ }+ infos = whoxy.launch(params=params)+ except Exception as e:+ Logger.error(+ self.sketch_id,+ {"message": f"[WHOXY] Whoxy exception for {individual_name}: {e}"},+ )+ return infos++ def __is_valid_domain_result(self, result: Dict[str, Any]) -> bool:+ """Check if a domain result is valid."""+ domain_name = result.get("domain_name")+ if not domain_name:+ return False+ return True++ def __extract_individual_from_contact(+ self, contact: Dict[str, Any], contact_type: str+ ) -> Individual:+ """Extract individual information from contact data."""+ full_name = contact.get("full_name", "")++ # Skip if name is redacted+ if "REDACTED" in full_name or "REDACTED FOR PRIVACY" in full_name:+ return None++ # Parse full name into first and last name+ name_parts = full_name.strip().split()+ first_name = name_parts[0] if name_parts else ""+ last_name = " ".join(name_parts[1:]) if len(name_parts) > 1 else ""++ # Extract email and phone+ email = contact.get("email_address", "")+ phone = contact.get("phone_number", "")++ # Skip if email is redacted or invalid+ if (+ "REDACTED" in email+ or "REDACTED FOR PRIVACY" in email+ or not self.__is_valid_email(email)+ ):+ email = ""++ # Skip if phone is redacted+ if "REDACTED" in phone or "REDACTED FOR PRIVACY" in phone:+ phone = ""++ # Extract address information+ address = contact.get("mailing_address", "")+ city = contact.get("city_name", "")+ zip_code = contact.get("zip_code", "")+ country = contact.get("country_name", "")++ # Skip if address is redacted+ if "REDACTED" in address or "REDACTED FOR PRIVACY" in address:+ address = ""+ if "REDACTED" in city or "REDACTED FOR PRIVACY" in city:+ city = ""+ if "REDACTED" in zip_code or "REDACTED FOR PRIVACY" in zip_code:+ zip_code = ""+ if "REDACTED" in country or "REDACTED FOR PRIVACY" in country:+ country = ""++ # Create individual object+ individual = Individual(+ first_name=first_name,+ last_name=last_name,+ full_name=full_name,+ email_addresses=[email] if email else None,+ phone_numbers=[phone] if phone else None,+ )++ return individual++ def __is_valid_email(self, email: str) -> bool:+ """Check if email is valid."""+ if not email:+ return False+ # Basic email validation+ pattern = r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$"+ return bool(re.match(pattern, email))++ def __extract_physical_address(self, contact: Dict[str, Any]) -> PhysicalAddress:+ """Extract physical address from contact data."""+ address = contact.get("mailing_address", "")+ city = contact.get("city_name", "")+ zip_code = contact.get("zip_code", "")+ country = contact.get("country_name", "")++ # Skip if any part is redacted+ if any("REDACTED" in field for field in [address, city, zip_code, country]):+ return None++ if not all([address, city, zip_code, country]):+ return None++ return PhysicalAddress(+ address=address, city=city, zip=zip_code, country=country+ )++ def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ """Create Neo4j nodes and relationships from extracted data."""+ if not self.neo4j_conn:+ return results++ # Track processed entities to avoid duplicates+ processed_domains: Set[str] = set()+ processed_individuals: Set[str] = set()+ processed_emails: Set[str] = set()+ processed_phones: Set[str] = set()+ processed_addresses: Set[str] = set()++ for extracted_info in self._extracted_data:+ individual = extracted_info["individual"]+ domain = extracted_info["domain"]+ domain_data = extracted_info["domain_data"]+ contacts = extracted_info["contacts"]++ domain_name = domain.domain+ if domain_name in processed_domains:+ continue+ processed_domains.add(domain_name)++ # Create individual node+ self.create_node(+ "individual",+ "full_name",+ individual.full_name,+ caption=individual.full_name,+ type="individual",+ )++ # Create domain node+ self.create_node(+ "domain",+ "domain",+ domain_name,+ label=domain_name,+ caption=domain_name,+ type="domain",+ )++ # Create relationship between individual and domain+ self.create_relationship(+ "individual",+ "full_name",+ individual.full_name,+ "domain",+ "domain",+ domain_name,+ "HAS_REGISTERED_DOMAIN",+ )++ # Process all contact types+ for contact_type, contact in contacts.items():+ if contact:+ self.__process_contact(+ contact,+ contact_type.upper(),+ domain_name,+ individual.full_name,+ processed_individuals,+ processed_emails,+ processed_phones,+ processed_addresses,+ )++ self.log_graph_message(+ f"Processed domain {domain_name} for individual {individual.full_name}"+ )++ return results++ def __process_contact(+ self,+ contact: Dict[str, Any],+ contact_type: str,+ domain_name: str,+ individual_name: str,+ processed_individuals: Set[str],+ processed_emails: Set[str],+ processed_phones: Set[str],+ processed_addresses: Set[str],+ ):+ """Process a contact and create all related entities and relationships."""++ # Extract individual+ contact_individual = self.__extract_individual_from_contact(contact, contact_type)+ if not contact_individual:+ return++ individual_id = (+ f"{contact_individual.first_name}_{contact_individual.last_name}_{contact_individual.full_name}"+ )+ if individual_id in processed_individuals:+ return++ processed_individuals.add(individual_id)++ # Create individual node+ self.create_node(+ "individual",+ "full_name",+ contact_individual.full_name,+ caption=contact_individual.full_name,+ type="individual",+ )++ # Create relationship between individual and domain+ self.create_relationship(+ "individual",+ "full_name",+ contact_individual.full_name,+ "domain",+ "domain",+ domain_name,+ f"IS_{contact_type}_CONTACT",+ )++ # Create relationship between contact individual and main individual+ self.create_relationship(+ "individual",+ "full_name",+ contact_individual.full_name,+ "individual",+ "full_name",+ individual_name,+ f"WORKS_FOR",+ )++ # Process email addresses+ if contact_individual.email_addresses:+ for email in contact_individual.email_addresses:+ if email and email not in processed_emails:+ processed_emails.add(email)++ # Create email node+ self.create_node(+ "email",+ "email",+ email,+ caption=email,+ type="email",+ )++ # Create relationship between individual and email+ self.create_relationship(+ "individual",+ "full_name",+ contact_individual.full_name,+ "email",+ "email",+ email,+ "HAS_EMAIL",+ )++ # Process phone numbers+ if contact_individual.phone_numbers:+ for phone in contact_individual.phone_numbers:+ if phone and phone not in processed_phones:+ processed_phones.add(phone)++ # Create phone node+ self.create_node(+ "phone",+ "number",+ phone,+ caption=phone,+ type="phone",+ )++ # Create relationship between individual and phone+ self.create_relationship(+ "individual",+ "full_name",+ contact_individual.full_name,+ "phone",+ "number",+ phone,+ "HAS_PHONE",+ )++ # Process physical address+ address = self.__extract_physical_address(contact)+ if address:+ address_id = (+ f"{address.address}_{address.city}_{address.zip}_{address.country}"+ )+ if address_id not in processed_addresses:+ processed_addresses.add(address_id)++ # Create address node+ self.create_node(+ "location",+ "address",+ address.address,+ caption=f"{address.address}, {address.city}",+ type="location",+ )++ # Create relationship between individual and address+ self.create_relationship(+ "individual",+ "full_name",+ contact_individual.full_name,+ "location",+ "address",+ address.address,+ "LIVES_AT",+ )+++# Make types available at module level for easy access+InputType = IndividualToDomainsScanner.InputType+OutputType = IndividualToDomainsScanner.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/individual/to_org.py b/flowsint-transforms/src/flowsint_transforms/individual/to_org.pyindex b4f3351c..085a09aa 100644--- a/flowsint-transforms/src/flowsint_transforms/individual/to_org.py+++ b/flowsint-transforms/src/flowsint_transforms/individual/to_org.py@@ -16,7 +16,7 @@ class IndividualToOrgScanner(Scanner):@classmethoddef name(cls) -> str:- return "to_org"+ return "individual_to_org"@classmethoddef category(cls) -> str:diff --git a/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py b/flowsint-transforms/src/flowsint_transforms/organization/to_domains.pynew file mode 100644index 00000000..64c1a60e--- /dev/null+++ b/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py@@ -0,0 +1,410 @@+import os+import re+from typing import Any, List, Union, Dict, Set+from flowsint_core.core.scanner_base import Scanner+from flowsint_types.domain import Domain+from flowsint_types.organization import Organization+from flowsint_types.individual import Individual+from flowsint_types.email import Email+from flowsint_types.phone import Phone+from flowsint_types.address import PhysicalAddress+from flowsint_core.core.logger import Logger+from tools.network.whoxy import WhoxyTool+from dotenv import load_dotenv++load_dotenv()++WHOXY_API_KEY = os.getenv("WHOXY_API_KEY")+++class OrgToDomainsScanner(Scanner):+ """[WHOXY] Takes an organization and returns the domains it registered."""++ # Define types as class attributes - base class handles schema generation automatically+ InputType = List[Organization]+ OutputType = List[Domain]++ @classmethod+ def name(cls) -> str:+ return "org_to_domains_scanner"++ @classmethod+ def category(cls) -> str:+ return "Organization"++ @classmethod+ def key(cls) -> str:+ return "name"++ def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:+ cleaned: InputType = []+ for item in data:+ org_obj = None+ if isinstance(item, str):+ org_obj = Organization(name=item)+ elif isinstance(item, dict) and "name" in item:+ org_obj = Organization(name=item["name"])+ elif isinstance(item, Organization):+ org_obj = item+ if org_obj:+ cleaned.append(org_obj)+ return cleaned++ async def scan(self, data: InputType) -> OutputType:+ """Find domains related to organizations using whoxy api."""+ domains: OutputType = []+ self._extracted_data = [] # Store all extracted data for postprocess++ for org in data:+ infos_data = self.__get_infos_from_whoxy(org.name)+ if infos_data and "search_result" in infos_data:+ # Process each domain result+ for result in infos_data["search_result"]:+ if self.__is_valid_domain_result(result):+ domain_name = result.get("domain_name")+ if domain_name:+ domain = Domain(domain=domain_name, root=True)+ domains.append(domain)++ # Store extracted data for postprocess+ extracted_info = {+ 'org': org,+ 'domain': domain,+ 'domain_data': result,+ 'contacts': {+ 'registrant': result.get("registrant_contact", {}),+ 'administrative': result.get("administrative_contact", {}),+ 'technical': result.get("technical_contact", {}),+ 'billing': result.get("billing_contact", {})+ }+ }+ self._extracted_data.append(extracted_info)+ else:+ Logger.info(+ self.sketch_id,+ {"message": f"[WHOXY] No domain found for org {org.name}."},+ )+ return domains++ def __get_infos_from_whoxy(self, org_name: str) -> Dict[str, Any]:+ infos: Dict[str, Any] = {}+ whoxy = WhoxyTool()+ try:+ params = {+ "key": WHOXY_API_KEY,+ "reverse": "whois",+ "company": org_name,+ }+ infos = whoxy.launch(params=params)+ except Exception as e:+ Logger.error(+ self.sketch_id,+ {"message": f"[WHOXY] Whoxy exception for {org_name}: {e}"},+ )+ return infos++ def __is_valid_domain_result(self, result: Dict[str, Any]) -> bool:+ """Check if a domain result is valid."""+ domain_name = result.get("domain_name")+ if not domain_name:+ return False+ return True++ def __extract_individual_from_contact(+ self, contact: Dict[str, Any], contact_type: str+ ) -> Individual:+ """Extract individual information from contact data."""+ full_name = contact.get("full_name", "")++ # Skip if name is redacted+ if "REDACTED" in full_name or "REDACTED FOR PRIVACY" in full_name:+ return None++ # Parse full name into first and last name+ name_parts = full_name.strip().split()+ first_name = name_parts[0] if name_parts else ""+ last_name = " ".join(name_parts[1:]) if len(name_parts) > 1 else ""++ # Extract email and phone+ email = contact.get("email_address", "")+ phone = contact.get("phone_number", "")++ # Skip if email is redacted or invalid+ if (+ "REDACTED" in email+ or "REDACTED FOR PRIVACY" in email+ or not self.__is_valid_email(email)+ ):+ email = ""++ # Skip if phone is redacted+ if "REDACTED" in phone or "REDACTED FOR PRIVACY" in phone:+ phone = ""++ # Extract address information+ address = contact.get("mailing_address", "")+ city = contact.get("city_name", "")+ zip_code = contact.get("zip_code", "")+ country = contact.get("country_name", "")++ # Skip if address is redacted+ if "REDACTED" in address or "REDACTED FOR PRIVACY" in address:+ address = ""+ if "REDACTED" in city or "REDACTED FOR PRIVACY" in city:+ city = ""+ if "REDACTED" in zip_code or "REDACTED FOR PRIVACY" in zip_code:+ zip_code = ""+ if "REDACTED" in country or "REDACTED FOR PRIVACY" in country:+ country = ""++ # Create individual object+ individual = Individual(+ first_name=first_name,+ last_name=last_name,+ full_name=full_name,+ email_addresses=[email] if email else None,+ phone_numbers=[phone] if phone else None,+ )++ return individual++ def __is_valid_email(self, email: str) -> bool:+ """Check if email is valid."""+ if not email:+ return False+ # Basic email validation+ pattern = r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$"+ return bool(re.match(pattern, email))++ def __extract_physical_address(self, contact: Dict[str, Any]) -> PhysicalAddress:+ """Extract physical address from contact data."""+ address = contact.get("mailing_address", "")+ city = contact.get("city_name", "")+ zip_code = contact.get("zip_code", "")+ country = contact.get("country_name", "")++ # Skip if any part is redacted+ if any("REDACTED" in field for field in [address, city, zip_code, country]):+ return None++ if not all([address, city, zip_code, country]):+ return None++ return PhysicalAddress(+ address=address, city=city, zip=zip_code, country=country+ )++ def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ """Create Neo4j nodes and relationships from extracted data."""+ if not self.neo4j_conn:+ return results++ # Track processed entities to avoid duplicates+ processed_domains: Set[str] = set()+ processed_individuals: Set[str] = set()+ processed_emails: Set[str] = set()+ processed_phones: Set[str] = set()+ processed_addresses: Set[str] = set()++ for extracted_info in self._extracted_data:+ org = extracted_info['org']+ domain = extracted_info['domain']+ domain_data = extracted_info['domain_data']+ contacts = extracted_info['contacts']++ domain_name = domain.domain+ if domain_name in processed_domains:+ continue+ processed_domains.add(domain_name)++ # Create organization node+ self.create_node(+ "organization",+ "name",+ org.name,+ caption=org.name,+ type="organization",+ )++ # Create domain node+ self.create_node(+ "domain",+ "domain",+ domain_name,+ label=domain_name,+ caption=domain_name,+ type="domain",+ )++ # Create relationship between organization and domain+ self.create_relationship(+ "organization",+ "name",+ org.name,+ "domain",+ "domain",+ domain_name,+ "HAS_REGISTERED_DOMAIN",+ )++ # Process all contact types+ for contact_type, contact in contacts.items():+ if contact:+ self.__process_contact(+ contact,+ contact_type.upper(),+ domain_name,+ org.name,+ processed_individuals,+ processed_emails,+ processed_phones,+ processed_addresses,+ )++ self.log_graph_message(+ f"Processed domain {domain_name} for organization {org.name}"+ )++ return results++ def __process_contact(+ self,+ contact: Dict[str, Any],+ contact_type: str,+ domain_name: str,+ org_name: str,+ processed_individuals: Set[str],+ processed_emails: Set[str],+ processed_phones: Set[str],+ processed_addresses: Set[str],+ ):+ """Process a contact and create all related entities and relationships."""++ # Extract individual+ individual = self.__extract_individual_from_contact(contact, contact_type)+ if not individual:+ return++ individual_id = (+ f"{individual.first_name}_{individual.last_name}_{individual.full_name}"+ )+ if individual_id in processed_individuals:+ return++ processed_individuals.add(individual_id)++ # Create individual node+ self.create_node(+ "individual",+ "full_name",+ individual.full_name,+ caption=individual.full_name,+ type="individual",+ )++ # Create relationship between individual and domain+ self.create_relationship(+ "individual",+ "full_name",+ individual.full_name,+ "domain",+ "domain",+ domain_name,+ f"IS_{contact_type}_CONTACT",+ )++ # Create relationship between individual and organization+ self.create_relationship(+ "individual",+ "full_name",+ individual.full_name,+ "organization",+ "name",+ org_name,+ f"WORKS_FOR",+ )++ # Process email addresses+ if individual.email_addresses:+ for email in individual.email_addresses:+ if email and email not in processed_emails:+ processed_emails.add(email)++ # Create email node+ self.create_node(+ "email",+ "email",+ email,+ caption=email,+ type="email",+ )++ # Create relationship between individual and email+ self.create_relationship(+ "individual",+ "full_name",+ individual.full_name,+ "email",+ "email",+ email,+ "HAS_EMAIL",+ )++ # Process phone numbers+ if individual.phone_numbers:+ for phone in individual.phone_numbers:+ if phone and phone not in processed_phones:+ processed_phones.add(phone)++ # Create phone node+ self.create_node(+ "phone",+ "number",+ phone,+ caption=phone,+ type="phone",+ )++ # Create relationship between individual and phone+ self.create_relationship(+ "individual",+ "full_name",+ individual.full_name,+ "phone",+ "number",+ phone,+ "HAS_PHONE",+ )++ # Process physical address+ address = self.__extract_physical_address(contact)+ if address:+ address_id = (+ f"{address.address}_{address.city}_{address.zip}_{address.country}"+ )+ if address_id not in processed_addresses:+ processed_addresses.add(address_id)++ # Create address node+ self.create_node(+ "location",+ "address",+ address.address,+ caption=f"{address.address}, {address.city}",+ type="location",+ )++ # Create relationship between individual and address+ self.create_relationship(+ "individual",+ "full_name",+ individual.full_name,+ "location",+ "address",+ address.address,+ "LIVES_AT",+ )+++InputType = OrgToDomainsScanner.InputType+OutputType = OrgToDomainsScanner.OutputTypediff --git a/flowsint-transforms/src/tools/network/whoxy.py b/flowsint-transforms/src/tools/network/whoxy.pynew file mode 100644index 00000000..3faabe0b--- /dev/null+++ b/flowsint-transforms/src/tools/network/whoxy.py@@ -0,0 +1,42 @@+from typing import Dict++import requests+from ..base import Tool+++class WhoxyTool(Tool):++ @classmethod+ def name(cls) -> str:+ return "whoxy"++ @classmethod+ def version(cls) -> str:+ return "1.0.0"++ @classmethod+ def description(cls) -> str:+ return "The WHOIS API returns consistent and well-structured WHOIS data in XML & JSON format. Returned data contain parsed WHOIS fields that can be easily understood. Along with WHOIS API, Whoxy also offer WHOIS History API and Reverse WHOIS API."++ @classmethod+ def category(cls) -> str:+ return "Network intelligence"++ def launch(self, params: Dict[str, str] = {}) -> list[Dict]:+ try:+ resp = requests.get(+ "https://api.whoxy.com/",+ params=params,+ timeout=10,+ )+ resp.raise_for_status()+ data = resp.json()+ if data.get("status") != 1:+ raise RuntimeError(+ f"Error querying Whoxy API: {str(data.get("status_reason") )}"+ )+ if data.get("total_results") == 0:+ raise ValueError(f"No match found for Whoxy search.")+ return data+ except Exception as e:+ raise RuntimeError(f"{str(e)}")diff --git a/flowsint-transforms/tests/tools/network/whoxy.py b/flowsint-transforms/tests/tools/network/whoxy.pynew file mode 100644index 00000000..f3545b24--- /dev/null+++ b/flowsint-transforms/tests/tools/network/whoxy.py@@ -0,0 +1,18 @@+from tools.network.whoxy import WhoxyTool++tool = WhoxyTool()+++def test_name():+ assert tool.name() == "whoxy"+++def test_description():+ assert (+ tool.description()+ == "The WHOIS API returns consistent and well-structured WHOIS data in XML & JSON format. Returned data contain parsed WHOIS fields that can be easily understood. Along with WHOIS API, Whoxy also offer WHOIS History API and Reverse WHOIS API."+ )+++def test_category():+ assert tool.category() == "Network intelligence"diff --git a/flowsint-types/src/flowsint_types/document.py b/flowsint-types/src/flowsint_types/document.pyindex f8e5f0d6..7be38dee 100644--- a/flowsint-types/src/flowsint_types/document.py+++ b/flowsint-types/src/flowsint_types/document.py@@ -6,7 +6,7 @@ class Document(BaseModel):"""Represents a document with metadata, security, and content information."""title: str = Field(..., description="Document title", title="Title")- type: Optional[str] = Field(+ doc_type: Optional[str] = Field(None, description="Type of document (PDF, DOC, etc.)", title="Document Type")file_size: Optional[int] = Field(diff --git a/poetry.lock b/poetry.lockindex 49ff50b1..62e72597 100644--- a/poetry.lock+++ b/poetry.lock@@ -1152,25 +1152,26 @@ wmi = ["wmi (>=1.5.1)"][[package]]name = "docker"-version = "6.1.3"+version = "7.1.0"description = "A Python library for the Docker Engine API."optional = false-python-versions = ">=3.7"+python-versions = ">=3.8"groups = ["main"]files = [- {file = "docker-6.1.3-py3-none-any.whl", hash = "sha256:aecd2277b8bf8e506e484f6ab7aec39abe0038e29fa4a6d3ba86c3fe01844ed9"},- {file = "docker-6.1.3.tar.gz", hash = "sha256:aa6d17830045ba5ef0168d5eaa34d37beeb113948c413affe1d5991fc11f9a20"},+ {file = "docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0"},+ {file = "docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c"},][package.dependencies]-packaging = ">=14.0"pywin32 = {version = ">=304", markers = "sys_platform == \"win32\""}requests = ">=2.26.0"urllib3 = ">=1.26.0"-websocket-client = ">=0.32.0"[package.extras]+dev = ["coverage (==7.2.7)", "pytest (==7.4.2)", "pytest-cov (==4.1.0)", "pytest-timeout (==2.1.0)", "ruff (==0.1.8)"]+docs = ["myst-parser (==0.18.0)", "sphinx (==5.1.1)"]ssh = ["paramiko (>=2.4.3)"]+websockets = ["websocket-client (>=1.3.0)"][[package]]name = "ecdsa"@@ -1331,7 +1332,6 @@ develop = truealembic = "1.13.0"asyncpg = "^0.29"celery = "^5.3"-docker = "^6.1"flowsint-transforms = {path = "../flowsint-transforms", develop = true}neo4j = "^5.0"networkx = "^2.6.3"@@ -1363,7 +1363,6 @@ develop = true[package.dependencies]dnspython = "^2.4"-docker = "^6.1"flowsint-core = {path = "../flowsint-core", develop = true}flowsint-types = {path = "../flowsint-types", develop = true}hibpwned = "^1.3.9"@@ -5290,23 +5289,6 @@ files = [{file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"},]-[[package]]-name = "websocket-client"-version = "1.8.0"-description = "WebSocket client for Python with low level API options"-optional = false-python-versions = ">=3.8"-groups = ["main"]-files = [- {file = "websocket_client-1.8.0-py3-none-any.whl", hash = "sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526"},- {file = "websocket_client-1.8.0.tar.gz", hash = "sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da"},-]--[package.extras]-docs = ["Sphinx (>=6.0)", "myst-parser (>=2.0.0)", "sphinx-rtd-theme (>=1.1.0)"]-optional = ["python-socks", "wsaccel"]-test = ["websockets"]-[[package]]name = "werkzeug"version = "3.1.3"@@ -5580,4 +5562,4 @@ propcache = ">=0.2.1"[metadata]lock-version = "2.1"python-versions = ">=3.11,<3.13"-content-hash = "17855cf5181cb53298fb8d9ae7d27342d9ffda473d2205f3493f22f943e6ad67"+content-hash = "9a8a3f39280330fdf9eff55a050f6809081947173c8eac811da2ef13373c59f7"diff --git a/pyproject.toml b/pyproject.tomlindex ef42a57e..ff8fbf67 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -13,6 +13,7 @@ flowsint-transforms = { path = "./flowsint-transforms", develop = true }flowsint-api = { path = "./flowsint-api", develop = true }pydantic = {extras = ["email"], version = "^2.11.7"}python-multipart = "^0.0.20"+docker = "^7.1.0"[build-system]requires = ["poetry-core"]