Skip to content

A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.

feat: whoxy scanners

dextmorgna80347b43e3e
diff --git a/.env.example b/.env.example
index fe25c03b..c35d829b 100644
--- a/.env.example
+++ b/.env.example
@@ -9,4 +9,5 @@ AUTH_TRUST_HOST=true
NEO4J_URI_BOLT=bolt://neo4j:7687
NEO4J_URI_WEB=https://neo4j.flowsint.localhost
NEO4J_USER=neo4j
-NEO4J_PASSWORD=password
\ No newline at end of file
+NEO4J_PASSWORD=password
+WHOXY_API_KEY=
\ No newline at end of file
diff --git a/flowsint-api/app/api/routes/transforms.py b/flowsint-api/app/api/routes/transforms.py
index 56b55c29..2dfd0e51 100644
--- a/flowsint-api/app/api/routes/transforms.py
+++ b/flowsint-api/app/api/routes/transforms.py
@@ -43,7 +43,7 @@ def get_transforms(
category, exclude=["n8n_connector"]
)
else:
- transforms = TransformRegistry.list(exclude=["n8n_conector"])
+ transforms = TransformRegistry.list(exclude=["n8n_connector"])
return transforms
diff --git a/flowsint-app/src/renderer/public/icons/breach.svg b/flowsint-app/src/renderer/public/icons/breach.svg
index 0f95168d..d96cb677 100644
--- a/flowsint-app/src/renderer/public/icons/breach.svg
+++ b/flowsint-app/src/renderer/public/icons/breach.svg
@@ -1 +1 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="#ffffff" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-globe-lock-icon lucide-globe-lock"><path d="M15.686 15A14.5 14.5 0 0 1 12 22a14.5 14.5 0 0 1 0-20 10 10 0 1 0 9.542 13"/><path d="M2 12h8.5"/><path d="M20 6V4a2 2 0 1 0-4 0v2"/><rect width="8" height="5" x="14" y="6" rx="1"/></svg>
\ No newline at end of file
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="#ffffff" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-globe-lock-icon lucide-globe-lock"><path d="M15.686 15A14.5 14.5 0 0 1 12 22a14.5 14.5 0 0 1 0-20 10 10 0 1 0 9.542 13"/><path d="M2 12h8.5"/><path d="M20 6V4a2 2 0 1 0-4 0v2"/><rect width="8" height="5" x="14" y="6" rx="1"/></svg>
\ No newline at end of file
diff --git a/flowsint-app/src/renderer/public/icons/gravatar.svg b/flowsint-app/src/renderer/public/icons/gravatar.svg
new file mode 100644
index 00000000..a502cbaa
--- /dev/null
+++ b/flowsint-app/src/renderer/public/icons/gravatar.svg
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="#ffffff" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-image-icon lucide-image"><rect width="18" height="18" x="3" y="3" rx="2" ry="2"/><circle cx="9" cy="9" r="2"/><path d="m21 15-3.086-3.086a2 2 0 0 0-2.828 0L6 21"/></svg>
\ No newline at end of file
diff --git a/flowsint-app/src/renderer/src/components/analyses/analysis-editor.tsx b/flowsint-app/src/renderer/src/components/analyses/analysis-editor.tsx
index aba27dad..7efb5a79 100644
--- a/flowsint-app/src/renderer/src/components/analyses/analysis-editor.tsx
+++ b/flowsint-app/src/renderer/src/components/analyses/analysis-editor.tsx
@@ -177,6 +177,7 @@ export const AnalysisEditor = ({
// Handle title update
const handleTitleUpdate = (newTitle: string) => {
+ if (!analysis || newTitle === analysis.title) return
setTitleValue(newTitle)
updateTitleMutation.mutate(newTitle)
setIsEditingTitle(false)
diff --git a/flowsint-app/src/renderer/src/components/analyses/notes-panel.tsx b/flowsint-app/src/renderer/src/components/analyses/notes-panel.tsx
index 5da8008b..8b612e9c 100644
--- a/flowsint-app/src/renderer/src/components/analyses/notes-panel.tsx
+++ b/flowsint-app/src/renderer/src/components/analyses/notes-panel.tsx
@@ -4,10 +4,10 @@ import { analysisService } from "@/api/analysis-service"
import type { Analysis } from "@/types"
import { useAnalysisPanelStore } from "@/stores/analysis-panel-store"
import { AnalysisEditor } from "./analysis-editor"
-import { useEffect } from "react"
+import { useEffect, memo } from "react"
import AnalysisSkeleton from "./analysis-skeleton"
-export const AnalysisPanel = () => {
+export const AnalysisPanel = memo(() => {
const { investigationId } = useParams({ strict: false }) as { investigationId: string }
const currentAnalysisId = useAnalysisPanelStore(s => s.currentAnalysisId)
const setCurrentAnalysisId = useAnalysisPanelStore(s => s.setCurrentAnalysisId)
@@ -79,6 +79,6 @@ export const AnalysisPanel = () => {
/>
</div>
)
-}
+})
-export default AnalysisPanel
\ No newline at end of file
+export default memo(AnalysisPanel)
\ No newline at end of file
diff --git a/flowsint-app/src/renderer/src/components/chat/floating-chat.tsx b/flowsint-app/src/renderer/src/components/chat/floating-chat.tsx
index 9757e714..8cc6722d 100644
--- a/flowsint-app/src/renderer/src/components/chat/floating-chat.tsx
+++ b/flowsint-app/src/renderer/src/components/chat/floating-chat.tsx
@@ -16,6 +16,7 @@ import { ChatSkeleton } from './chat-skeleton'
import { ResizableChat } from './resizable-chat'
import ChatHistory from './chat-history'
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '../ui/tooltip'
+import { CopyButton } from '../copy'
function FloatingChat() {
const [editorValue, setEditorValue] = useState<any>("")
@@ -252,8 +253,8 @@ function FloatingChat() {
</ul>
</div> */}
<div className="text-xs space-y-1">
- <p className="font-medium text-left">Investigation Help</p>
- <ul className="space-y-1 text-left opacity-60">
+ <p className="font-medium text-left">Investigation Help</p>
+ <ul className="space-y-1 text-left opacity-60">
<li>• "Suggest next steps for this investigation"</li>
<li>• "What should I look for next?"</li>
<li>• "Help me organize this investigation"</li>
@@ -287,32 +288,46 @@ const ChatMessageComponent = ({ message }: { message: ChatMessage }) => {
if (message.is_bot)
return (
- <div className={cn("justify-start",
- "flex w-full",
- )}>
- <div className={cn("w-full",
- "p-3 rounded-xl max-w-full",
- "flex flex-col gap-2"
+ <MessageContainer copyContent={message.content}>
+ <div className={cn("justify-start",
+ "flex w-full",
)}>
- <MemoizedMarkdown id={message.id} content={message.content} />
+ <div className={cn("w-full",
+ "p-3 rounded-xl max-w-full",
+ "flex flex-col gap-2"
+ )}>
+ <MemoizedMarkdown id={message.id} content={message.content} />
+ </div>
</div>
- </div>
+ </MessageContainer>
)
return (
- <div className={cn("items-end",
- "flex w-full flex-col gap-2",
- )}>
- <div className={cn("bg-muted-foreground/5 max-w-[80%] border border-border",
- "p-1 rounded-lg",
- "flex flex-col items-end overflow-x-hidden"
+ <MessageContainer copyContent={message.content} >
+ <div className={cn("items-end",
+ "flex w-full flex-col gap-2",
)}>
- {/* {message?.context && message.context.length > 0 && <div className='flex items-center w-full overflow-x-auto justify-end mb-2'><ContextList context={message.context} /></div>} */}
- <span className='px-3'><MemoizedMarkdown id={message.id} content={message.content} /></span>
- </div>
- </div >)
+ <div className={cn("bg-muted-foreground/5 max-w-[80%] border border-border",
+ "p-1 rounded-lg",
+ "flex flex-col items-end overflow-x-hidden"
+ )}>
+ {/* {message?.context && message.context.length > 0 && <div className='flex items-center w-full overflow-x-auto justify-end mb-2'><ContextList context={message.context} /></div>} */}
+ <span className='px-3'><MemoizedMarkdown id={message.id} content={message.content} /></span>
+ </div>
+ </div >
+ </MessageContainer>)
}
+const MessageContainer = memo(({ children, copyContent }: { children: React.ReactNode, copyContent?: string }) => {
+ return (
+ <div className='relative group'>
+ {children}
+ <div className='-mt-1 group-hover:opacity-100 opacity-0 flex items-center justify-end z-1'>
+ <div className='border rounded-md bg-background '><CopyButton content={copyContent ?? ""} /></div>
+ </div>
+ </div>
+ )
+})
const MemoizedFloatingChat = memo(FloatingChat)
diff --git a/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx b/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx
index 53c88641..af819410 100644
--- a/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx
@@ -6,6 +6,7 @@ import { Skeleton } from '@/components/ui/skeleton';
import { Button } from '@/components/ui/button';
import { Input } from '@/components/ui/input';
import { Tabs, TabsList, TabsTrigger, TabsContent } from '@/components/ui/tabs';
+import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip';
import { FileCode2, Search, Info, Star, Zap } from 'lucide-react';
import { Transform, Flow } from '@/types';
import { GraphNode } from '@/stores/graph-store';
@@ -190,9 +191,7 @@ export default function ContextMenu({
</div>
<div className='flex items-center gap-1'>
<FavoriteButton isFavorite={false} />
- <Button className='w-5 h-5' variant='ghost' size={"icon"}>
- <Info className='w-4 h-4 opacity-50' strokeWidth={1.5} />
- </Button>
+ <InfoButton description={transform.description} />
</div>
</button>
))}
@@ -262,9 +261,7 @@ export default function ContextMenu({
</div>
<div className='flex items-center gap-1'>
<FavoriteButton isFavorite={false} />
- <Button className='w-5 h-5' variant='ghost' size={"icon"}>
- <Info className='w-4 h-4 opacity-50' strokeWidth={1.5} />
- </Button>
+ <InfoButton description={flow.description} />
</div>
</button>
))}
@@ -297,4 +294,23 @@ const FavoriteButton = memo(({ isFavorite }: { isFavorite: boolean }) => {
<Star fill={favorite ? 'yellow' : 'none'} className={cn('w-4 h-4 opacity-50', favorite && 'text-yellow-500 opacity-100')} strokeWidth={1.5} />
</Button>
)
-})
\ No newline at end of file
+})
+
+const InfoButton = ({ description }: { description?: string }) => {
+ if (!description) return null;
+
+ return (
+ <Tooltip>
+ <TooltipTrigger asChild>
+ <div>
+ <Button onClick={(e) => e.stopPropagation()} className='w-5 h-5' variant='ghost' size={"icon"}>
+ <Info className='w-4 h-4 opacity-50' strokeWidth={1.5} />
+ </Button>
+ </div>
+ </TooltipTrigger>
+ <TooltipContent>
+ <p className="whitespace-pre-wrap">{description}</p>
+ </TooltipContent>
+ </Tooltip >
+ );
+};
\ No newline at end of file
diff --git a/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx b/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx
index 65dae61a..583e9508 100644
--- a/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx
@@ -13,7 +13,7 @@ import NeighborsGraph from "./neighbors"
import Relationships from "./relationships"
import { ResizablePanelGroup, ResizablePanel, ResizableHandle } from "../../ui/resizable"
-export default function DetailsPanel({ node }: { node: GraphNode | null }) {
+const DetailsPanel = memo(({ node }: { node: GraphNode | null }) => {
const { id: sketchId } = useParams({ strict: false })
if (!node) {
return (
@@ -61,36 +61,38 @@ export default function DetailsPanel({ node }: { node: GraphNode | null }) {
<NodeActions node={node} />
</div>
</div>
- {node.data?.description && (
- <div className="px-4 py-3 border-b border-border">
- <div
- className="text-sm text-muted-foreground prose prose-sm max-w-none"
- dangerouslySetInnerHTML={{ __html: node.data.description }}
- />
- </div>
- )}
<div className="flex-1 min-h-0">
<ResizablePanelGroup direction="vertical" className="h-full">
- <ResizablePanel defaultSize={20} minSize={20}>
+ <ResizablePanel defaultSize={15} minSize={20}>
<div className="h-full overflow-auto">
- <KeyValueDisplay data={node.data} />
+ {node.data?.description && (
+ <div className="px-4 py-3 border-b border-border">
+ <div
+ className="text-sm text-muted-foreground prose prose-sm max-w-none"
+ dangerouslySetInnerHTML={{ __html: node.data.description }}
+ />
+ </div>
+ )}
+ <MemoizedKeyValueDisplay data={node.data} />
</div>
</ResizablePanel>
<ResizableHandle />
- <ResizablePanel defaultSize={35} minSize={25}>
+ <ResizablePanel defaultSize={40} minSize={25}>
<div className="h-full p-3">
<NeighborsGraph sketchId={sketchId as string} nodeId={node.id} />
</div>
</ResizablePanel>
<ResizableHandle />
- <ResizablePanel defaultSize={45} minSize={20}>
+ <ResizablePanel defaultSize={35} minSize={20}>
<Relationships sketchId={sketchId as string} nodeId={node.id} />
</ResizablePanel>
</ResizablePanelGroup>
</div>
</div>
)
-}
+})
+
+export default DetailsPanel
interface KeyValueDisplayProps {
data: Record<string, any>
@@ -141,16 +143,3 @@ function KeyValueDisplay({ data, className }: KeyValueDisplayProps) {
}
export const MemoizedKeyValueDisplay = memo(KeyValueDisplay)
-
-const ItemHero = ({ node }: { node: GraphNode }) => {
- const ItemIcon = useIcon(node.data.type, node.data.src)
- return (
- <div className="flex bg-card border-b items-center w-full justify-between p-4 gap-4">
- <div className="truncate">
- <p className="text-xl font-semibold truncate text-ellipsis">{node.data.label}</p>
- <p className="text-sm font-normal opacity-60 truncate text-ellipsis">{node.data.type}</p>
- </div>
- <ItemIcon size={50} />
- </div>
- )
-}
\ No newline at end of file
diff --git a/flowsint-app/src/renderer/src/components/graphs/details-panel/neighbors.tsx b/flowsint-app/src/renderer/src/components/graphs/details-panel/neighbors.tsx
index 5d2a6f40..e477b39a 100644
--- a/flowsint-app/src/renderer/src/components/graphs/details-panel/neighbors.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/details-panel/neighbors.tsx
@@ -2,8 +2,9 @@ import { sketchService } from "@/api/sketch-service";
import { useQuery } from "@tanstack/react-query";
import ForceGraphViewer from "../graph-viewer";
import Loader from "@/components/loader";
+import { memo } from "react";
-const NeighborsGraph = ({ sketchId, nodeId }: { sketchId: string, nodeId: string }) => {
+const NeighborsGraph = memo(({ sketchId, nodeId }: { sketchId: string, nodeId: string }) => {
const { data: neighborsData, isLoading } = useQuery({
queryKey: ['neighbors', sketchId, nodeId],
@@ -32,6 +33,6 @@ const NeighborsGraph = ({ sketchId, nodeId }: { sketchId: string, nodeId: string
</>}
</div>
)
-}
+})
export default NeighborsGraph
\ No newline at end of file
diff --git a/flowsint-app/src/renderer/src/components/graphs/details-panel/relationships.tsx b/flowsint-app/src/renderer/src/components/graphs/details-panel/relationships.tsx
index b6524ff7..9f31dafa 100644
--- a/flowsint-app/src/renderer/src/components/graphs/details-panel/relationships.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/details-panel/relationships.tsx
@@ -25,7 +25,7 @@ const getInlineRelationships = (nodes: GraphNode[], edges: GraphEdge[]): Relatio
return relationships
}
-const Relationships = ({ sketchId, nodeId }: { sketchId: string, nodeId: string }) => {
+const Relationships = memo(({ sketchId, nodeId }: { sketchId: string, nodeId: string }) => {
const { data: neighborsData, isLoading } = useQuery({
queryKey: ['neighbors', sketchId, nodeId],
queryFn: () => sketchService.getNodeNeighbors(sketchId, nodeId),
@@ -80,7 +80,7 @@ const Relationships = ({ sketchId, nodeId }: { sketchId: string, nodeId: string
</div>
</div>
)
-}
+})
export default Relationships
diff --git a/flowsint-app/src/renderer/src/components/graphs/force-controls.tsx b/flowsint-app/src/renderer/src/components/graphs/force-controls.tsx
index b3feb81f..9e645637 100644
--- a/flowsint-app/src/renderer/src/components/graphs/force-controls.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/force-controls.tsx
@@ -20,18 +20,18 @@ const ForceControls = ({ children }: { children: React.ReactNode }) => {
const quickControls = [
{
- key: 'chargeStrength',
- label: 'Repulsion',
+ key: 'd3AlphaDecay',
+ label: 'Convergence',
icon: <Zap className="w-3 h-3" />,
},
{
- key: 'linkStrength',
- label: 'Link Force',
+ key: 'd3VelocityDecay',
+ label: 'Friction',
icon: <Move className="w-3 h-3" />,
},
{
- key: 'd3VelocityDecay',
- label: 'Friction',
+ key: 'cooldownTicks',
+ label: 'Simulation',
icon: <Target className="w-3 h-3" />,
}
];
diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx
index e6c0f039..f56e1785 100644
--- a/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx
@@ -15,8 +15,9 @@ const GraphMain = () => {
const [menu, setMenu] = React.useState<any>(null)
- const handleNodeClick = useCallback((node: any) => {
- toggleNodeSelection(node, false)
+ const handleNodeClick = useCallback((node: any, event: MouseEvent) => {
+ const isMultiSelect = event.ctrlKey || event.shiftKey;
+ toggleNodeSelection(node, isMultiSelect)
}, [toggleNodeSelection])
const handleBackgroundClick = useCallback(() => {
diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-settings.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-settings.tsx
index d1aaab7c..6112510e 100644
--- a/flowsint-app/src/renderer/src/components/graphs/graph-settings.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/graph-settings.tsx
@@ -22,17 +22,9 @@ const GraphSettings = () => {
// Categorize settings
const categories = {
- 'Force Physics': [
- 'chargeStrength',
- 'linkStrength',
- 'centerStrength',
- 'collisionRadius',
- 'collisionStrength'
- ],
'Simulation Control': [
'd3AlphaDecay',
'd3AlphaMin',
- 'd3AlphaTarget',
'd3VelocityDecay',
'cooldownTicks',
'cooldownTime',
diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx
index 710fc746..9b2f5e36 100644
--- a/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx
@@ -8,7 +8,7 @@ import ForceGraph2D from 'react-force-graph-2d';
import { Button } from '../ui/button';
import { useTheme } from '@/components/theme-provider'
import { GRAPH_COLORS } from '../flows/scanner-data';
-import { Share2 } from 'lucide-react';
+import { Share2, Type } from 'lucide-react';
function truncateText(text: string, limit: number = 16) {
if (text.length <= limit)
@@ -23,7 +23,7 @@ interface GraphViewerProps {
height?: number;
nodeColors?: Record<string, string>;
nodeSizes?: Record<string, number>;
- onNodeClick?: (node: GraphNode) => void;
+ onNodeClick?: (node: GraphNode, event: MouseEvent) => void;
onNodeRightClick?: (node: GraphNode, event: MouseEvent) => void;
onBackgroundClick?: () => void;
showLabels?: boolean;
@@ -204,7 +204,7 @@ const GraphViewer: React.FC<GraphViewerProps> = ({
nodes.length > CONSTANTS.NODE_COUNT_THRESHOLD || currentZoom < 1.5
, [nodes.length, currentZoom]);
- // Optimized graph data transformation with proper memoization dependencies
+ // Memoized graph data transformation with proper memoization dependencies
const graphData = useMemo(() => {
// Transform nodes
const transformedNodes = nodes.map(node => {
@@ -291,6 +291,8 @@ const GraphViewer: React.FC<GraphViewerProps> = ({
};
}, [nodes, edges, nodeColors, getSize, view]);
+
+
// New function to determine which labels should be visible based on zoom and weight
const getVisibleLabels = useMemo(() => {
if (!showLabels) return new Set<string>();
@@ -317,32 +319,9 @@ const GraphViewer: React.FC<GraphViewerProps> = ({
return new Set(visibleNodes.map(node => node.id));
}, [graphData.nodes, currentZoom, showLabels]);
- // Get current label layer info for display
- const currentLabelLayer = useMemo(() => {
- if (!showLabels) return null;
- return CONSTANTS.LABEL_LAYERS.find(layer => currentZoom >= layer.minZoom);
- }, [currentZoom, showLabels]);
-
- // Get label count info for current layer
- const labelInfo = useMemo(() => {
- if (!currentLabelLayer || !showLabels) return null;
-
- const totalNodes = graphData.nodes.length;
- const visibleCount = getVisibleLabels.size;
- const weightThreshold = currentLabelLayer.minWeight;
-
- return {
- visibleCount,
- totalNodes,
- weightThreshold,
- zoomLevel: currentZoom,
- layer: currentLabelLayer
- };
- }, [currentLabelLayer, showLabels, getVisibleLabels.size, graphData.nodes.length, currentZoom]);
-
// Event handlers with proper memoization
- const handleNodeClick = useCallback((node: any) => {
- onNodeClick?.(node);
+ const handleNodeClick = useCallback((node: any, event: MouseEvent) => {
+ onNodeClick?.(node, event);
}, [onNodeClick]);
const handleNodeRightClick = useCallback((node: any, event: MouseEvent) => {
@@ -408,6 +387,7 @@ const GraphViewer: React.FC<GraphViewerProps> = ({
data: {
label: string,
connections: string
+ type: string
} | null;
visible: boolean;
}>({
@@ -446,7 +426,7 @@ const GraphViewer: React.FC<GraphViewerProps> = ({
// Adjust Y position if tooltip would go off-screen
if (y < tooltipHeight + 10) {
- y = screenCoords.y + 30; // Position below the node instead
+ y = screenCoords.y + 100; // Position below the node instead
}
setTooltip({
@@ -454,6 +434,7 @@ const GraphViewer: React.FC<GraphViewerProps> = ({
y,
data: {
label,
+ type: node.data.type,
connections: weight
},
visible: true
@@ -469,12 +450,6 @@ const GraphViewer: React.FC<GraphViewerProps> = ({
handleNodeHover(node);
}, [handleNodeHover]);
- // Remove mouse tracking since we don't need it anymore
- // const [mousePosition, setMousePosition] = useState({ x: 0, y: 0 });
- // const handleMouseMove = useCallback((event: React.MouseEvent) => {
- // setMousePosition({ x: event.clientX, y: event.clientY });
- // }, []);
-
const handleLinkHover = useCallback((link: any) => {
// Throttle hover updates to max 60fps
const now = Date.now();
@@ -835,9 +810,10 @@ const GraphViewer: React.FC<GraphViewerProps> = ({
>
<div className="whitespace-pre-line truncate flex flex-col gap-1">
<span className='text-md font-semibold'>{tooltip.data?.label}</span>
- <span className='flex items-center gap-1'><Share2 className='h-3 w-3 opacity-60' /> connections: <span className='font-semibold'>{tooltip.data?.connections}</span></span>
+ <span className='flex items-center gap-1'><span className='flex items-center gap-1 opacity-60'><Type className='h-3 w-3' /> type:</span> <span className='font-medium'>{tooltip.data?.type}</span></span>
+ <span className='flex items-center gap-1'><span className='flex items-center gap-1 opacity-60'><Share2 className='h-3 w-3' /> connections:</span> <span className='font-medium'>{tooltip.data?.connections}</span></span>
</div>
- </div>
+ </div >
)}
<ForceGraph2D
ref={graphRef}
@@ -870,7 +846,7 @@ const GraphViewer: React.FC<GraphViewerProps> = ({
onNodeHover={handleNodeHoverWithTooltip}
onLinkHover={handleLinkHover}
/>
- </div>
+ </div >
);
};
diff --git a/flowsint-app/src/renderer/src/components/graphs/graph.tsx b/flowsint-app/src/renderer/src/components/graphs/graph.tsx
index 35bb46e4..0f3e08cc 100644
--- a/flowsint-app/src/renderer/src/components/graphs/graph.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/graph.tsx
@@ -86,7 +86,7 @@ const GRAPH_CONFIG = {
const GraphContent = memo(() => {
const nodes = useGraphStore(s => s.nodes)
const edges = useGraphStore(s => s.edges)
- const currentNode = useGraphStore(s => s.currentNode)
+ // const currentNode = useGraphStore(s => s.currentNode)
const { isCosmographReady, loadingStage, handleSimulationStart, handleSimulationEnd } = useCosmographLoader(nodes, edges)
const clearSelectedNodes = useGraphStore(s => s.clearSelectedNodes)
const toggleNodeSelection = useGraphStore(s => s.toggleNodeSelection)
diff --git a/flowsint-app/src/renderer/src/components/graphs/launch-transform.tsx b/flowsint-app/src/renderer/src/components/graphs/launch-transform.tsx
index 558efc7a..276abec0 100644
--- a/flowsint-app/src/renderer/src/components/graphs/launch-transform.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/launch-transform.tsx
@@ -1,5 +1,5 @@
-import { useCallback, useState } from "react"
+import { useCallback, useState, memo } from "react"
import { Button } from "@/components/ui/button"
import {
Sheet,
@@ -26,7 +26,7 @@ import { Skeleton } from "@/components/ui/skeleton"
import { Search, FileCode2, Zap } from "lucide-react"
import { Transform, Flow } from "@/types"
-const LaunchTransformOrFlowPanel = ({ values, type, children }: { values: string[], type: string, children?: React.ReactNode }) => {
+const LaunchTransformOrFlowPanel = memo(({ values, type, children }: { values: string[], type: string, children?: React.ReactNode }) => {
const { launchTransform } = useLaunchTransform()
const { launchFlow } = useLaunchFlow()
const { id: sketch_id } = useParams({ strict: false })
@@ -300,6 +300,6 @@ const LaunchTransformOrFlowPanel = ({ values, type, children }: { values: string
</Sheet>
</div>
)
-}
+})
export default LaunchTransformOrFlowPanel
diff --git a/flowsint-app/src/renderer/src/components/graphs/node-actions.tsx b/flowsint-app/src/renderer/src/components/graphs/node-actions.tsx
index 734ddfb4..a17fb3e6 100644
--- a/flowsint-app/src/renderer/src/components/graphs/node-actions.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/node-actions.tsx
@@ -15,7 +15,7 @@ import {
import { useLayoutStore } from '@/stores/layout-store';
-const NodeActions = ({ node, setMenu }: { node: GraphNode, setMenu?: (menu: any | null) => void }) => {
+const NodeActions = memo(({ node, setMenu }: { node: GraphNode, setMenu?: (menu: any | null) => void }) => {
const { id: sketchId } = useParams({ strict: false })
const { confirm } = useConfirm();
const setOpenMainDialog = useGraphStore(state => state.setOpenMainDialog)
@@ -140,6 +140,6 @@ const NodeActions = ({ node, setMenu }: { node: GraphNode, setMenu?: (menu: any
</TooltipProvider>
</div>
)
-}
+})
-export default memo(NodeActions)
\ No newline at end of file
+export default NodeActions
\ No newline at end of file
diff --git a/flowsint-app/src/renderer/src/components/graphs/selected-items-panel.tsx b/flowsint-app/src/renderer/src/components/graphs/selected-items-panel.tsx
index 45991267..a9b20442 100644
--- a/flowsint-app/src/renderer/src/components/graphs/selected-items-panel.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/selected-items-panel.tsx
@@ -39,7 +39,7 @@ export default memo(SelectedItemsPanel)
const SelectedNodeItem = memo(({ node, color }: { node: GraphNode, color: string }) => {
- return (<Badge variant="secondary" className="flex w-full items-center justify-between text-left border border-border gap-1.5 text-xs p-2 py-1.5">
+ return (<Badge variant="outline" className="flex w-full items-center justify-between text-left border border-border gap-1.5 text-xs p-2 py-1.5">
<span className='flex items-center gap-2 truncate text-ellipsis'>
<span style={{ background: color }} className="w-2 h-2 rounded-full" />
<span className='truncate text-ellipsis'>{node.data?.label || 'Unknown'}</span>
@@ -61,7 +61,7 @@ export const SelectedList = () => {
<SelectedNodeItem key={index} node={item} color={color} />
)
})}
- {remainingCount > 0 && <Badge variant="secondary" className="flex w-full items-center justify-between text-left border border-border gap-1.5 text-xs p-2 py-1.5">
+ {remainingCount > 0 && <Badge variant="outline" className="flex w-full items-center justify-between text-left border border-border gap-1.5 text-xs p-2 py-1.5">
<span className='flex items-center gap-2 truncate text-ellipsis'>
<span className='truncate text-ellipsis'>{remainingCount}+ other</span>
</span>
diff --git a/flowsint-app/src/renderer/src/components/layout/log-panel.tsx b/flowsint-app/src/renderer/src/components/layout/log-panel.tsx
index afecd56f..63685263 100644
--- a/flowsint-app/src/renderer/src/components/layout/log-panel.tsx
+++ b/flowsint-app/src/renderer/src/components/layout/log-panel.tsx
@@ -16,7 +16,7 @@ import {
FileText,
Trash2,
} from "lucide-react"
-import { useEffect, useRef } from "react"
+import { useEffect, useRef, memo } from "react"
import { CopyButton } from "../copy"
import { EventLevel } from "@/types"
import { cn } from "@/lib/utils"
@@ -97,7 +97,7 @@ const defaultConfig = {
emoji: "📝",
}
-export function LogPanel() {
+export const LogPanel = memo(() => {
const { id: sketch_id } = useParams({ strict: false })
const { confirm } = useConfirm()
const bottomRef = useRef<HTMLDivElement | null>(null)
@@ -191,4 +191,4 @@ export function LogPanel() {
</div>
</div>
)
-}
+})
diff --git a/flowsint-app/src/renderer/src/components/layout/root.layout.tsx b/flowsint-app/src/renderer/src/components/layout/root.layout.tsx
index 5f873d7e..8dfa9ca8 100644
--- a/flowsint-app/src/renderer/src/components/layout/root.layout.tsx
+++ b/flowsint-app/src/renderer/src/components/layout/root.layout.tsx
@@ -1,4 +1,4 @@
-import { useEffect, type ReactNode } from "react"
+import { type ReactNode, useMemo } from "react"
import { Sidebar } from "./sidebar"
import { TopNavbar } from "./top-navbar"
import { StatusBar } from "./status-bar"
@@ -7,7 +7,7 @@ import { ConfirmContextProvider } from "@/components/use-confirm-dialog"
import { ResizableHandle, ResizablePanel, ResizablePanelGroup } from "../ui/resizable"
import { LogPanel } from "./log-panel"
import { useLayoutStore } from "@/stores/layout-store"
-// import NotesPanel from "../analyses/notes-panel"
+import NotesPanel from "../analyses/notes-panel"
import { useKeyboardShortcut } from "@/hooks/use-keyboard-shortcut"
import { useParams } from "@tanstack/react-router"
import DetailsPanel from "../graphs/details-panel/details-panel"
@@ -21,24 +21,15 @@ export default function RootLayout({ children }: LayoutProps) {
const isOpenConsole = useLayoutStore(s => s.isOpenConsole)
const toggleConsole = useLayoutStore(s => s.toggleConsole)
const isOpenPanel = useLayoutStore(s => s.isOpenPanel)
- // const isOpenAnalysis = useLayoutStore(s => s.isOpenAnalysis)
+ const isOpenAnalysis = useLayoutStore(s => s.isOpenAnalysis)
const isOpenDetails = useLayoutStore(s => s.isOpenDetails)
const toggleDetails = useLayoutStore(s => s.toggleDetails)
const togglePanel = useLayoutStore(s => s.togglePanel)
const closePanel = useLayoutStore(s => s.closePanel)
const openPanel = useLayoutStore(s => s.openPanel)
const closeDetails = useLayoutStore(s => s.closeDetails)
- const openDetails = useLayoutStore(s => s.openDetails)
const currentNode = useGraphStore((s) => s.currentNode)
- const { id } = useParams({ strict: false })
-
- // useEffect(() => {
- // if (!currentNode) {
- // return
- // }
- // openDetails()
- // }, [currentNode, openDetails])
-
+ const { id, type } = useParams({ strict: false })
// Set up keyboard shortcut for chat panel
useKeyboardShortcut({
@@ -57,105 +48,142 @@ export default function RootLayout({ children }: LayoutProps) {
callback: toggleConsole
})
- return (
- <ConfirmContextProvider>
- <div className="flex flex-col h-screen w-screen overflow-hidden">
- {/* Top navbar */}
- <TopNavbar />
- {/* Main layout */}
- <div className="flex grow">
- <Sidebar />
- <ResizablePanelGroup autoSaveId="conditional" direction="vertical" className="flex-1 min-h-0">
- {/* Main content area with optional console */}
- <ResizablePanel order={1} id="main" defaultSize={isOpenConsole ? 70 : 100} minSize={30}>
- <ResizablePanelGroup
- autoSaveId="conditional2"
- direction="vertical"
- className="h-full"
- >
- {/* Main content */}
- <ResizablePanel id="content" order={1} defaultSize={100} minSize={30}>
- <ResizablePanelGroup
- direction="horizontal"
- autoSaveId="conditional3"
- className="h-full"
- >
+ // Memoize the details panel section to prevent unnecessary re-renders
+ const detailsPanelSection = useMemo(() => {
+ if (!id || type != "graph") return null;
+
+ // Priority 1: NotesPanel if isOpenAnalysis is true
+ if (isOpenAnalysis) {
+ return (
+ <>
+ <ResizableHandle withHandle />
+ <ResizablePanel
+ id="details-panel"
+ order={4}
+ defaultSize={20}
+ minSize={16}
+ maxSize={40}
+ onCollapse={closeDetails}
+ onExpand={() => { }} // No auto-open here
+ collapsible={true}
+ collapsedSize={2}
+ >
+ <NotesPanel />
+ </ResizablePanel>
+ </>
+ );
+ }
+
+ // Priority 2: DetailsPanel when currentNode exists (auto-opens via useEffect)
+ if (currentNode) {
+ return (
+ <>
+ <ResizableHandle withHandle />
+ <ResizablePanel
+ id="details-panel"
+ order={4}
+ defaultSize={20}
+ minSize={16}
+ maxSize={40}
+ onCollapse={closeDetails}
+ onExpand={() => { }} // No auto-open here
+ collapsible={true}
+ collapsedSize={2}
+ >
+ <DetailsPanel node={currentNode} />
+ </ResizablePanel>
+ </>
+ );
+ }
- {isOpenPanel && (
- <>
- <ResizablePanel
- id="sidebar"
- className="min-w-64"
- order={2}
- defaultSize={20}
- minSize={16}
- maxSize={30}
- onCollapse={closePanel}
- onExpand={openPanel}
- collapsible={true}
- collapsedSize={0}
- >
- {/* <div className="h-10 bg-card flex items-center justify-end px-2 text-sm w-full font-semibold border-b">
- <PathBreadcrumb />
- </div> */}
- <div className="h-full flex flex-col overflow-hidden bg-card">
- <SecondaryNavigation />
- </div>
- </ResizablePanel>
- <ResizableHandle />
- </>
- )}
- <ResizablePanel className="h-full w-full" id="children" order={3}>
- {children}
- </ResizablePanel>
- {isOpenDetails && id && currentNode && (
- <>
- <ResizableHandle withHandle />
- <ResizablePanel
- id="details-panel"
- order={4}
- defaultSize={20}
- minSize={16}
- maxSize={40}
- onCollapse={closeDetails}
- onExpand={openDetails}
- collapsible={true}
- collapsedSize={2}
- >
- {<DetailsPanel node={currentNode} />}
- </ResizablePanel>
- </>
- )}
- </ResizablePanelGroup>
- </ResizablePanel>
+ // No panel should be shown
+ return null;
+ }, [id, currentNode?.id, currentNode?.data, isOpenAnalysis, closeDetails]);
- {/* Status bar */}
- <div className="h-8 shrink-0">
- <StatusBar />
- </div>
+ // Memoize the entire layout content to prevent unnecessary re-renders
+ const layoutContent = useMemo(() => (
+ <div className="flex flex-col h-screen w-screen overflow-hidden">
+ {/* Top navbar */}
+ <TopNavbar />
+ {/* Main layout */}
+ <div className="flex grow">
+ <Sidebar />
+ <ResizablePanelGroup autoSaveId="conditional" direction="vertical" className="flex-1 min-h-0">
+ {/* Main content area with optional console */}
+ <ResizablePanel order={1} id="main" defaultSize={isOpenConsole ? 70 : 100} minSize={30}>
+ <ResizablePanelGroup
+ autoSaveId="conditional2"
+ direction="vertical"
+ className="h-full"
+ >
+ {/* Main content */}
+ <ResizablePanel id="content" order={1} defaultSize={100} minSize={30}>
+ <ResizablePanelGroup
+ direction="horizontal"
+ autoSaveId="conditional3"
+ className="h-full"
+ >
- {/* Console panel - only shown when isOpen is true */}
- {isOpenConsole && id && (
- <>
- <ResizableHandle />
- <ResizablePanel
- id="console"
- order={5}
- defaultSize={30}
- minSize={10}
- maxSize={50}
- >
- <div className="h-full overflow-hidden">
- <LogPanel />
- </div>
- </ResizablePanel>
- </>
- )}
- </ResizablePanelGroup>
- </ResizablePanel>
- </ResizablePanelGroup>
- </div>
+ {isOpenPanel && (
+ <>
+ <ResizablePanel
+ id="sidebar"
+ className="min-w-64"
+ order={2}
+ defaultSize={20}
+ minSize={16}
+ maxSize={30}
+ onCollapse={closePanel}
+ onExpand={openPanel}
+ collapsible={true}
+ collapsedSize={0}
+ >
+ <div className="h-full flex flex-col overflow-hidden bg-card">
+ <SecondaryNavigation />
+ </div>
+ </ResizablePanel>
+ <ResizableHandle />
+ </>
+ )}
+ <ResizablePanel className="h-full w-full" id="children" order={3}>
+ {children}
+ </ResizablePanel>
+ {detailsPanelSection}
+ </ResizablePanelGroup>
+ </ResizablePanel>
+
+ {/* Status bar */}
+ <div className="h-8 shrink-0">
+ <StatusBar />
+ </div>
+
+ {/* Console panel - only shown when isOpen is true */}
+ {isOpenConsole && id && (
+ <>
+ <ResizableHandle />
+ <ResizablePanel
+ id="console"
+ order={5}
+ defaultSize={30}
+ minSize={10}
+ maxSize={50}
+ >
+ <div className="h-full overflow-hidden">
+ <LogPanel />
+ </div>
+ </ResizablePanel>
+ </>
+ )}
+ </ResizablePanelGroup>
+ </ResizablePanel>
+ </ResizablePanelGroup>
</div>
+ </div>
+ ), [isOpenConsole, isOpenPanel, isOpenAnalysis, isOpenDetails, id, children, detailsPanelSection, closePanel, openPanel, closeDetails]);
+
+ return (
+ <ConfirmContextProvider>
+ {layoutContent}
</ConfirmContextProvider>
)
}
diff --git a/flowsint-app/src/renderer/src/components/layout/secondary-navigation.tsx b/flowsint-app/src/renderer/src/components/layout/secondary-navigation.tsx
index cd20e9ae..76789e63 100644
--- a/flowsint-app/src/renderer/src/components/layout/secondary-navigation.tsx
+++ b/flowsint-app/src/renderer/src/components/layout/secondary-navigation.tsx
@@ -4,8 +4,9 @@ import GraphNavigation from "../graphs/graph-navigation"
import TransformNavigation from "../flows/flow-navigation"
import SketchList from "../investigations/sketch-list"
import AnalysesList from "../analyses/analyses-list"
+import { memo } from "react"
-const SecondaryNavigation = () => {
+const SecondaryNavigation = memo(() => {
const { id, investigationId, type } = useParams({ strict: false })
const { pathname } = useLocation()
@@ -50,6 +51,6 @@ const SecondaryNavigation = () => {
return (
<div>secondary-navigation</div>
)
-}
+})
export default SecondaryNavigation
\ No newline at end of file
diff --git a/flowsint-app/src/renderer/src/components/layout/sidebar.tsx b/flowsint-app/src/renderer/src/components/layout/sidebar.tsx
index 337b7938..d5c2c02d 100644
--- a/flowsint-app/src/renderer/src/components/layout/sidebar.tsx
+++ b/flowsint-app/src/renderer/src/components/layout/sidebar.tsx
@@ -15,6 +15,7 @@ import {
TooltipProvider,
TooltipTrigger,
} from "../ui/tooltip"
+import { memo } from "react"
interface NavItem {
icon: LucideIcon
@@ -22,7 +23,7 @@ interface NavItem {
href?: string
}
-export function Sidebar() {
+export const Sidebar = memo(() => {
const togglePanel = useLayoutStore(s => s.togglePanel)
const navItems: NavItem[] = [
@@ -86,4 +87,4 @@ export function Sidebar() {
</div>
</div>
)
-}
+})
diff --git a/flowsint-app/src/renderer/src/components/layout/status-bar.tsx b/flowsint-app/src/renderer/src/components/layout/status-bar.tsx
index 961b210c..96467385 100644
--- a/flowsint-app/src/renderer/src/components/layout/status-bar.tsx
+++ b/flowsint-app/src/renderer/src/components/layout/status-bar.tsx
@@ -5,8 +5,9 @@ import { useLayoutStore } from "@/stores/layout-store"
import Legend from "../graphs/legend"
import { Link, useParams } from "@tanstack/react-router"
import InfoDialog from "./info"
+import { memo } from "react"
-export function StatusBar() {
+export const StatusBar = memo(() => {
const { id: sketch_id } = useParams({ strict: false })
const isOpenConsole = useLayoutStore(s => s.isOpenConsole)
const toggleConsole = useLayoutStore(s => s.toggleConsole)
@@ -43,4 +44,4 @@ export function StatusBar() {
</div>
</div >
)
-}
+})
diff --git a/flowsint-app/src/renderer/src/components/layout/top-navbar.tsx b/flowsint-app/src/renderer/src/components/layout/top-navbar.tsx
index 6d7f2129..665cda40 100644
--- a/flowsint-app/src/renderer/src/components/layout/top-navbar.tsx
+++ b/flowsint-app/src/renderer/src/components/layout/top-navbar.tsx
@@ -1,15 +1,36 @@
-import { Button } from "@/components/ui/button"
-import { Settings } from "lucide-react"
import { Command } from "../command"
import { NavUser } from "../nav-user"
import { Link, useParams } from "@tanstack/react-router"
import InvestigationSelector from "./investigation-selector"
import SketchSelector from "./sketch-selector"
-import { useEffect, useState } from "react"
+import { useEffect, useState, memo, useCallback } from "react"
+import { Switch } from "../ui/switch"
+import { Label } from "../ui/label"
+import { useLayoutStore } from "@/stores/layout-store"
+import { Button } from "@/components/ui/button"
+import {
+ DropdownMenu,
+ DropdownMenuContent,
+ DropdownMenuGroup,
+ DropdownMenuItem,
+ DropdownMenuLabel,
+ DropdownMenuPortal,
+ DropdownMenuSeparator,
+ DropdownMenuShortcut,
+ DropdownMenuSub,
+ DropdownMenuSubContent,
+ DropdownMenuSubTrigger,
+ DropdownMenuTrigger,
+} from "@/components/ui/dropdown-menu"
+import { Ellipsis } from "lucide-react"
-export function TopNavbar() {
- const { investigationId, id } = useParams({ strict: false })
+export const TopNavbar = memo(() => {
+ const { investigationId, id, type } = useParams({ strict: false })
const [isFullscreen, setIsFullscreen] = useState(false)
+ const toggleAnalysis = useLayoutStore(s => s.toggleAnalysis)
+ const isOpenAnalysis = useLayoutStore(s => s.isOpenAnalysis)
+
+ const handleToggleAnalysis = useCallback(() => toggleAnalysis(), [toggleAnalysis])
useEffect(() => {
const checkWindowState = async () => {
@@ -18,7 +39,7 @@ export function TopNavbar() {
setIsFullscreen(windowState.isFullscreen)
} catch (error) {
console.error('Failed to get window state:', error)
- setIsFullscreen(false)
+ setIsFullscreen(true)
}
}
checkWindowState()
@@ -49,12 +70,80 @@ export function TopNavbar() {
<div className="grow flex items-center justify-center">
<Command />
</div>
- <div className="flex items-center gap-2 -webkit-app-region-no-drag">
- <Button variant="ghost" size="icon">
- <Settings className="h-5 w-5 opacity-60" />
- </Button>
+ <div className="flex items-center gap-4 -webkit-app-region-no-drag">
+ <div className="flex items-center space-x-2">
+ {type === "graph" &&
+ <>
+ <Switch checked={isOpenAnalysis} onCheckedChange={handleToggleAnalysis} id="notes" />
+ <Label htmlFor="notes">Toggle notes</Label>
+ </>
+ }
+ </div>
+ <InvestigationMenu />
<NavUser />
</div>
</header>
)
+})
+
+
+export function InvestigationMenu() {
+ return (
+ <DropdownMenu>
+ <DropdownMenuTrigger asChild>
+ <div>
+ <Button size="icon" variant="ghost"><Ellipsis /></Button>
+ </div>
+ </DropdownMenuTrigger>
+ <DropdownMenuContent className="w-56" align="start">
+ <DropdownMenuLabel>My Account</DropdownMenuLabel>
+ <DropdownMenuGroup>
+ <DropdownMenuItem>
+ Profile
+ <DropdownMenuShortcut>⇧⌘P</DropdownMenuShortcut>
+ </DropdownMenuItem>
+ <DropdownMenuItem>
+ Billing
+ <DropdownMenuShortcut>⌘B</DropdownMenuShortcut>
+ </DropdownMenuItem>
+ <DropdownMenuItem>
+ Settings
+ <DropdownMenuShortcut>⌘S</DropdownMenuShortcut>
+ </DropdownMenuItem>
+ <DropdownMenuItem>
+ Keyboard shortcuts
+ <DropdownMenuShortcut>⌘K</DropdownMenuShortcut>
+ </DropdownMenuItem>
+ </DropdownMenuGroup>
+ <DropdownMenuSeparator />
+ <DropdownMenuGroup>
+ <DropdownMenuItem>Team</DropdownMenuItem>
+ <DropdownMenuSub>
+ <DropdownMenuSubTrigger>Invite users</DropdownMenuSubTrigger>
+ <DropdownMenuPortal>
+ <DropdownMenuSubContent>
+ <DropdownMenuItem>Email</DropdownMenuItem>
+ <DropdownMenuItem>Message</DropdownMenuItem>
+ <DropdownMenuSeparator />
+ <DropdownMenuItem>More...</DropdownMenuItem>
+ </DropdownMenuSubContent>
+ </DropdownMenuPortal>
+ </DropdownMenuSub>
+ <DropdownMenuItem>
+ New Team
+ <DropdownMenuShortcut>⌘+T</DropdownMenuShortcut>
+ </DropdownMenuItem>
+ </DropdownMenuGroup>
+ <DropdownMenuSeparator />
+ <DropdownMenuItem>GitHub</DropdownMenuItem>
+ <DropdownMenuItem>Support</DropdownMenuItem>
+ <DropdownMenuItem disabled>API</DropdownMenuItem>
+ <DropdownMenuSeparator />
+ <DropdownMenuItem>
+ Log out
+ <DropdownMenuShortcut>⇧⌘Q</DropdownMenuShortcut>
+ </DropdownMenuItem>
+ </DropdownMenuContent>
+ </DropdownMenu>
+ )
}
diff --git a/flowsint-app/src/renderer/src/components/table/nodes-view.tsx b/flowsint-app/src/renderer/src/components/table/nodes-view.tsx
index 73d93f84..a7527b08 100644
--- a/flowsint-app/src/renderer/src/components/table/nodes-view.tsx
+++ b/flowsint-app/src/renderer/src/components/table/nodes-view.tsx
@@ -74,7 +74,7 @@ export default function NodesTable({ nodes }: NodesTableProps) {
const onNodeClick = useCallback((node: GraphNode) => {
setCurrentNode(node)
// setOpenNodeEditorModal(true)
- }, [setCurrentNode,
+ }, [setCurrentNode,
// setOpenNodeEditorModal
])
diff --git a/flowsint-app/src/renderer/src/components/ui/dropdown-menu.tsx b/flowsint-app/src/renderer/src/components/ui/dropdown-menu.tsx
index 4ce8e00d..fe3d6fbc 100644
--- a/flowsint-app/src/renderer/src/components/ui/dropdown-menu.tsx
+++ b/flowsint-app/src/renderer/src/components/ui/dropdown-menu.tsx
@@ -41,7 +41,7 @@ function DropdownMenuContent({
data-slot="dropdown-menu-content"
sideOffset={sideOffset}
className={cn(
- "bg-popover text-popover-foreground data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 max-h-(--radix-dropdown-menu-content-available-height) min-w-[8rem] origin-(--radix-dropdown-menu-content-transform-origin) overflow-x-hidden overflow-y-auto rounded-md border p-1 shadow-md",
+ "bg-popover text-popover-foreground data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 max-h-(--radix-dropdown-menu-content-available-height) min-w-[8rem] origin-(--radix-dropdown-menu-content-transform-origin) overflow-x-hidden overflow-y-auto rounded-md border p-1 shadow-md mx-3",
className
)}
{...props}
diff --git a/flowsint-app/src/renderer/src/stores/graph-settings-store.ts b/flowsint-app/src/renderer/src/stores/graph-settings-store.ts
index ef9cda4e..4dcd00da 100644
--- a/flowsint-app/src/renderer/src/stores/graph-settings-store.ts
+++ b/flowsint-app/src/renderer/src/stores/graph-settings-store.ts
@@ -87,49 +87,7 @@ const DEFAULT_SETTINGS = {
step: 1,
description: "Number of simulation cycles to run in background before starting to render (improves initial layout)"
},
- // Additional Force Parameters
- d3AlphaTarget: {
- value: 0,
- min: 0,
- max: 1,
- step: 0.01,
- description: "Target alpha value for the simulation (0 = stop, higher values = continuous motion)"
- },
- linkStrength: {
- value: 1,
- min: 0,
- max: 2,
- step: 0.1,
- description: "Strength of the link force (higher = stronger connection between linked nodes)"
- },
- chargeStrength: {
- value: -30,
- min: -500,
- max: 100,
- step: 10,
- description: "Strength of the charge force (negative = repulsion, positive = attraction)"
- },
- centerStrength: {
- value: 1,
- min: 0,
- max: 2,
- step: 0.1,
- description: "Strength of the centering force (higher = stronger pull towards center)"
- },
- collisionRadius: {
- value: 0,
- min: 0,
- max: 50,
- step: 1,
- description: "Collision radius for nodes (0 = no collision detection, higher = larger collision area)"
- },
- collisionStrength: {
- value: 1,
- min: 0,
- max: 2,
- step: 0.1,
- description: "Strength of collision force (higher = stronger collision avoidance)"
- }
+
}
type GraphSettingsStore = {
@@ -174,40 +132,34 @@ export const useGraphSettingsStore = create<GraphSettingsStore>()(
// Force Presets Implementation
getPresets: () => ({
'Tight Clusters': {
- chargeStrength: -10,
- linkStrength: 2.5,
- centerStrength: 1.5,
+ d3AlphaDecay: 0.1,
d3VelocityDecay: 0.8,
- collisionRadius: 8
+ cooldownTicks: 300,
+ cooldownTime: 20000
},
'Compact Network': {
- chargeStrength: -20,
- linkStrength: 2.0,
- centerStrength: 1.2,
+ d3AlphaDecay: 0.08,
d3VelocityDecay: 0.7,
- collisionRadius: 5
+ cooldownTicks: 250,
+ cooldownTime: 18000
},
'Balanced Layout': {
- chargeStrength: -40,
- linkStrength: 1.5,
- centerStrength: 1.0,
+ d3AlphaDecay: 0.06,
d3VelocityDecay: 0.6,
- collisionRadius: 6
+ cooldownTicks: 200,
+ cooldownTime: 15000
},
'Loose Organic': {
- chargeStrength: -80,
- linkStrength: 1.0,
- centerStrength: 0.8,
+ d3AlphaDecay: 0.04,
d3VelocityDecay: 0.4,
- collisionRadius: 10
+ cooldownTicks: 150,
+ cooldownTime: 12000
},
'High Energy': {
- chargeStrength: -60,
- linkStrength: 1.2,
- centerStrength: 0.5,
+ d3AlphaDecay: 0.02,
d3VelocityDecay: 0.3,
- d3AlphaTarget: 0.1,
- collisionRadius: 12
+ cooldownTicks: 100,
+ cooldownTime: 10000
}
}),
diff --git a/flowsint-app/src/renderer/src/stores/graph-store.ts b/flowsint-app/src/renderer/src/stores/graph-store.ts
index eebc2fad..4c8b01aa 100644
--- a/flowsint-app/src/renderer/src/stores/graph-store.ts
+++ b/flowsint-app/src/renderer/src/stores/graph-store.ts
@@ -254,13 +254,20 @@ export const useGraphStore = create<GraphState>()(
const { selectedNodes, isCurrent } = get()
return selectedNodes.some((node) => node.id === nodeId) || isCurrent(nodeId)
},
- setCurrentNode: (node) => set({ currentNode: node }),
+ setCurrentNode: (node) => {
+ const { currentNode } = get()
+ // Only update if the node is actually different
+ if (currentNode?.id !== node?.id) {
+ set({ currentNode: node })
+ }
+ },
setSelectedNodes: (nodes) => set({ selectedNodes: nodes }),
clearSelectedNodes: () => set({ selectedNodes: [], currentNode: null }),
toggleNodeSelection: (node, multiSelect = false) => {
const { selectedNodes, currentNode } = get()
const isSelected = selectedNodes.some((n) => n.id === node.id)
let newSelected: GraphNode[]
+ let newCurrentNode = currentNode
if (multiSelect) {
newSelected = isSelected
@@ -268,12 +275,23 @@ export const useGraphStore = create<GraphState>()(
: [...selectedNodes, node]
} else {
newSelected = isSelected && selectedNodes.length === 1 ? [] : [node]
+ // Only update currentNode if it's actually different
+ if (!multiSelect) {
+ newCurrentNode = isSelected ? null : node
+ }
}
- set({
- selectedNodes: newSelected,
- currentNode: multiSelect ? currentNode : (isSelected ? null : node),
- })
+ // Only update if there are actual changes
+ const hasSelectionChanges = newSelected.length !== selectedNodes.length ||
+ newSelected.some((n, i) => n.id !== selectedNodes[i]?.id)
+ const hasCurrentNodeChanges = newCurrentNode?.id !== currentNode?.id
+
+ if (hasSelectionChanges || hasCurrentNodeChanges) {
+ set({
+ selectedNodes: newSelected,
+ currentNode: newCurrentNode,
+ })
+ }
},
// === Relation ===
@@ -291,7 +309,16 @@ export const useGraphStore = create<GraphState>()(
setOpenNodeEditorModal: (open) => set({ openNodeEditorModal: open }),
// === Action Type for Edit form ===
- handleEdit: (node) => set({ currentNode: node, openNodeEditorModal: true }),
+ handleEdit: (node) => {
+ const { currentNode, openNodeEditorModal } = get()
+ // Only update if the node is actually different
+ if (currentNode?.id !== node.id) {
+ set({ currentNode: node, openNodeEditorModal: true })
+ } else if (!openNodeEditorModal) {
+ // Only open modal if it's not already open
+ set({ openNodeEditorModal: true })
+ }
+ },
// === Action Type for Form ===
currentNodeType: null,
diff --git a/flowsint-app/src/renderer/src/styles.css b/flowsint-app/src/renderer/src/styles.css
index ebeb1e37..6c73f52d 100644
--- a/flowsint-app/src/renderer/src/styles.css
+++ b/flowsint-app/src/renderer/src/styles.css
@@ -6,19 +6,27 @@
/* Container Queries */
@container (min-width: 640px) {
- .cq-sm\:grid-cols-2 { grid-template-columns: repeat(2, minmax(0, 1fr)); }
+ .cq-sm\:grid-cols-2 {
+ grid-template-columns: repeat(2, minmax(0, 1fr));
+ }
}
@container (min-width: 768px) {
- .cq-md\:grid-cols-3 { grid-template-columns: repeat(3, minmax(0, 1fr)); }
+ .cq-md\:grid-cols-3 {
+ grid-template-columns: repeat(3, minmax(0, 1fr));
+ }
}
@container (min-width: 1024px) {
- .cq-lg\:grid-cols-4 { grid-template-columns: repeat(4, minmax(0, 1fr)); }
+ .cq-lg\:grid-cols-4 {
+ grid-template-columns: repeat(4, minmax(0, 1fr));
+ }
}
@container (min-width: 1280px) {
- .cq-xl\:grid-cols-5 { grid-template-columns: repeat(5, minmax(0, 1fr)); }
+ .cq-xl\:grid-cols-5 {
+ grid-template-columns: repeat(5, minmax(0, 1fr));
+ }
}
/* Electron title bar draggable region */
@@ -951,3 +959,163 @@ pre {
@apply rounded-2xl overflow-hidden bg-background border border-border;
}
+
+:root {
+ --background: oklch(1.0000 0 0);
+ --foreground: oklch(0.2101 0.0318 264.6645);
+ --card: oklch(1.0000 0 0);
+ --card-foreground: oklch(0.2101 0.0318 264.6645);
+ --popover: oklch(1.0000 0 0);
+ --popover-foreground: oklch(0.2101 0.0318 264.6645);
+ --primary: oklch(0.6716 0.1368 48.5130);
+ --primary-foreground: oklch(1.0000 0 0);
+ --secondary: oklch(0.5360 0.0398 196.0280);
+ --secondary-foreground: oklch(1.0000 0 0);
+ --muted: oklch(0.9670 0.0029 264.5419);
+ --muted-foreground: oklch(0.5510 0.0234 264.3637);
+ --accent: oklch(0.9491 0 0);
+ --accent-foreground: oklch(0.2101 0.0318 264.6645);
+ --destructive: oklch(0.6368 0.2078 25.3313);
+ --destructive-foreground: oklch(0.9851 0 0);
+ --border: oklch(0.9276 0.0058 264.5313);
+ --input: oklch(0.9276 0.0058 264.5313);
+ --ring: oklch(0.6716 0.1368 48.5130);
+ --chart-1: oklch(0.5940 0.0443 196.0233);
+ --chart-2: oklch(0.7214 0.1337 49.9802);
+ --chart-3: oklch(0.8721 0.0864 68.5474);
+ --chart-4: oklch(0.6268 0 0);
+ --chart-5: oklch(0.6830 0 0);
+ --sidebar: oklch(0.9670 0.0029 264.5419);
+ --sidebar-foreground: oklch(0.2101 0.0318 264.6645);
+ --sidebar-primary: oklch(0.6716 0.1368 48.5130);
+ --sidebar-primary-foreground: oklch(1.0000 0 0);
+ --sidebar-accent: oklch(1.0000 0 0);
+ --sidebar-accent-foreground: oklch(0.2101 0.0318 264.6645);
+ --sidebar-border: oklch(0.9276 0.0058 264.5313);
+ --sidebar-ring: oklch(0.6716 0.1368 48.5130);
+ --font-sans: Geist Mono, ui-monospace, monospace;
+ --font-serif: serif;
+ --font-mono: JetBrains Mono, monospace;
+ --radius: 0.75rem;
+ --shadow-2xs: 0px 1px 4px 0px hsl(0 0% 0% / 0.03);
+ --shadow-xs: 0px 1px 4px 0px hsl(0 0% 0% / 0.03);
+ --shadow-sm: 0px 1px 4px 0px hsl(0 0% 0% / 0.05), 0px 1px 2px -1px hsl(0 0% 0% / 0.05);
+ --shadow: 0px 1px 4px 0px hsl(0 0% 0% / 0.05), 0px 1px 2px -1px hsl(0 0% 0% / 0.05);
+ --shadow-md: 0px 1px 4px 0px hsl(0 0% 0% / 0.05), 0px 2px 4px -1px hsl(0 0% 0% / 0.05);
+ --shadow-lg: 0px 1px 4px 0px hsl(0 0% 0% / 0.05), 0px 4px 6px -1px hsl(0 0% 0% / 0.05);
+ --shadow-xl: 0px 1px 4px 0px hsl(0 0% 0% / 0.05), 0px 8px 10px -1px hsl(0 0% 0% / 0.05);
+ --shadow-2xl: 0px 1px 4px 0px hsl(0 0% 0% / 0.13);
+ --tracking-normal: 0rem;
+ --spacing: 0.25rem;
+}
+
+.dark {
+ --background: oklch(0.20 0 0);
+ --foreground: oklch(0.8109 0 0);
+ --card: oklch(0.1822 0 0);
+ --card-foreground: oklch(0.8109 0 0);
+ --popover: oklch(0.1797 0.0043 308.1928);
+ --popover-foreground: oklch(0.8109 0 0);
+ --primary: oklch(0.7214 0.1337 49.9802);
+ --primary-foreground: oklch(0.1797 0.0043 308.1928);
+ --secondary: oklch(0.5940 0.0443 196.0233);
+ --secondary-foreground: oklch(0.1797 0.0043 308.1928);
+ --muted: oklch(0.2520 0 0);
+ --muted-foreground: oklch(0.6268 0 0);
+ --accent: oklch(0.3211 0 0);
+ --accent-foreground: oklch(0.8109 0 0);
+ --destructive: oklch(0.5940 0.0443 196.0233);
+ --destructive-foreground: oklch(0.1797 0.0043 308.1928);
+ --border: oklch(0.2520 0 0);
+ --input: oklch(0.2520 0 0);
+ --ring: oklch(0.7214 0.1337 49.9802);
+ --chart-1: oklch(0.5940 0.0443 196.0233);
+ --chart-2: oklch(0.7214 0.1337 49.9802);
+ --chart-3: oklch(0.8721 0.0864 68.5474);
+ --chart-4: oklch(0.6268 0 0);
+ --chart-5: oklch(0.6830 0 0);
+ --sidebar: oklch(0.1822 0 0);
+ --sidebar-foreground: oklch(0.8109 0 0);
+ --sidebar-primary: oklch(0.7214 0.1337 49.9802);
+ --sidebar-primary-foreground: oklch(0.1797 0.0043 308.1928);
+ --sidebar-accent: oklch(0.3211 0 0);
+ --sidebar-accent-foreground: oklch(0.8109 0 0);
+ --sidebar-border: oklch(0.2520 0 0);
+ --sidebar-ring: oklch(0.7214 0.1337 49.9802);
+ --font-sans: Geist Mono, ui-monospace, monospace;
+ --font-serif: serif;
+ --font-mono: JetBrains Mono, monospace;
+ --radius: 0.75rem;
+ --shadow-2xs: 0px 1px 4px 0px hsl(0 0% 0% / 0.03);
+ --shadow-xs: 0px 1px 4px 0px hsl(0 0% 0% / 0.03);
+ --shadow-sm: 0px 1px 4px 0px hsl(0 0% 0% / 0.05), 0px 1px 2px -1px hsl(0 0% 0% / 0.05);
+ --shadow: 0px 1px 4px 0px hsl(0 0% 0% / 0.05), 0px 1px 2px -1px hsl(0 0% 0% / 0.05);
+ --shadow-md: 0px 1px 4px 0px hsl(0 0% 0% / 0.05), 0px 2px 4px -1px hsl(0 0% 0% / 0.05);
+ --shadow-lg: 0px 1px 4px 0px hsl(0 0% 0% / 0.05), 0px 4px 6px -1px hsl(0 0% 0% / 0.05);
+ --shadow-xl: 0px 1px 4px 0px hsl(0 0% 0% / 0.05), 0px 8px 10px -1px hsl(0 0% 0% / 0.05);
+ --shadow-2xl: 0px 1px 4px 0px hsl(0 0% 0% / 0.13);
+}
+
+@theme inline {
+ --color-background: var(--background);
+ --color-foreground: var(--foreground);
+ --color-card: var(--card);
+ --color-card-foreground: var(--card-foreground);
+ --color-popover: var(--popover);
+ --color-popover-foreground: var(--popover-foreground);
+ --color-primary: var(--primary);
+ --color-primary-foreground: var(--primary-foreground);
+ --color-secondary: var(--secondary);
+ --color-secondary-foreground: var(--secondary-foreground);
+ --color-muted: var(--muted);
+ --color-muted-foreground: var(--muted-foreground);
+ --color-accent: var(--accent);
+ --color-accent-foreground: var(--accent-foreground);
+ --color-destructive: var(--destructive);
+ --color-destructive-foreground: var(--destructive-foreground);
+ --color-border: var(--border);
+ --color-input: var(--input);
+ --color-ring: var(--ring);
+ --color-chart-1: var(--chart-1);
+ --color-chart-2: var(--chart-2);
+ --color-chart-3: var(--chart-3);
+ --color-chart-4: var(--chart-4);
+ --color-chart-5: var(--chart-5);
+ --color-sidebar: var(--sidebar);
+ --color-sidebar-foreground: var(--sidebar-foreground);
+ --color-sidebar-primary: var(--sidebar-primary);
+ --color-sidebar-primary-foreground: var(--sidebar-primary-foreground);
+ --color-sidebar-accent: var(--sidebar-accent);
+ --color-sidebar-accent-foreground: var(--sidebar-accent-foreground);
+ --color-sidebar-border: var(--sidebar-border);
+ --color-sidebar-ring: var(--sidebar-ring);
+
+ --font-sans: var(--font-sans);
+ --font-mono: var(--font-mono);
+ --font-serif: var(--font-serif);
+
+ --radius-sm: calc(var(--radius) - 4px);
+ --radius-md: calc(var(--radius) - 2px);
+ --radius-lg: var(--radius);
+ --radius-xl: calc(var(--radius) + 4px);
+
+ --shadow-2xs: var(--shadow-2xs);
+ --shadow-xs: var(--shadow-xs);
+ --shadow-sm: var(--shadow-sm);
+ --shadow: var(--shadow);
+ --shadow-md: var(--shadow-md);
+ --shadow-lg: var(--shadow-lg);
+ --shadow-xl: var(--shadow-xl);
+ --shadow-2xl: var(--shadow-2xl);
+
+ --tracking-tighter: calc(var(--tracking-normal) - 0.05em);
+ --tracking-tight: calc(var(--tracking-normal) - 0.025em);
+ --tracking-normal: var(--tracking-normal);
+ --tracking-wide: calc(var(--tracking-normal) + 0.025em);
+ --tracking-wider: calc(var(--tracking-normal) + 0.05em);
+ --tracking-widest: calc(var(--tracking-normal) + 0.1em);
+}
+
+body {
+ letter-spacing: var(--tracking-normal);
+}
\ No newline at end of file
diff --git a/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.py b/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.py
index 54bfb9e6..b4fb0c21 100644
--- a/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.py
+++ b/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.py
@@ -17,7 +17,7 @@ class AsnToCidrsScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "asn_to_cidrs_scanner"
+ return "asn_to_cidrs"
@classmethod
def category(cls) -> str:
diff --git a/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.py b/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.py
index 9b40b1fe..1192479f 100644
--- a/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.py
+++ b/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.py
@@ -15,7 +15,7 @@ class CidrToIpsScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "cidr_to_ips_scanner"
+ return "cidr_to_ips"
@classmethod
def key(cls) -> str:
diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py
index 597a4180..a3985ef1 100644
--- a/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py
+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py
@@ -65,7 +65,7 @@ class CryptoWalletAddressToNFTs(Scanner):
@classmethod
def name(cls) -> str:
- return "wallet_to_nfts"
+ return "cryptowallet_to_nfts"
@classmethod
def category(cls) -> str:
diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py
index 59ddb2da..fc7b99e4 100644
--- a/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py
+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py
@@ -70,7 +70,7 @@ class CryptoWalletAddressToTransactions(Scanner):
@classmethod
def name(cls) -> str:
- return "wallet_to_transactions"
+ return "cryptowallet_to_transactions"
@classmethod
def category(cls) -> str:
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_asn.py b/flowsint-transforms/src/flowsint_transforms/domain/to_asn.py
index 21d6b162..635ced2a 100644
--- a/flowsint-transforms/src/flowsint_transforms/domain/to_asn.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_asn.py
@@ -18,7 +18,7 @@ class DomainToAsnScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "domain_to_asn_scanner"
+ return "domain_to_asn"
@classmethod
def category(cls) -> str:
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_history.py b/flowsint-transforms/src/flowsint_transforms/domain/to_history.py
new file mode 100644
index 00000000..9cedbcef
--- /dev/null
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_history.py
@@ -0,0 +1,519 @@
+import os
+import re
+import json
+from typing import Any, List, Union, Dict, Set
+from flowsint_core.core.scanner_base import Scanner
+from flowsint_types.domain import Domain
+from flowsint_types.individual import Individual
+from flowsint_core.utils import is_valid_domain, is_root_domain
+from flowsint_types.address import PhysicalAddress
+from flowsint_core.core.logger import Logger
+from tools.network.whoxy import WhoxyTool
+from dotenv import load_dotenv
+
+load_dotenv()
+
+WHOXY_API_KEY = os.getenv("WHOXY_API_KEY")
+
+
+class DomainToHistoryScanner(Scanner):
+ """[WHOXY] Takes a domain and returns history infos about it (history, organization, owners, emails, etc.)."""
+
+ # Define types as class attributes - base class handles schema generation automatically
+ InputType = List[Domain]
+ OutputType = List[Domain]
+
+ @classmethod
+ def name(cls) -> str:
+ return "domain_to_history"
+
+ @classmethod
+ def category(cls) -> str:
+ return "Domain"
+
+ @classmethod
+ def key(cls) -> str:
+ return "domain"
+
+ def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:
+ cleaned: InputType = []
+ for item in data:
+ domain_obj = None
+ if isinstance(item, str):
+ domain_obj = Domain(domain=item, root=is_root_domain(item))
+ elif isinstance(item, dict) and "domain" in item:
+ domain_obj = Domain(
+ domain=item["domain"], root=is_root_domain(item["domain"])
+ )
+ elif isinstance(item, Domain):
+ # If the Domain object already exists, update its root field
+ domain_obj = Domain(
+ domain=item.domain, root=is_root_domain(item.domain)
+ )
+ if domain_obj and is_valid_domain(domain_obj.domain):
+ cleaned.append(domain_obj)
+ return cleaned
+
+ async def scan(self, data: InputType) -> OutputType:
+ """Find infos related to domains using whoxy api."""
+ domains: OutputType = []
+ self._extracted_data = [] # Store all extracted data for postprocess
+ self._extracted_individuals = [] # Store extracted individuals for testing
+
+ for domain in data:
+ infos_data = self.__get_infos_from_whoxy(domain.domain)
+ if infos_data and "whois_records" in infos_data:
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"[WHOXY] Found {len(infos_data['whois_records'])} WHOIS records for {domain.domain}"
+ },
+ )
+
+ # Process each WHOIS record
+ for record in infos_data["whois_records"]:
+ if self.__is_valid_whois_record(record):
+ domain_name = record.get("domain_name")
+ if domain_name:
+ domain_obj = Domain(domain=domain_name, root=True)
+ domains.append(domain_obj)
+
+ # Store extracted data for postprocess
+ extracted_info = {
+ "domain": domain_obj,
+ "original_domain": domain,
+ "whois_record": record,
+ "contacts": {
+ "registrant": record.get("registrant_contact", {}),
+ "administrative": record.get(
+ "administrative_contact", {}
+ ),
+ "technical": record.get("technical_contact", {}),
+ "billing": record.get("billing_contact", {}),
+ },
+ }
+ self._extracted_data.append(extracted_info)
+
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"[WHOXY] Processing WHOIS record {record.get('num', 'unknown')} for {domain_name}"
+ },
+ )
+
+ # Process contacts and extract individuals during scan
+ self.__process_contacts_during_scan(extracted_info)
+ else:
+ Logger.info(
+ self.sketch_id,
+ {"message": f"[WHOXY] No info found for domain {domain.domain}."},
+ )
+ return domains
+
+ def __process_contacts_during_scan(self, extracted_info: Dict[str, Any]):
+ """Process contacts and extract individuals during scan method."""
+ domain_name = extracted_info["domain"].domain
+ contacts = extracted_info["contacts"]
+
+ for contact_type, contact in contacts.items():
+ if contact and self.__has_non_redacted_info(contact):
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"[WHOXY] Processing {contact_type} contact for {domain_name}"
+ },
+ )
+
+ # Extract individual
+ individual = self.__extract_individual_from_contact(
+ contact, contact_type
+ )
+ if individual:
+ # Store the extracted individual for testing/debugging
+ individual_info = {
+ "individual": individual,
+ "contact_type": contact_type,
+ "domain_name": domain_name,
+ "original_domain": extracted_info["original_domain"].domain,
+ "contact_data": contact,
+ }
+ self._extracted_individuals.append(individual_info)
+
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"[WHOXY] Extracted individual: {individual.full_name} ({contact_type}) for {domain_name}"
+ },
+ )
+ else:
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"[WHOXY] Failed to extract individual from {contact_type} contact for {domain_name}"
+ },
+ )
+ elif contact:
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"[WHOXY] Skipping {contact_type} contact (all info redacted) for {domain_name}"
+ },
+ )
+
+ def __get_infos_from_whoxy(self, domain: str) -> Dict[str, Any]:
+ """Get WHOIS history information from Whoxy API or test data."""
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"[WHOXY] Getting info for domain: '{domain}' (length: {len(domain)})"
+ },
+ )
+
+ infos: Dict[str, Any] = {}
+ whoxy = WhoxyTool()
+ try:
+ params = {
+ "key": WHOXY_API_KEY,
+ "history": domain,
+ }
+ infos = whoxy.launch(params=params)
+ except Exception as e:
+ Logger.error(
+ self.sketch_id,
+ {"message": f"[WHOXY] Whoxy exception for {domain}: {e}"},
+ )
+ return infos
+
+ def __is_valid_whois_record(self, record: Dict[str, Any]) -> bool:
+ """Check if a WHOIS record is valid and contains useful information."""
+ domain_name = record.get("domain_name")
+ if not domain_name:
+ return False
+
+ # A record is valid if it has a domain name - we'll filter contacts individually later
+ return True
+
+ def __has_non_redacted_info(self, contact: Dict[str, Any]) -> bool:
+ """Check if a contact has any non-redacted information."""
+ if not contact:
+ return False
+
+ # Check key fields for non-redacted information
+ fields_to_check = [
+ "full_name",
+ "email_address",
+ "phone_number",
+ "mailing_address",
+ "city_name",
+ "zip_code",
+ "country_name",
+ ]
+
+ for field in fields_to_check:
+ value = contact.get(field, "")
+ if value and not self.__is_redacted(value):
+ return True
+
+ return False
+
+ return False
+
+ def __is_redacted(self, value: str) -> bool:
+ """Check if a value is redacted."""
+ if not value:
+ return True
+ return "REDACTED FOR PRIVACY" in value.upper() or "PRIVACY" in value.upper()
+
+ def __extract_individual_from_contact(
+ self, contact: Dict[str, Any], contact_type: str
+ ) -> Individual:
+ """Extract individual information from contact data."""
+ full_name = contact.get("full_name", "")
+
+ # Skip if name is redacted
+ if self.__is_redacted(full_name):
+ Logger.info(
+ self.sketch_id,
+ {"message": f"[WHOXY] Skipping redacted contact: {full_name}"},
+ )
+ return None
+
+ # Parse full name into first and last name
+ name_parts = full_name.strip().split()
+ first_name = name_parts[0] if name_parts else ""
+ last_name = " ".join(name_parts[1:]) if len(name_parts) > 1 else ""
+
+ # Extract email and phone
+ email_raw = contact.get("email_address", "")
+ phone = contact.get("phone_number", "")
+
+ # Handle comma-separated emails
+ emails = []
+ if email_raw and not self.__is_redacted(email_raw):
+ # Split by comma and clean up each email
+ email_list = [e.strip() for e in email_raw.split(",")]
+ for email in email_list:
+ if email and self.__is_valid_email(email):
+ emails.append(email)
+
+ # Skip if phone is redacted
+ if self.__is_redacted(phone):
+ phone = ""
+
+ # Extract address information
+ address = contact.get("mailing_address", "")
+ city = contact.get("city_name", "")
+ zip_code = contact.get("zip_code", "")
+ country = contact.get("country_name", "")
+
+ # Skip if address is redacted
+ if self.__is_redacted(address):
+ address = ""
+ if self.__is_redacted(city):
+ city = ""
+ if self.__is_redacted(zip_code):
+ zip_code = ""
+ if self.__is_redacted(country):
+ country = ""
+
+ # Create individual object
+ individual = Individual(
+ first_name=first_name,
+ last_name=last_name,
+ full_name=full_name,
+ email_addresses=emails if emails else None,
+ phone_numbers=[phone] if phone else None,
+ )
+
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"[WHOXY] Extracted individual: {full_name} ({contact_type}) with {len(emails)} emails"
+ },
+ )
+
+ return individual
+
+ def __is_valid_email(self, email: str) -> bool:
+ """Check if email is valid."""
+ if not email:
+ return False
+ # Basic email validation
+ pattern = r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$"
+ return bool(re.match(pattern, email))
+
+ def __extract_physical_address(self, contact: Dict[str, Any]) -> PhysicalAddress:
+ """Extract physical address from contact data."""
+ address = contact.get("mailing_address", "")
+ city = contact.get("city_name", "")
+ zip_code = contact.get("zip_code", "")
+ country = contact.get("country_name", "")
+
+ # Skip if any part is redacted
+ if any(
+ self.__is_redacted(field) for field in [address, city, zip_code, country]
+ ):
+ return None
+
+ if not all([address, city, zip_code, country]):
+ return None
+
+ return PhysicalAddress(
+ address=address, city=city, zip=zip_code, country=country
+ )
+
+ def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ """Create Neo4j nodes and relationships from extracted data."""
+ if not self.neo4j_conn:
+ Logger.info(
+ self.sketch_id,
+ {"message": "[WHOXY] No Neo4j connection, skipping postprocess"},
+ )
+ return results
+
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"[WHOXY] Starting postprocess with {len(self._extracted_individuals)} individuals"
+ },
+ )
+
+ # Track processed entities to avoid duplicates
+ processed_domains: Set[str] = set()
+ processed_individuals: Set[str] = set()
+ processed_emails: Set[str] = set()
+ processed_phones: Set[str] = set()
+ processed_addresses: Set[str] = set()
+
+ # Process extracted individuals (already filtered and extracted during scan)
+ for individual_info in self._extracted_individuals:
+ individual = individual_info["individual"]
+ contact_type = individual_info["contact_type"]
+ domain_name = individual_info["domain_name"]
+ original_domain_name = individual_info["original_domain"]
+
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"[WHOXY] Processing individual: {individual.full_name} ({contact_type}) for {domain_name}"
+ },
+ )
+
+ # Create domain node if not already processed
+ if domain_name not in processed_domains:
+ processed_domains.add(domain_name)
+ Logger.info(
+ self.sketch_id,
+ {"message": f"[WHOXY] Creating domain node: {domain_name}"},
+ )
+ self.create_node(
+ "domain",
+ "domain",
+ domain_name,
+ label=domain_name,
+ caption=domain_name,
+ type="domain",
+ )
+
+ # Create relationship between original domain and found domain
+ self.create_relationship(
+ "domain",
+ "domain",
+ original_domain_name,
+ "domain",
+ "domain",
+ domain_name,
+ "HAS_RELATED_DOMAIN",
+ )
+
+ # Create individual node if not already processed
+ individual_id = (
+ f"{individual.first_name}_{individual.last_name}_{individual.full_name}"
+ )
+ if individual_id not in processed_individuals:
+ processed_individuals.add(individual_id)
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"[WHOXY] Creating individual node: {individual.full_name}"
+ },
+ )
+ self.create_node(
+ "individual",
+ "full_name",
+ individual.full_name,
+ caption=individual.full_name,
+ type="individual",
+ )
+
+ # Create relationship between individual and domain
+ self.create_relationship(
+ "individual",
+ "full_name",
+ individual.full_name,
+ "domain",
+ "domain",
+ domain_name,
+ f"IS_{contact_type.upper()}_CONTACT",
+ )
+
+ # Process email addresses
+ if individual.email_addresses:
+ for email in individual.email_addresses:
+ if email and email not in processed_emails:
+ processed_emails.add(email)
+ Logger.info(
+ self.sketch_id,
+ {"message": f"[WHOXY] Creating email node: {email}"},
+ )
+ self.create_node(
+ "email",
+ "email",
+ email,
+ caption=email,
+ type="email",
+ )
+ self.create_relationship(
+ "individual",
+ "full_name",
+ individual.full_name,
+ "email",
+ "email",
+ email,
+ "HAS_EMAIL",
+ )
+
+ # Process phone numbers
+ if individual.phone_numbers:
+ for phone in individual.phone_numbers:
+ if phone and phone not in processed_phones:
+ processed_phones.add(phone)
+ Logger.info(
+ self.sketch_id,
+ {"message": f"[WHOXY] Creating phone node: {phone}"},
+ )
+ self.create_node(
+ "phone",
+ "number",
+ phone,
+ caption=phone,
+ type="phone",
+ )
+ self.create_relationship(
+ "individual",
+ "full_name",
+ individual.full_name,
+ "phone",
+ "number",
+ phone,
+ "HAS_PHONE",
+ )
+
+ # Process physical address from contact data
+ contact_data = individual_info["contact_data"]
+ address = self.__extract_physical_address(contact_data)
+ if address:
+ address_id = (
+ f"{address.address}_{address.city}_{address.zip}_{address.country}"
+ )
+ if address_id not in processed_addresses:
+ processed_addresses.add(address_id)
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"[WHOXY] Creating address node: {address.address}"
+ },
+ )
+ self.create_node(
+ "location",
+ "address",
+ address.address,
+ caption=f"{address.address}, {address.city}",
+ type="location",
+ )
+ self.create_relationship(
+ "individual",
+ "full_name",
+ individual.full_name,
+ "location",
+ "address",
+ address.address,
+ "LIVES_AT",
+ )
+
+ self.log_graph_message(
+ f"Processed individual {individual.full_name} ({contact_type}) for domain {domain_name}"
+ )
+
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"[WHOXY] Postprocess completed. Processed {len(processed_individuals)} individuals"
+ },
+ )
+
+ return results
+
+
+InputType = DomainToHistoryScanner.InputType
+OutputType = DomainToHistoryScanner.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py b/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py
index ee65289f..e5b5590a 100644
--- a/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py
@@ -294,7 +294,7 @@ Check FlowSint logs for detailed resolution information.
domain_obj.domain,
type="domain",
)
- self.create_node("ip", "address", ip_obj.address, type="ip")
+ self.create_node("ip", "address", ip_obj.address, **ip_obj.__dict__)
self.create_relationship(
"domain",
"domain",
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py b/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py
index a0fcbc59..30a0a98e 100644
--- a/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py
@@ -74,10 +74,10 @@ class DomainToRootDomain(Scanner):
continue
# Create root domain node
- self.create_node("domain", "domain", root_domain.domain, type="domain")
+ self.create_node("domain", "domain", root_domain.domain, **root_domain.__dict__)
# Create original domain node
- self.create_node("domain", "domain", original_domain.domain, type="domain")
+ self.create_node("domain", "domain", original_domain.domain, **original_domain.__dict__)
# Create relationship from root domain to original domain
self.create_relationship(
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py b/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py
index 4cb590d5..55cd497b 100644
--- a/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py
@@ -124,7 +124,7 @@ class SubdomainScanner(Scanner):
)
# Create subdomain node
- self.create_node("domain", "domain", subdomain, type="domain")
+ self.create_node("domain", "domain", subdomain, domain=subdomain)
# Create relationship from parent domain to subdomain
self.create_relationship(
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py b/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py
index 615bacd0..7f08f685 100644
--- a/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py
@@ -113,7 +113,7 @@ class WhoisScanner(Scanner):
continue
# Create domain node
- self.create_node("domain", "domain", whois_obj.domain, type="domain")
+ self.create_node("domain", "domain", whois_obj.domain, **whois_obj.__dict__)
# Create whois node
whois_key = f"{whois_obj.domain}_{self.sketch_id}"
@@ -173,7 +173,7 @@ class WhoisScanner(Scanner):
)
if whois_obj.email:
- self.create_node("email", "email", whois_obj.email.email, type="email")
+ self.create_node("email", "email", whois_obj.email.email, **whois_obj.email.__dict__)
self.create_relationship(
"whois",
"whois_id",
diff --git a/flowsint-transforms/src/flowsint_transforms/email/to_domains.py b/flowsint-transforms/src/flowsint_transforms/email/to_domains.py
index 46dde35c..a1ee8f12 100644
--- a/flowsint-transforms/src/flowsint_transforms/email/to_domains.py
+++ b/flowsint-transforms/src/flowsint_transforms/email/to_domains.py
@@ -24,7 +24,7 @@ class EmailToDomainsScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "email_to_domains_scanner"
+ return "email_to_domains"
@classmethod
def category(cls) -> str:
diff --git a/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py b/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py
index 77ff2de3..318f9918 100644
--- a/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py
+++ b/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py
@@ -46,14 +46,21 @@ class EmailToGravatarScanner(Scanner):
try:
# Generate MD5 hash of email
email_hash = hashlib.md5(email.email.lower().encode()).hexdigest()
-
# Query Gravatar API
gravatar_url = f"https://www.gravatar.com/avatar/{email_hash}?d=404"
+ Logger.warn(
+ self.sketch_id,
+ {"message": f"email url: {gravatar_url}"},
+ )
response = requests.head(gravatar_url, timeout=10)
if response.status_code == 200:
# Gravatar found, get profile info
profile_url = f"https://www.gravatar.com/{email_hash}.json"
+ Logger.warn(
+ self.sketch_id,
+ {"message": f"Gravatar url: {profile_url}"},
+ )
profile_response = requests.get(profile_url, timeout=10)
gravatar_data = {
@@ -95,23 +102,12 @@ class EmailToGravatarScanner(Scanner):
continue
# Create email node
- self.create_node("email", "email", email_obj.email, type="email")
+ self.create_node("email", "email", email_obj.email, **email_obj.__dict__)
# Create gravatar node
gravatar_key = f"{email_obj.email}_{self.sketch_id}"
self.create_node(
- "gravatar",
- "gravatar_id",
- gravatar_key,
- email=email_obj.email,
- hash=gravatar_obj.hash,
- src=str(gravatar_obj.src),
- display_name=gravatar_obj.display_name,
- location=gravatar_obj.location,
- about_me=gravatar_obj.about_me,
- exists=gravatar_obj.exists,
- label="Gravatar",
- type="gravatar",
+ "gravatar", "gravatar_id", gravatar_key, **gravatar_obj.__dict__
)
# Create relationship between email and gravatar
diff --git a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py
index 9741805a..7cd6dc09 100644
--- a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py
+++ b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py
@@ -16,7 +16,7 @@ HIBP_API_KEY = os.getenv("HIBP_API_KEY")
class EmailToBreachesScanner(Scanner):
- """From email to breaches using Have I Been Pwned API."""
+ """[HIBPWNED] From email to breaches using Have I Been Pwned API."""
InputType = List[Email]
OutputType = List[tuple] # List of (email, breach) tuples
@@ -62,7 +62,8 @@ class EmailToBreachesScanner(Scanner):
"name": "HIBP_API_KEY",
"type": "vaultSecret",
"description": "The HIBP API key to use for breaches lookup.",
- "required": True,
+ "required": False,
+ "default": HIBP_API_KEY,
},
{
"name": "HIBP_API_URL",
@@ -192,7 +193,7 @@ class EmailToBreachesScanner(Scanner):
if not self.neo4j_conn:
continue
# Create email node
- self.create_node("email", "email", email_obj.email, type="email")
+ self.create_node("email", "email", email_obj.email, **email_obj.__dict__)
Logger.info(
self.sketch_id, {"message": f"Created email node: {email_obj.email}"}
)
diff --git a/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py b/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py
index 27bfb0d0..062396f6 100644
--- a/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py
+++ b/flowsint-transforms/src/flowsint_transforms/individual/to_domains.py
@@ -24,7 +24,7 @@ class IndividualToDomainsScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "individual_to_domains_scanner"
+ return "individual_to_domains"
@classmethod
def category(cls) -> str:
@@ -98,7 +98,9 @@ class IndividualToDomainsScanner(Scanner):
else:
Logger.info(
self.sketch_id,
- {"message": f"[WHOXY] No domain found for individual {individual.full_name}."},
+ {
+ "message": f"[WHOXY] No domain found for individual {individual.full_name}."
+ },
)
return domains
@@ -238,8 +240,7 @@ class IndividualToDomainsScanner(Scanner):
"individual",
"full_name",
individual.full_name,
- caption=individual.full_name,
- type="individual",
+ **individual.__dict__,
)
# Create domain node
@@ -247,9 +248,7 @@ class IndividualToDomainsScanner(Scanner):
"domain",
"domain",
domain_name,
- label=domain_name,
- caption=domain_name,
- type="domain",
+ **domain.__dict__,
)
# Create relationship between individual and domain
@@ -297,13 +296,13 @@ class IndividualToDomainsScanner(Scanner):
"""Process a contact and create all related entities and relationships."""
# Extract individual
- contact_individual = self.__extract_individual_from_contact(contact, contact_type)
+ contact_individual = self.__extract_individual_from_contact(
+ contact, contact_type
+ )
if not contact_individual:
return
- individual_id = (
- f"{contact_individual.first_name}_{contact_individual.last_name}_{contact_individual.full_name}"
- )
+ individual_id = f"{contact_individual.first_name}_{contact_individual.last_name}_{contact_individual.full_name}"
if individual_id in processed_individuals:
return
@@ -314,8 +313,7 @@ class IndividualToDomainsScanner(Scanner):
"individual",
"full_name",
contact_individual.full_name,
- caption=contact_individual.full_name,
- type="individual",
+ **contact_individual.__dict__,
)
# Create relationship between individual and domain
@@ -351,8 +349,7 @@ class IndividualToDomainsScanner(Scanner):
"email",
"email",
email,
- caption=email,
- type="email",
+ email=email,
)
# Create relationship between individual and email
@@ -377,8 +374,7 @@ class IndividualToDomainsScanner(Scanner):
"phone",
"number",
phone,
- caption=phone,
- type="phone",
+ number=phone,
)
# Create relationship between individual and phone
@@ -406,8 +402,7 @@ class IndividualToDomainsScanner(Scanner):
"location",
"address",
address.address,
- caption=f"{address.address}, {address.city}",
- type="location",
+ address=address,
)
# Create relationship between individual and address
diff --git a/flowsint-transforms/src/flowsint_transforms/individual/to_org.py b/flowsint-transforms/src/flowsint_transforms/individual/to_org.py
index 085a09aa..282370bd 100644
--- a/flowsint-transforms/src/flowsint_transforms/individual/to_org.py
+++ b/flowsint-transforms/src/flowsint_transforms/individual/to_org.py
@@ -1,5 +1,5 @@
import requests
-from typing import List, Dict, Union
+from typing import List, Dict, Union, Set
from flowsint_core.core.scanner_base import Scanner
from flowsint_types.organization import Organization
from flowsint_types.individual import Individual
@@ -8,7 +8,7 @@ from tools.organizations.sirene import SireneTool
class IndividualToOrgScanner(Scanner):
- """Find organization from a person with data from SIRENE (France only)."""
+ """[SIRENE] Find organization from a person with data from SIRENE (France only)."""
# Define types as class attributes - base class handles schema generation automatically
InputType = List[Individual]
@@ -16,7 +16,7 @@ class IndividualToOrgScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "individual_to_org"
+ return "individual_to_organization"
@classmethod
def category(cls) -> str:
@@ -283,25 +283,59 @@ class IndividualToOrgScanner(Scanner):
if not self.neo4j_conn:
return results
+ # Track processed entities to avoid duplicates
+ processed_organizations: Set[str] = set()
+ processed_individuals: Set[str] = set()
+
+ # First, create all organization nodes
for org in results:
- # Create or update the organization node with all SIRENE properties
org_key = f"{org.name}_FR"
+ if org_key in processed_organizations:
+ continue
+ processed_organizations.add(org_key)
+
self.create_node(
"Organization",
"org_id",
org_key,
- name=org.name,
- country="FR",
- siren=org.siren,
- siege_siret=org.siege_siret,
- nom_complet=org.nom_complet,
- nom_raison_sociale=org.nom_raison_sociale,
- sigle=org.sigle,
- caption=org.name,
- type="organization",
+ **org.__dict__,
+ )
+
+ # Then, create all individual nodes
+ for individual in original_input:
+ individual_id = individual.full_name
+ if individual_id in processed_individuals:
+ continue
+ processed_individuals.add(individual_id)
+
+ # Create individual node
+ self.create_node(
+ "Individual",
+ "full_name",
+ individual_id,
+ **individual.__dict__,
)
- self.log_graph_message(f"Found organization: {org.name}")
+ # Finally, create relationships between all individuals and all organizations
+ for individual in original_input:
+ individual_id = individual.full_name
+ for org in results:
+ org_key = f"{org.name}_FR"
+
+ # Create relationship between individual and organization
+ self.create_relationship(
+ "Individual",
+ "full_name",
+ individual_id,
+ "Organization",
+ "org_id",
+ org_key,
+ "WORKS_FOR",
+ )
+
+ self.log_graph_message(
+ f"Created {len(results)} organizations and {len(original_input)} individuals with relationships"
+ )
return results
diff --git a/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py b/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py
index 0989b673..fb4910d3 100644
--- a/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py
+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py
@@ -20,7 +20,7 @@ class ReverseResolveScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "ip_to_domain_scanner"
+ return "ip_to_domain"
@classmethod
def category(cls) -> str:
@@ -85,21 +85,23 @@ class ReverseResolveScanner(Scanner):
return results
- def postprocess(
- self, results: OutputType, original_input: InputType
- ) -> OutputType:
+ def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
# Create nodes and relationships for each resolved domain
for ip_obj in original_input:
# Create IP node
- self.create_node("ip", "address", ip_obj.address, type="ip")
-
+ self.create_node("ip", "address", ip_obj.address, **ip_obj.__dict__)
+
# Create domain nodes and relationships for each resolved domain
for domain_obj in results:
self.create_node(
"domain",
"domain",
domain_obj.domain,
- type="domain" if "." not in domain_obj.domain.split(".")[1:] else "subdomain",
+ type=(
+ "domain"
+ if "." not in domain_obj.domain.split(".")[1:]
+ else "subdomain"
+ ),
)
self.create_relationship(
"ip",
@@ -113,7 +115,7 @@ class ReverseResolveScanner(Scanner):
self.log_graph_message(
f"Domain found for IP {ip_obj.address} -> {domain_obj.domain}"
)
-
+
return results
diff --git a/flowsint-transforms/src/flowsint_transforms/ip/to_geolocation.py b/flowsint-transforms/src/flowsint_transforms/ip/to_geolocation.py
index da1509cf..f8f623dd 100644
--- a/flowsint-transforms/src/flowsint_transforms/ip/to_geolocation.py
+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_geolocation.py
@@ -18,7 +18,7 @@ class GeolocationScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "ip_to_geolocation_scanner"
+ return "ip_to_geolocation"
@classmethod
def category(cls) -> str:
diff --git a/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.py
index 2ac37d72..c5b310e3 100644
--- a/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.py
+++ b/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.py
@@ -20,7 +20,7 @@ class HibpScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "hibp_scanner"
+ return "to_hibp_leaks"
@classmethod
def category(cls) -> str:
@@ -86,7 +86,7 @@ class HibpScanner(Scanner):
email = result["email"]
# Create email node
- self.create_node("email", "address", email, caption=email, type="email")
+ self.create_node("email", "address", email, email=email)
# Create breach relationships
for breach in result.get("breaches", []):
diff --git a/flowsint-transforms/src/flowsint_transforms/organization/to_asn.py b/flowsint-transforms/src/flowsint_transforms/organization/to_asn.py
index ff93f57b..c7ca4c62 100644
--- a/flowsint-transforms/src/flowsint_transforms/organization/to_asn.py
+++ b/flowsint-transforms/src/flowsint_transforms/organization/to_asn.py
@@ -16,7 +16,7 @@ class OrgToAsnScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "org_to_asn_scanner"
+ return "org_to_asn"
@classmethod
def category(cls) -> str:
diff --git a/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py b/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py
index 64c1a60e..2ef16c2e 100644
--- a/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py
+++ b/flowsint-transforms/src/flowsint_transforms/organization/to_domains.py
@@ -26,7 +26,7 @@ class OrgToDomainsScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "org_to_domains_scanner"
+ return "org_to_domains"
@classmethod
def category(cls) -> str:
diff --git a/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py b/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py
index c4cfb3d4..da781460 100644
--- a/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py
+++ b/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py
@@ -14,7 +14,7 @@ class IgnorantScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "phone_to_infos_scanner"
+ return "phone_to_infos"
@classmethod
def category(cls) -> str:
diff --git a/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py b/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py
index ba6dbf07..e599885d 100644
--- a/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py
+++ b/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py
@@ -11,7 +11,7 @@ false_positives = ["LeagueOfLegends"]
class MaigretScanner(Scanner):
- """Scans usernames for associated social accounts using Maigret."""
+ """[MAIGRET] Scans usernames for associated social accounts using Maigret."""
# Define types as class attributes - base class handles schema generation automatically
InputType = List[SocialProfile]
@@ -19,7 +19,7 @@ class MaigretScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "maigret_scanner"
+ return "username_to_socials"
@classmethod
def category(cls) -> str:
@@ -159,7 +159,7 @@ class MaigretScanner(Scanner):
)
# Create username node
- self.create_node("username", "username", profile.username, type="username")
+ self.create_node("username", "username", profile.username, username=profile.username)
# Create relationship
self.create_relationship(
diff --git a/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py b/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py
index 73609be7..b305cb23 100644
--- a/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py
+++ b/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py
@@ -8,7 +8,7 @@ from flowsint_core.core.logger import Logger
class SherlockScanner(Scanner):
- """Scans the usernames for associated social accounts using Sherlock."""
+ """[SHERLOCK] Scans the usernames for associated social accounts using Sherlock."""
# Define types as class attributes - base class handles schema generation automatically
InputType = List[SocialProfile]
@@ -16,7 +16,7 @@ class SherlockScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "sherlock_scanner"
+ return "username_to_socials"
@classmethod
def category(cls) -> str:
diff --git a/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py b/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py
index 4a7e16ad..cb5f77e8 100644
--- a/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py
+++ b/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py
@@ -105,17 +105,17 @@ class WebsiteToWebtrackersScanner(Scanner):
# Create tracker nodes and relationships
for tracker in trackers:
self.create_node(
- "tracker",
+ "webtracker",
"tracker_id",
tracker.tracker_id,
caption=tracker.name,
- type="tracker"
+ type="webtracker"
)
self.create_relationship(
"website",
"url",
website_url,
- "tracker",
+ "webtracker",
"tracker_id",
tracker.tracker_id,
"HAS_TRACKER",
diff --git a/flowsint-transforms/tests/scanners/domain/to_history.py b/flowsint-transforms/tests/scanners/domain/to_history.py
new file mode 100644
index 00000000..bdb62db5
--- /dev/null
+++ b/flowsint-transforms/tests/scanners/domain/to_history.py
@@ -0,0 +1,393 @@
+import pytest
+import json
+import os
+from unittest.mock import Mock
+from flowsint_transforms.domain.to_history import DomainToHistoryScanner
+from flowsint_types.domain import Domain
+
+
+class MockNeo4jConn:
+ def __init__(self):
+ self.nodes_created = []
+ self.relationships_created = []
+
+ def create_node(self, label, key, value, **kwargs):
+ node_info = {"label": label, "key": key, "value": value, **kwargs}
+ self.nodes_created.append(node_info)
+
+ def create_relationship(
+ self,
+ from_label,
+ from_key,
+ from_value,
+ to_label,
+ to_key,
+ to_value,
+ relationship_type,
+ ):
+ rel_info = {
+ "from": f"{from_label}:{from_value}",
+ "to": f"{to_label}:{to_value}",
+ "type": relationship_type,
+ }
+ self.relationships_created.append(rel_info)
+
+ def query(self, query, params):
+ """Mock query method to avoid errors."""
+ pass
+
+
+class MockScanner(DomainToHistoryScanner):
+ def __init__(self):
+ self.sketch_id = "test_sketch_123"
+ self.neo4j_conn = MockNeo4jConn()
+ self._extracted_data = []
+ self._extracted_individuals = []
+
+ def log_graph_message(self, message):
+ """Mock log_graph_message method."""
+ pass
+
+
+def scanner():
+ """Create a scanner instance for testing."""
+ scanner = MockScanner()
+ return scanner
+
+
+def test_data():
+ """Load test data from data.json."""
+ current_dir = os.path.dirname(os.path.abspath(__file__))
+ data_file = os.path.join(current_dir, "..", "..", "test_data", "data.json")
+ with open(data_file, "r") as f:
+ return json.load(f)
+
+
+def test_preprocess_valid_domains(scanner):
+ """Test preprocessing with valid domains."""
+ domains = [
+ Domain(domain="example.com"),
+ Domain(domain="example2.com"),
+ ]
+ result = scanner.preprocess(domains)
+
+ result_domains = [d.domain for d in result]
+ expected_domains = [d.domain for d in domains]
+
+ assert result_domains == expected_domains
+
+
+def test_preprocess_string_domains(scanner):
+ """Test preprocessing with string domains."""
+ domains = ["example.com", "example2.com"]
+ result = scanner.preprocess(domains)
+
+ assert len(result) == 2
+ assert all(isinstance(d, Domain) for d in result)
+ assert result[0].domain == "example.com"
+ assert result[1].domain == "example2.com"
+
+
+def test_preprocess_dict_domains(scanner):
+ """Test preprocessing with dict domains."""
+ domains = [{"domain": "example.com"}, {"domain": "example2.com"}]
+ result = scanner.preprocess(domains)
+
+ assert len(result) == 2
+ assert all(isinstance(d, Domain) for d in result)
+ assert result[0].domain == "example.com"
+ assert result[1].domain == "example2.com"
+
+
+def test_preprocess_invalid_domains(scanner):
+ """Test preprocessing with invalid domains."""
+ domains = [
+ Domain(domain="example.com"),
+ Domain(domain="invalid_domain"),
+ Domain(domain="example.org"),
+ ]
+ result = scanner.preprocess(domains)
+
+ result_domains = [d.domain for d in result]
+ assert "example.com" in result_domains
+ assert "example.org" in result_domains
+ assert "invalid_domain" not in result_domains
+
+
+def test_is_redacted(scanner):
+ """Test the __is_redacted method."""
+ # Should be redacted
+ assert scanner._DomainToHistoryScanner__is_redacted("REDACTED FOR PRIVACY")
+ assert scanner._DomainToHistoryScanner__is_redacted("redacted for privacy")
+ assert scanner._DomainToHistoryScanner__is_redacted("Some text with PRIVACY in it")
+
+ # Should NOT be redacted
+ assert not scanner._DomainToHistoryScanner__is_redacted("JOHN DOE")
+ assert not scanner._DomainToHistoryScanner__is_redacted("[email protected]")
+ assert not scanner._DomainToHistoryScanner__is_redacted("123 JOHN STREET")
+ assert not scanner._DomainToHistoryScanner__is_redacted("DOE CITY")
+
+
+def test_has_non_redacted_info(scanner):
+ """Test the __has_non_redacted_info method."""
+ # Contact with valid information
+ valid_contact = {
+ "full_name": "JOHN DOE",
+ "email_address": "[email protected], [email protected]",
+ "phone_number": "+123456789",
+ "mailing_address": "123 JOHN STREET",
+ "city_name": "DOE CITY",
+ "zip_code": "12345",
+ "country_name": "United States",
+ }
+ assert scanner._DomainToHistoryScanner__has_non_redacted_info(valid_contact)
+
+ # Contact with all redacted information
+ redacted_contact = {
+ "full_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ }
+ assert not scanner._DomainToHistoryScanner__has_non_redacted_info(redacted_contact)
+
+ # Empty contact
+ assert not scanner._DomainToHistoryScanner__has_non_redacted_info({})
+
+
+def test_extract_individual_from_contact(scanner):
+ """Test the __extract_individual_from_contact method."""
+ # Valid contact
+ valid_contact = {
+ "full_name": "JOHN DOE",
+ "email_address": "[email protected], [email protected]",
+ "phone_number": "+123456789",
+ "mailing_address": "123 JOHN STREET",
+ "city_name": "DOE CITY",
+ "zip_code": "12345",
+ "country_name": "United States",
+ }
+
+ individual = scanner._DomainToHistoryScanner__extract_individual_from_contact(
+ valid_contact, "REGISTRANT"
+ )
+
+ assert individual is not None
+ assert individual.first_name == "MARC"
+ assert individual.last_name == "DESCOLLONGES"
+ assert individual.full_name == "JOHN DOE"
+ assert len(individual.email_addresses) == 2
+ assert "[email protected]" in individual.email_addresses
+ assert "[email protected]" in individual.email_addresses
+ assert individual.phone_numbers == ["+123456789"]
+
+
+def test_extract_individual_redacted_name(scanner):
+ """Test that individuals with redacted names are skipped."""
+ redacted_contact = {
+ "full_name": "REDACTED FOR PRIVACY",
+ "email_address": "[email protected]",
+ "phone_number": "+1234567890",
+ }
+
+ individual = scanner._DomainToHistoryScanner__extract_individual_from_contact(
+ redacted_contact, "REGISTRANT"
+ )
+ assert individual is None
+
+
+def test_is_valid_email(scanner):
+ """Test the __is_valid_email method."""
+ # Valid emails
+ assert scanner._DomainToHistoryScanner__is_valid_email("[email protected]")
+ assert scanner._DomainToHistoryScanner__is_valid_email("[email protected]")
+ assert scanner._DomainToHistoryScanner__is_valid_email("[email protected]")
+
+ # Invalid emails
+ assert not scanner._DomainToHistoryScanner__is_valid_email("invalid-email")
+ assert not scanner._DomainToHistoryScanner__is_valid_email("@example.com")
+ assert not scanner._DomainToHistoryScanner__is_valid_email("test@")
+ assert not scanner._DomainToHistoryScanner__is_valid_email("")
+
+
+def test_extract_physical_address(scanner):
+ """Test the __extract_physical_address method."""
+ # Valid address
+ valid_contact = {
+ "mailing_address": "123 JOHN STREET",
+ "city_name": "DOE CITY",
+ "zip_code": "12345",
+ "country_name": "United States",
+ }
+
+ address = scanner._DomainToHistoryScanner__extract_physical_address(valid_contact)
+
+ assert address is not None
+ assert address.address == "123 JOHN STREET"
+ assert address.city == "DOE CITY"
+ assert address.zip == "12345"
+ assert address.country == "United States"
+
+ # Address with redacted parts
+ redacted_contact = {
+ "mailing_address": "123 JOHN STREET",
+ "city_name": "REDACTED FOR PRIVACY",
+ "zip_code": "12345",
+ "country_name": "United States",
+ }
+
+ address = scanner._DomainToHistoryScanner__extract_physical_address(
+ redacted_contact
+ )
+ assert address is None
+
+
+async def test_scan_with_test_data(scanner, test_data, monkeypatch):
+ """Test the scan method with test data."""
+
+ # Mock the __get_infos_from_whoxy method to return test data
+ def mock_get_infos(domain):
+ if domain == "epios.com":
+ return test_data
+ return {}
+
+ monkeypatch.setattr(
+ scanner, "_DomainToHistoryScanner__get_infos_from_whoxy", mock_get_infos
+ )
+
+ # Test with epios.com domain
+ input_domains = [Domain(domain="epios.com")]
+ results = await scanner.scan(input_domains)
+
+ # Should find the domain (one for each WHOIS record)
+ assert len(results) == 16 # 16 WHOIS records in the test data
+ assert all(r.domain == "epios.com" for r in results)
+
+ # Should have extracted data
+ assert len(scanner._extracted_data) == 16
+
+ # Should have extracted individuals
+ assert (
+ len(scanner._extracted_individuals) > 0
+ ), "Should have extracted some individuals"
+
+ # Check that JOHN DOE is in the extracted individuals
+ marc_found = False
+ marc_individuals = []
+
+ for individual_info in scanner._extracted_individuals:
+ individual = individual_info["individual"]
+ if "JOHN DOE" in individual.full_name:
+ marc_found = True
+ marc_individuals.append(individual_info)
+ print(
+ f"Found MARC: {individual.full_name} ({individual_info['contact_type']})"
+ )
+ print(f" Emails: {individual.email_addresses}")
+ print(f" Phones: {individual.phone_numbers}")
+
+ assert marc_found, "JOHN DOE should be found in the extracted individuals"
+ assert (
+ len(marc_individuals) > 0
+ ), f"Expected to find JOHN DOE, but found {len(marc_individuals)} instances"
+
+ # Print summary of all extracted individuals
+ print(f"\n=== Summary of extracted individuals ===")
+ for individual_info in scanner._extracted_individuals:
+ individual = individual_info["individual"]
+ print(
+ f"- {individual.full_name} ({individual_info['contact_type']}) for {individual_info['domain_name']}"
+ )
+ if individual.email_addresses:
+ print(f" Emails: {individual.email_addresses}")
+ if individual.phone_numbers:
+ print(f" Phones: {individual.phone_numbers}")
+
+
+def test_postprocess_creates_nodes_and_relationships(scanner, test_data, monkeypatch):
+ """Test that postprocess creates the expected nodes and relationships."""
+
+ # Mock the __get_infos_from_whoxy method
+ def mock_get_infos(domain):
+ if domain == "epios.com":
+ return test_data
+ return {}
+
+ monkeypatch.setattr(
+ scanner, "_DomainToHistoryScanner__get_infos_from_whoxy", mock_get_infos
+ )
+
+ # First run scan to populate _extracted_data and _extracted_individuals
+ import asyncio
+
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
+
+ try:
+ input_domains = [Domain(domain="epios.com")]
+ results = loop.run_until_complete(scanner.scan(input_domains))
+
+ # Debug: Check what individuals were extracted
+ print(
+ f"\n=== DEBUG: _extracted_individuals has {len(scanner._extracted_individuals)} individuals ==="
+ )
+ for i, individual_info in enumerate(scanner._extracted_individuals):
+ individual = individual_info["individual"]
+ print(
+ f"Individual {i+1}: {individual.full_name} ({individual_info['contact_type']}) for {individual_info['domain_name']}"
+ )
+ if individual.email_addresses:
+ print(f" Emails: {individual.email_addresses}")
+ if individual.phone_numbers:
+ print(f" Phones: {individual.phone_numbers}")
+
+ # Now run postprocess
+ print(f"\n=== Running postprocess ===")
+ scanner.postprocess(results, input_domains)
+
+ # Debug: Check what happened during postprocess
+ print(f"=== Postprocess completed ===")
+ print(f"Nodes created: {len(scanner.neo4j_conn.nodes_created)}")
+ print(f"Relationships created: {len(scanner.neo4j_conn.relationships_created)}")
+
+ # Should have created some nodes
+ assert len(scanner.neo4j_conn.nodes_created) > 0
+
+ # Should have created some relationships
+ assert len(scanner.neo4j_conn.relationships_created) > 0
+
+ # Check for domain node
+ domain_nodes = [
+ n for n in scanner.neo4j_conn.nodes_created if n["label"] == "domain"
+ ]
+ assert len(domain_nodes) > 0
+
+ # Check for individual nodes (should include JOHN DOE)
+ individual_nodes = [
+ n for n in scanner.neo4j_conn.nodes_created if n["label"] == "individual"
+ ]
+ assert len(individual_nodes) > 0
+
+ # Check that JOHN DOE is in the individual nodes
+ marc_nodes = [n for n in individual_nodes if "JOHN DOE" in n["value"]]
+ assert (
+ len(marc_nodes) > 0
+ ), "JOHN DOE should be in the individual nodes"
+
+ finally:
+ loop.close()
+
+
+def test_schemas(scanner):
+ """Test that the scanner has the expected schemas."""
+ input_schema = scanner.input_schema()
+ output_schema = scanner.output_schema()
+
+ assert input_schema is not None
+ assert output_schema is not None
diff --git a/flowsint-transforms/tests/test_data/data.json b/flowsint-transforms/tests/test_data/data.json
new file mode 100644
index 00000000..2322a9df
--- /dev/null
+++ b/flowsint-transforms/tests/test_data/data.json
@@ -0,0 +1,934 @@
+{
+ "status": 1,
+ "api_query": "whois_history",
+ "total_records_found": 16,
+ "whois_records": [
+ {
+ "num": 1,
+ "domain_name": "epios.com",
+ "query_time": "2016-08-21 03:27:59",
+ "create_date": "1998-09-01",
+ "update_date": "2016-08-02",
+ "expiry_date": "2017-08-31",
+ "domain_registrar": {
+ "iana_id": 48,
+ "registrar_name": "eNom, Inc.",
+ "whois_server": "whois.enom.com",
+ "website_url": "http://www.enom.com",
+ "registrar_status": "Accredited"
+ },
+ "registrant_contact": {
+ "full_name": "JOHN DOE",
+ "mailing_address": "123 JOHN STREET",
+ "city_name": "DOE CITY",
+ "state_name": "CA",
+ "zip_code": "12345",
+ "country_name": "United States",
+ "country_code": "US",
+ "email_address": "[email protected], [email protected]",
+ "phone_number": "+123456789"
+ },
+ "administrative_contact": {
+ "full_name": "JOHN DOE",
+ "mailing_address": "123 JOHN STREET",
+ "city_name": "DOE CITY",
+ "state_name": "CA",
+ "zip_code": "12345",
+ "country_name": "United States",
+ "country_code": "US",
+ "email_address": "[email protected], [email protected]",
+ "phone_number": "+123456789"
+ },
+ "technical_contact": {
+ "full_name": "JOHN DOE",
+ "mailing_address": "123 JOHN STREET",
+ "city_name": "DOE CITY",
+ "state_name": "CA",
+ "zip_code": "12345",
+ "country_name": "United States",
+ "country_code": "US",
+ "email_address": "[email protected], [email protected]",
+ "phone_number": "+123456789"
+ },
+ "name_servers": [
+ "ns1.server270.com",
+ "ns2.server270.com"
+ ],
+ "domain_status": [
+ "clientTransferProhibited"
+ ]
+ },
+ {
+ "num": 2,
+ "domain_name": "epios.com",
+ "query_time": "2017-10-13 22:27:51",
+ "create_date": "1998-09-01",
+ "update_date": "2016-08-02",
+ "expiry_date": "2019-08-31",
+ "domain_registrar": {
+ "iana_id": 48,
+ "registrar_name": "eNom, Inc.",
+ "whois_server": "whois.enom.com",
+ "website_url": "http://www.enom.com",
+ "registrar_status": "Accredited"
+ },
+ "registrant_contact": {
+ "full_name": "JOHN DOE",
+ "mailing_address": "123 JOHN STREET",
+ "city_name": "DOE CITY",
+ "state_name": "CA",
+ "zip_code": "12345",
+ "country_name": "United States",
+ "country_code": "US",
+ "email_address": "[email protected], [email protected]",
+ "phone_number": "+123456789"
+ },
+ "administrative_contact": {
+ "full_name": "JOHN DOE",
+ "mailing_address": "123 JOHN STREET",
+ "city_name": "DOE CITY",
+ "state_name": "CA",
+ "zip_code": "12345",
+ "country_name": "United States",
+ "country_code": "US",
+ "email_address": "[email protected], [email protected]",
+ "phone_number": "+123456789"
+ },
+ "technical_contact": {
+ "full_name": "JOHN DOE",
+ "mailing_address": "123 JOHN STREET",
+ "city_name": "DOE CITY",
+ "state_name": "CA",
+ "zip_code": "12345",
+ "country_name": "United States",
+ "country_code": "US",
+ "email_address": "[email protected], [email protected]",
+ "phone_number": "+123456789"
+ },
+ "name_servers": [
+ "ns1.server270.com",
+ "ns2.server270.com"
+ ],
+ "domain_status": [
+ "clientTransferProhibited"
+ ]
+ },
+ {
+ "num": 3,
+ "domain_name": "epios.com",
+ "query_time": "2023-06-02 21:14:04",
+ "create_date": "1998-09-01",
+ "update_date": "2022-08-01",
+ "expiry_date": "2023-08-31",
+ "domain_registrar": {
+ "iana_id": 48,
+ "registrar_name": "eNom, Inc.",
+ "whois_server": "whois.enom.com",
+ "website_url": "http://www.enom.com",
+ "registrar_status": "Accredited"
+ },
+ "registrant_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "CA",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "United States",
+ "country_code": "US",
+ "email_address": "https://tieredaccess.com/contact/2d4e326b-fc3d-4269-a998-b62ba07aee51",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "administrative_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "technical_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "name_servers": [
+ "dns1.name-services.com",
+ "dns2.name-services.com",
+ "dns3.name-services.com",
+ "dns4.name-services.com"
+ ],
+ "domain_status": [
+ "clientTransferProhibited"
+ ]
+ },
+ {
+ "num": 4,
+ "domain_name": "epios.com",
+ "query_time": "2023-08-09 19:21:55",
+ "create_date": "1998-09-01",
+ "update_date": "2022-08-01",
+ "expiry_date": "2023-08-31",
+ "domain_registrar": {
+ "iana_id": 48,
+ "registrar_name": "eNom, Inc.",
+ "whois_server": "whois.enom.com",
+ "website_url": "http://www.enom.com",
+ "registrar_status": "Accredited"
+ },
+ "registrant_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "CA",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "United States",
+ "country_code": "US",
+ "email_address": "https://tieredaccess.com/contact/e811856a-515d-486d-bfa3-2771ec4b8bb4",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "administrative_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "technical_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "name_servers": [
+ "dns1.name-services.com",
+ "dns2.name-services.com",
+ "dns3.name-services.com",
+ "dns4.name-services.com"
+ ],
+ "domain_status": [
+ "clientTransferProhibited"
+ ]
+ },
+ {
+ "num": 5,
+ "domain_name": "epios.com",
+ "query_time": "2023-09-10 05:34:39",
+ "create_date": "1998-09-01",
+ "update_date": "2023-08-16",
+ "expiry_date": "2024-08-31",
+ "domain_registrar": {
+ "iana_id": 48,
+ "registrar_name": "eNom, Inc.",
+ "whois_server": "whois.enom.com",
+ "website_url": "http://www.enom.com",
+ "registrar_status": "Accredited"
+ },
+ "registrant_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "CA",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "United States",
+ "country_code": "US",
+ "email_address": "https://tieredaccess.com/contact/e966535a-18b4-4fc2-ba62-6b773211ea7d",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "administrative_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "technical_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "name_servers": [
+ "dns1.name-services.com",
+ "dns2.name-services.com",
+ "dns3.name-services.com",
+ "dns4.name-services.com"
+ ],
+ "domain_status": [
+ "clientTransferProhibited"
+ ]
+ },
+ {
+ "num": 6,
+ "domain_name": "epios.com",
+ "query_time": "2023-11-26 06:52:14",
+ "create_date": "1998-09-01",
+ "update_date": "2023-08-16",
+ "expiry_date": "2024-08-31",
+ "domain_registrar": {
+ "iana_id": 48,
+ "registrar_name": "eNom, Inc.",
+ "whois_server": "whois.enom.com",
+ "website_url": "http://www.enom.com",
+ "registrar_status": "Accredited"
+ },
+ "registrant_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "CA",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "United States",
+ "country_code": "US",
+ "email_address": "https://tieredaccess.com/contact/85a6b638-dbc5-43ca-8e81-1564fc0e1718",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "administrative_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "technical_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "name_servers": [
+ "dns1.name-services.com",
+ "dns2.name-services.com",
+ "dns3.name-services.com",
+ "dns4.name-services.com"
+ ],
+ "domain_status": [
+ "clientTransferProhibited"
+ ]
+ },
+ {
+ "num": 7,
+ "domain_name": "epios.com",
+ "query_time": "2024-02-08 04:00:50",
+ "create_date": "1998-09-01",
+ "update_date": "2023-08-16",
+ "expiry_date": "2024-08-31",
+ "domain_registrar": {
+ "iana_id": 48,
+ "registrar_name": "eNom, Inc.",
+ "whois_server": "whois.enom.com",
+ "website_url": "http://www.enom.com",
+ "registrar_status": "Accredited"
+ },
+ "name_servers": [
+ "dns1.name-services.com",
+ "dns2.name-services.com",
+ "dns3.name-services.com",
+ "dns4.name-services.com"
+ ],
+ "domain_status": [
+ "clientTransferProhibited"
+ ]
+ },
+ {
+ "num": 8,
+ "domain_name": "epios.com",
+ "query_time": "2024-02-13 01:44:26",
+ "create_date": "1998-09-01",
+ "update_date": "2023-08-16",
+ "expiry_date": "2024-08-31",
+ "domain_registrar": {
+ "iana_id": 48,
+ "registrar_name": "eNom, Inc.",
+ "whois_server": "whois.enom.com",
+ "website_url": "http://www.enom.com",
+ "registrar_status": "Accredited"
+ },
+ "registrant_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "CA",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "United States",
+ "country_code": "US",
+ "email_address": "https://tieredaccess.com/contact/fd6e5334-dad3-4247-9970-ef471bc3d887",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "administrative_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "technical_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "name_servers": [
+ "dns1.name-services.com",
+ "dns2.name-services.com",
+ "dns3.name-services.com",
+ "dns4.name-services.com"
+ ],
+ "domain_status": [
+ "clientTransferProhibited"
+ ]
+ },
+ {
+ "num": 9,
+ "domain_name": "epios.com",
+ "query_time": "2024-02-22 07:49:19",
+ "create_date": "1998-09-01",
+ "update_date": "2023-08-16",
+ "expiry_date": "2024-08-31",
+ "domain_registrar": {
+ "iana_id": 48,
+ "registrar_name": "eNom, Inc.",
+ "whois_server": "whois.enom.com",
+ "website_url": "http://www.enom.com",
+ "registrar_status": "Accredited"
+ },
+ "registrant_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "CA",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "United States",
+ "country_code": "US",
+ "email_address": "https://tieredaccess.com/contact/7f676f3b-cac7-4336-9787-1295d039c7fb",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "administrative_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "technical_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "name_servers": [
+ "dns1.name-services.com",
+ "dns2.name-services.com",
+ "dns3.name-services.com",
+ "dns4.name-services.com"
+ ],
+ "domain_status": [
+ "clientTransferProhibited"
+ ]
+ },
+ {
+ "num": 10,
+ "domain_name": "epios.com",
+ "query_time": "2024-04-06 03:08:26",
+ "create_date": "1998-09-01",
+ "update_date": "2023-08-16",
+ "expiry_date": "2024-08-31",
+ "domain_registrar": {
+ "iana_id": 48,
+ "registrar_name": "eNom, Inc.",
+ "whois_server": "whois.enom.com",
+ "website_url": "http://www.enom.com",
+ "registrar_status": "Accredited"
+ },
+ "registrant_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "CA",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "United States",
+ "country_code": "US",
+ "email_address": "https://tieredaccess.com/contact/a5ff0f8e-eb63-4648-86c7-d7a25b822b25",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "administrative_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "technical_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "name_servers": [
+ "dns1.name-services.com",
+ "dns2.name-services.com",
+ "dns3.name-services.com",
+ "dns4.name-services.com"
+ ],
+ "domain_status": [
+ "clientTransferProhibited"
+ ]
+ },
+ {
+ "num": 11,
+ "domain_name": "epios.com",
+ "query_time": "2024-06-05 00:00:58",
+ "create_date": "1998-09-01",
+ "update_date": "2023-08-16",
+ "expiry_date": "2024-08-31",
+ "domain_registrar": {
+ "iana_id": 48,
+ "registrar_name": "eNom, Inc.",
+ "whois_server": "whois.enom.com",
+ "website_url": "http://www.enom.com",
+ "registrar_status": "Accredited"
+ },
+ "registrant_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "CA",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "United States",
+ "country_code": "US",
+ "email_address": "https://tieredaccess.com/contact/c24d86c9-870b-425e-9650-38052534f131",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "administrative_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "technical_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "name_servers": [
+ "dns1.name-services.com",
+ "dns2.name-services.com",
+ "dns3.name-services.com",
+ "dns4.name-services.com"
+ ],
+ "domain_status": [
+ "clientTransferProhibited"
+ ]
+ },
+ {
+ "num": 12,
+ "domain_name": "epios.com",
+ "query_time": "2024-06-13 02:54:54",
+ "create_date": "1998-09-01",
+ "update_date": "2023-08-16",
+ "expiry_date": "2024-08-31",
+ "domain_registrar": {
+ "iana_id": 48,
+ "registrar_name": "eNom, Inc.",
+ "whois_server": "whois.enom.com",
+ "website_url": "http://www.enom.com",
+ "registrar_status": "Accredited"
+ },
+ "registrant_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "CA",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "United States",
+ "country_code": "US",
+ "email_address": "https://tieredaccess.com/contact/2d35cecc-0e79-4a18-b9d7-9cb7029442f1",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "administrative_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "technical_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "name_servers": [
+ "dns1.name-services.com",
+ "dns2.name-services.com",
+ "dns3.name-services.com",
+ "dns4.name-services.com"
+ ],
+ "domain_status": [
+ "clientTransferProhibited"
+ ]
+ },
+ {
+ "num": 13,
+ "domain_name": "epios.com",
+ "query_time": "2024-08-05 15:03:48",
+ "create_date": "1998-09-01",
+ "update_date": "2024-08-01",
+ "expiry_date": "2025-08-31",
+ "domain_registrar": {
+ "iana_id": 48,
+ "registrar_name": "eNom, Inc.",
+ "whois_server": "whois.enom.com",
+ "website_url": "http://www.enom.com",
+ "registrar_status": "Accredited"
+ },
+ "registrant_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "CA",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "United States",
+ "country_code": "US",
+ "email_address": "https://tieredaccess.com/contact/7654cb74-0744-475d-bba1-d5bb6f91b5aa",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "administrative_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "technical_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "name_servers": [
+ "dns1.name-services.com",
+ "dns2.name-services.com",
+ "dns3.name-services.com",
+ "dns4.name-services.com"
+ ],
+ "domain_status": [
+ "clientTransferProhibited",
+ "renewPeriod"
+ ]
+ },
+ {
+ "num": 14,
+ "domain_name": "epios.com",
+ "query_time": "2024-10-01 09:11:19",
+ "create_date": "1998-09-01",
+ "update_date": "2024-08-01",
+ "expiry_date": "2025-08-31",
+ "domain_registrar": {
+ "iana_id": 48,
+ "registrar_name": "eNom, Inc.",
+ "whois_server": "whois.enom.com",
+ "website_url": "http://www.enom.com",
+ "registrar_status": "Accredited"
+ },
+ "registrant_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "CA",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "United States",
+ "country_code": "US",
+ "email_address": "https://tieredaccess.com/contact/db7c6dea-a389-4e83-ad2c-98c7c2f8cc18",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "administrative_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "technical_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "name_servers": [
+ "dns1.name-services.com",
+ "dns2.name-services.com",
+ "dns3.name-services.com",
+ "dns4.name-services.com"
+ ],
+ "domain_status": [
+ "clientTransferProhibited"
+ ]
+ },
+ {
+ "num": 15,
+ "domain_name": "epios.com",
+ "query_time": "2024-11-24 18:45:22",
+ "create_date": "1998-09-01",
+ "update_date": "2024-11-24",
+ "expiry_date": "2025-08-31",
+ "domain_registrar": {
+ "iana_id": 48,
+ "registrar_name": "eNom, Inc.",
+ "whois_server": "whois.enom.com",
+ "website_url": "http://www.enom.com",
+ "registrar_status": "Accredited"
+ },
+ "registrant_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "CA",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "United States",
+ "country_code": "US",
+ "email_address": "https://tieredaccess.com/contact/a5e74127-d4d4-450c-a314-1ebbe1254e20",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "administrative_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "technical_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "redacted for privacy",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "name_servers": [
+ "dns1.name-services.com",
+ "dns2.name-services.com",
+ "dns3.name-services.com",
+ "dns4.name-services.com"
+ ],
+ "domain_status": [
+ "PENDINGTRANSFER"
+ ]
+ },
+ {
+ "num": 16,
+ "domain_name": "epios.com",
+ "query_time": "2025-08-24 08:10:49",
+ "create_date": "1998-09-01",
+ "update_date": "2024-11-30",
+ "expiry_date": "2026-08-31",
+ "domain_registrar": {
+ "iana_id": 433,
+ "registrar_name": "OVH sas",
+ "whois_server": "whois.ovh.com",
+ "website_url": "http://www.ovh.com",
+ "registrar_status": "Accredited"
+ },
+ "registrant_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "Epieos",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "France",
+ "country_code": "FR",
+ "email_address": "REDACTED FOR PRIVACY - Send message to contact by visiting https://www.ovhcloud.com/en/lp/request-ov",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "administrative_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "REDACTED FOR PRIVACY - Send message to contact by visiting https://www.ovhcloud.com/en/lp/request-ov",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "technical_contact": {
+ "full_name": "REDACTED FOR PRIVACY",
+ "company_name": "REDACTED FOR PRIVACY",
+ "mailing_address": "REDACTED FOR PRIVACY",
+ "city_name": "REDACTED FOR PRIVACY",
+ "state_name": "REDACTED FOR PRIVACY",
+ "zip_code": "REDACTED FOR PRIVACY",
+ "country_name": "REDACTED FOR PRIVACY",
+ "email_address": "REDACTED FOR PRIVACY - Send message to contact by visiting https://www.ovhcloud.com/en/lp/request-ov",
+ "phone_number": "REDACTED FOR PRIVACY",
+ "fax_number": "REDACTED FOR PRIVACY"
+ },
+ "name_servers": [
+ "dns200.anycast.me",
+ "ns200.anycast.me"
+ ],
+ "domain_status": [
+ "clientDeleteProhibited",
+ "clientTransferProhibited"
+ ]
+ }
+ ],
+ "api_execution_time": 0.01
+}
\ No newline at end of file