Skip to content

A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.

feat: filters

dextmorgn49c455c47780
diff --git a/.env.example b/.env.example
index a66fd592..fe25c03b 100644
--- a/.env.example
+++ b/.env.example
@@ -1,6 +1,7 @@
NODE_ENV=production
NEXT_PUBLIC_AUTH_REDIRECT=http://app.flowsint.localhost/auth/callback
HIBP_API_KEY=
+ETHERSCAN_API_KEY=
NEXT_PUBLIC_FLOWSINT_API=https://api.flowsint.localhost/api
NEXT_PUBLIC_DOCKER_FLOWSINT_API=http://flowsint-api:5000/api
AUTH_SECRET="superscretchangeitplz"
diff --git a/flowsint-api/app/api/routes/transforms.py b/flowsint-api/app/api/routes/transforms.py
index d44d3caf..56b55c29 100644
--- a/flowsint-api/app/api/routes/transforms.py
+++ b/flowsint-api/app/api/routes/transforms.py
@@ -39,9 +39,11 @@ def get_transforms(
# current_user: Profile = Depends(get_current_user),
):
if category is not None and category != "undefined":
- transforms = TransformRegistry.list_by_input_type(category)
+ transforms = TransformRegistry.list_by_input_type(
+ category, exclude=["n8n_connector"]
+ )
else:
- transforms = TransformRegistry.list()
+ transforms = TransformRegistry.list(exclude=["n8n_conector"])
return transforms
diff --git a/flowsint-api/app/api/routes/types.py b/flowsint-api/app/api/routes/types.py
index e98e5f02..3f5dab46 100644
--- a/flowsint-api/app/api/routes/types.py
+++ b/flowsint-api/app/api/routes/types.py
@@ -1,5 +1,5 @@
from typing import Any, Dict, Optional, Type
-from uuid import UUID, uuid4
+from uuid import uuid4
from fastapi import APIRouter
from pydantic import BaseModel, TypeAdapter
from flowsint_types import (
diff --git a/flowsint-app/src/renderer/src/components/flows/flow-sheet.tsx b/flowsint-app/src/renderer/src/components/flows/flow-sheet.tsx
index eb411bee..0a9f1511 100644
--- a/flowsint-app/src/renderer/src/components/flows/flow-sheet.tsx
+++ b/flowsint-app/src/renderer/src/components/flows/flow-sheet.tsx
@@ -13,11 +13,11 @@ import { TooltipProvider, Tooltip, TooltipContent, TooltipTrigger } from "@/comp
import { useNodesDisplaySettings } from "@/stores/node-display-settings"
import { Scanner } from "@/types/transform"
import { useQuery } from "@tanstack/react-query"
-import { transformService } from "@/api/transfrom-service"
import { Input } from "../ui/input"
import { Alert, AlertDescription } from "../ui/alert"
import { categoryColors } from "./scanner-data"
import { useIcon } from "@/hooks/use-icon"
+import { flowService } from "@/api/flow-service"
const FlowSheet = ({ onLayout }: { onLayout: () => void }) => {
const openFlowSheet = useFlowStore(state => state.openFlowSheet)
@@ -29,7 +29,7 @@ const FlowSheet = ({ onLayout }: { onLayout: () => void }) => {
const { data: materials, isLoading, error } = useQuery({
queryKey: ["raw_material", selectedNode?.data.outputs.type],
enabled: !!selectedNode?.data.outputs.type,
- queryFn: () => transformService.getFlowsRawMaterialForType(selectedNode?.data.outputs.type.toLowerCase() || ""),
+ queryFn: () => flowService.getRawMaterialForType(selectedNode?.data.outputs.type.toLowerCase() || ""),
})
const [searchTerm, setSearchTerm] = useState<string>("")
diff --git a/flowsint-app/src/renderer/src/components/flows/new-flow.tsx b/flowsint-app/src/renderer/src/components/flows/new-flow.tsx
index c9213f79..96d10148 100644
--- a/flowsint-app/src/renderer/src/components/flows/new-flow.tsx
+++ b/flowsint-app/src/renderer/src/components/flows/new-flow.tsx
@@ -1,8 +1,8 @@
-import { transformService } from '@/api/transfrom-service'
import { useCallback, ReactNode, cloneElement, isValidElement } from 'react'
import { toast } from 'sonner'
import { useNavigate } from '@tanstack/react-router'
import { useFlowStore } from '@/stores/flow-store'
+import { flowService } from '@/api/flow-service'
const NewFlow = ({ children }: { children: ReactNode }) => {
const navigate = useNavigate()
@@ -15,7 +15,7 @@ const NewFlow = ({ children }: { children: ReactNode }) => {
// Reset nodes and edges
setNodes([])
setEdges([])
- const response = await transformService.create(JSON.stringify({
+ const response = await flowService.create(JSON.stringify({
name: "New flow",
description: "A new example flow.",
category: [],
@@ -27,7 +27,7 @@ const NewFlow = ({ children }: { children: ReactNode }) => {
{
loading: 'Creating transform...',
success: 'Transform created successfully.',
- error: 'Failed to create transform.'
+ error: 'Failed to create flow.'
}
)
diff --git a/flowsint-app/src/renderer/src/components/graphs/filters.tsx b/flowsint-app/src/renderer/src/components/graphs/filters.tsx
new file mode 100644
index 00000000..41828b1d
--- /dev/null
+++ b/flowsint-app/src/renderer/src/components/graphs/filters.tsx
@@ -0,0 +1,70 @@
+import { useCallback } from 'react';
+import { useGraphStore } from '@/stores/graph-store';
+import { useActionItems } from '@/hooks/use-action-items';
+import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuLabel, DropdownMenuSeparator, DropdownMenuTrigger } from '../ui/dropdown-menu';
+import { Button } from '../ui/button';
+import { XIcon } from 'lucide-react';
+import { cn, getAllNodeTypes } from '@/lib/utils';
+import { Badge } from '../ui/badge';
+
+const Filters = ({ children }: { children: React.ReactNode }) => {
+ const filters = useGraphStore(s => s.filters);
+ const setFilters = useGraphStore(s => s.setFilters);
+ const { actionItems } = useActionItems()
+
+ const clearFilters = useCallback(() => {
+ setFilters(null)
+ }, [setFilters, filters])
+
+ const toggleFilter = useCallback((type: string | null) => {
+ if (type === null) {
+ clearFilters()
+ } else {
+ const currentTypeFilters = filters || []
+ const isChecked = currentTypeFilters.includes(type)
+ const newTypeFilters = isChecked ? currentTypeFilters.filter((t: string) => t !== type) : [...currentTypeFilters, type.toLowerCase()]
+ setFilters(newTypeFilters)
+ }
+ }, [setFilters, filters, clearFilters])
+
+ return (
+ <DropdownMenu>
+ <div>
+ <DropdownMenuTrigger asChild>
+ <div>
+ {children}
+ </div>
+ </DropdownMenuTrigger>
+ </div>
+ <DropdownMenuContent className="w-48 max-h-[50vh] space-y-1 overflow-y-auto">
+ <DropdownMenuLabel>
+ {filters?.length ? (
+ <Badge variant={"outline"} className="flex items-center gap-1 pr-1">
+ {filters?.length} filter(s)
+ <Button size={"icon"} variant={"ghost"} className="h-5 w-5 rounded-full" onClick={clearFilters}>
+ <XIcon className="h-3 w-3" />
+ </Button>
+ </Badge>
+ ) : (
+ "filters"
+ )}
+ </DropdownMenuLabel>
+ <DropdownMenuSeparator />
+ <DropdownMenuItem className={cn(filters == null && "bg-primary")} onClick={() => toggleFilter(null)}>
+ All
+ </DropdownMenuItem>
+ {getAllNodeTypes(actionItems || []).map((type) => (
+ <DropdownMenuItem
+ className={cn(filters?.includes(type.toLowerCase()) && "bg-primary/30")}
+ key={type}
+ onClick={() => toggleFilter(type)}
+ >
+ {type || "unknown"}
+ </DropdownMenuItem>
+ ))}
+ </DropdownMenuContent>
+ </DropdownMenu>
+ );
+};
+
+export default Filters;
\ No newline at end of file
diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx
index a87d3433..dab6878a 100644
--- a/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx
@@ -4,9 +4,9 @@ import GraphViewer from './graph-viewer'
import ContextMenu from './context-menu'
const GraphMain = () => {
- const nodes = useGraphStore(s => s.nodes)
- const edges = useGraphStore(s => s.edges)
- const currentNode = useGraphStore(s => s.currentNode)
+ const filteredNodes = useGraphStore(s => s.filteredNodes)
+ const filteredEdges = useGraphStore(s => s.filteredEdges)
+ // const currentNode = useGraphStore(s => s.currentNode)
const toggleNodeSelection = useGraphStore(s => s.toggleNodeSelection)
const clearSelectedNodes = useGraphStore(s => s.clearSelectedNodes)
@@ -14,12 +14,13 @@ const GraphMain = () => {
const containerRef = useRef<HTMLDivElement>(null)
const [menu, setMenu] = React.useState<any>(null)
- // Handle current node centering
- useEffect(() => {
- if (!currentNode || !graphRef.current) return
- graphRef.current.centerAt(currentNode.x, currentNode.y, 500)
- graphRef.current.zoom(5, 500)
- }, [currentNode])
+
+ // // Handle current node centering
+ // useEffect(() => {
+ // if (!currentNode || !graphRef.current) return
+ // graphRef.current.centerAt(currentNode.position.x, currentNode.position.x, 500)
+ // graphRef.current.zoom(5, 500)
+ // }, [currentNode])
const handleNodeClick = useCallback((node: any) => {
toggleNodeSelection(node, false)
@@ -59,8 +60,8 @@ const GraphMain = () => {
return (
<div ref={containerRef} className="relative h-full w-full bg-background">
<GraphViewer
- nodes={nodes}
- edges={edges}
+ nodes={filteredNodes}
+ edges={filteredEdges}
onNodeClick={handleNodeClick}
onNodeRightClick={onNodeContextMenu}
onBackgroundClick={handleBackgroundClick}
diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx
index 7838f010..945be343 100644
--- a/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx
@@ -1,5 +1,5 @@
import { useLoaderData } from '@tanstack/react-router'
-import { useEffect, useRef, memo, useState, lazy, Suspense } from 'react'
+import { useEffect, useRef, memo, useState, lazy, Suspense, useMemo } from 'react'
import { useGraphStore, type GraphNode, type GraphEdge } from '@/stores/graph-store'
import { Toolbar } from './toolbar'
import { cn } from '@/lib/utils'
@@ -21,7 +21,7 @@ const RelationshipsTable = lazy(() => import('@/components/table/relationships-v
const Graph = lazy(() => import('./graph'))
// const Wall = lazy(() => import('./wall/wall'))
-const NODE_COUNT_THRESHOLD = 4500;
+const NODE_COUNT_THRESHOLD = 1500;
// Separate component for the drag overlay
const DragOverlay = memo(({ isDragging }: { isDragging: boolean }) => (
@@ -56,6 +56,48 @@ const GraphPanel = ({ graphData, isLoading }: GraphPanelProps) => {
})
const [isDraggingOver, setIsDraggingOver] = useState(false)
+ // Determine which graph component to use based on node count with debouncing
+ const [stableGraphType, setStableGraphType] = useState<'cosmograph' | 'force-graph' | null>(null)
+
+ const shouldUseCosmograph = useMemo(() => {
+ if (!nodes || nodes.length === 0) return false
+ return nodes.length > NODE_COUNT_THRESHOLD
+ }, [nodes?.length])
+
+ // Debounce graph type changes to prevent rapid switching
+ useEffect(() => {
+ if (!nodes || nodes.length === 0) {
+ setStableGraphType(null)
+ return
+ }
+
+ const newType = shouldUseCosmograph ? 'cosmograph' : 'force-graph'
+
+ // Only update if the type actually changed
+ if (stableGraphType !== newType) {
+ const timer = setTimeout(() => {
+ setStableGraphType(newType)
+ }, 500) // 500ms delay to prevent rapid switching
+
+ return () => clearTimeout(timer)
+ }
+
+ // Return empty cleanup function for the case when type hasn't changed
+ return () => { }
+ }, [shouldUseCosmograph, stableGraphType, nodes?.length])
+
+ const graphComponentKey = stableGraphType || 'loading'
+
+ // Memoize the graph component to prevent unnecessary re-renders
+ const graphComponent = useMemo(() => {
+ if (stableGraphType === 'cosmograph') {
+ return <Graph />
+ } else if (stableGraphType === 'force-graph') {
+ return <GraphMain />
+ }
+ return null
+ }, [stableGraphType])
+
useEffect(() => {
if (graphData?.nds && graphData?.rls) {
updateGraphData(graphData.nds, graphData.rls)
@@ -128,21 +170,28 @@ const GraphPanel = ({ graphData, isLoading }: GraphPanelProps) => {
</div>
</div>
}>
- {nodes?.length > NODE_COUNT_THRESHOLD ? (
- <>{view === "table" && <NodesTable />}
- {["force", "hierarchy"].includes(view) && <Graph />}
- {view === "map" && <MapPanel />}
- {view === "relationships" && <RelationshipsTable />}
- </>
- ) : (
- <>
- {/* {view === "hierarchy" && <Wall isLoading={false} isRefetching={false} />} */}
- {/* {view === "force" && <GraphMain />} */}
- {["force", "hierarchy"].includes(view) && <GraphMain />}
+ {/* Only render graph components when we have actual data */}
+ {nodes && nodes.length > 0 && stableGraphType ? (
+ <div key={graphComponentKey} className="h-full w-full flex relative outline-2 outline-transparent bg-background"
+ >
{view === "table" && <NodesTable />}
{view === "map" && <MapPanel />}
{view === "relationships" && <RelationshipsTable />}
- </>
+ {["force", "hierarchy"].includes(view) && graphComponent}
+ </div>
+ ) : (
+ // Show loading state while waiting for data or graph type to stabilize
+ <div className="h-full w-full flex items-center justify-center">
+ <div className="text-center flex flex-col items-center gap-2">
+ <Loader />
+ <span className="text-muted-foreground">
+ {!nodes || nodes.length === 0
+ ? 'Loading data...'
+ : 'Preparing data...'
+ }
+ </span>
+ </div>
+ </div>
)}
</Suspense>
<DragOverlay isDragging={isDraggingOver} />
diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx
index 86c02e42..7714afca 100644
--- a/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx
@@ -9,6 +9,12 @@ import { Button } from '../ui/button';
import { useTheme } from '@/components/theme-provider'
import { GRAPH_COLORS } from '../flows/scanner-data';
+function truncateText(text: string, limit: number = 16) {
+ if (text.length <= limit)
+ return text
+ return text.substring(0, limit) + "..."
+}
+
interface GraphViewerProps {
nodes: GraphNode[];
edges: GraphEdge[];
@@ -388,7 +394,7 @@ const GraphViewer: React.FC<GraphViewerProps> = ({
// Optimized label rendering
if (showLabels && globalScale > 3) {
- const label = node.nodeLabel || node.label || node.id;
+ const label = truncateText(node.nodeLabel || node.label || node.id, 58);
if (label) {
// Only show labels for highlighted nodes when there's any highlighting
// or show all labels when there's no highlighting
diff --git a/flowsint-app/src/renderer/src/components/graphs/graph.tsx b/flowsint-app/src/renderer/src/components/graphs/graph.tsx
index 6d5c014d..11da00da 100644
--- a/flowsint-app/src/renderer/src/components/graphs/graph.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/graph.tsx
@@ -236,8 +236,8 @@ GraphProvider.displayName = "GraphProvider"
// Composant principal avec optimisations maximales
const Graph = memo(() => {
- const nodes = useGraphStore(s => s.nodes)
- const edges = useGraphStore(s => s.edges)
+ const nodes = useGraphStore(s => s.filteredNodes)
+ const edges = useGraphStore(s => s.filteredEdges)
if (!nodes.length) {
return <EmptyState />
diff --git a/flowsint-app/src/renderer/src/components/graphs/nodes-panel.tsx b/flowsint-app/src/renderer/src/components/graphs/nodes-panel.tsx
index 57857767..b3964ffc 100644
--- a/flowsint-app/src/renderer/src/components/graphs/nodes-panel.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/nodes-panel.tsx
@@ -5,10 +5,10 @@ import { useGraphStore } from "@/stores/graph-store"
import { Button } from "@/components/ui/button"
import { Badge } from "@/components/ui/badge"
import { TypeBadge } from "@/components/type-badge"
-import { Search, FilterIcon, XIcon } from "lucide-react"
+import { Search, FunnelPlus, XIcon } from "lucide-react"
import { Input } from "@/components/ui/input"
import { useActionItems } from "@/hooks/use-action-items"
-import { cn } from "@/lib/utils"
+import { cn, getAllNodeTypes } from "@/lib/utils"
import {
DropdownMenu,
DropdownMenuContent,
@@ -21,22 +21,6 @@ import type { GraphNode } from "@/stores/graph-store"
import { Checkbox } from "@/components/ui/checkbox"
const ITEM_HEIGHT = 40
-
-const getAllNodeTypes = (actionItems: any[]) => {
- const types: string[] = []
- actionItems.forEach(item => {
- if (item.children) {
- item.children.forEach(child => {
- if (child.type && !types.includes(child.type)) {
- types.push(child.type)
- }
- })
- } else if (item.type && !types.includes(item.type)) {
- types.push(item.type)
- }
- })
- return types.sort()
-}
// Mémoiser le composant NodeRenderer
const NodeRenderer = memo(
({
@@ -106,7 +90,8 @@ const NodesPanel = memo(({ nodes, isLoading }: { nodes: GraphNode[]; isLoading?:
const setSelectedNodes = useGraphStore((state) => state.setSelectedNodes)
const selectedNodes = useGraphStore((state) => state.selectedNodes || [])
const [searchQuery, setSearchQuery] = useState<string>("")
- const [filters, setFilters] = useState<null | string[]>(null)
+ const filters = useGraphStore(s => s.filters);
+ const setFilters = useGraphStore(s => s.setFilters);
const { actionItems } = useActionItems()
// Ref pour le conteneur parent du virtualizer
@@ -139,13 +124,12 @@ const NodesPanel = memo(({ nodes, isLoading }: { nodes: GraphNode[]; isLoading?:
if (filter === null) {
setFilters(null)
} else {
- setFilters((prev) => {
- if (prev === null) return [filter]
- const next = prev.includes(filter) ? prev.filter((f) => f !== filter) : [...prev, filter]
- return next.length === 0 ? null : next
- })
+ const currentFilters = filters || []
+ const isChecked = currentFilters.includes(filter)
+ const newFilters = isChecked ? currentFilters.filter((f) => f !== filter) : [...currentFilters, filter.toLowerCase()]
+ setFilters(newFilters.length === 0 ? null : newFilters)
}
- }, [])
+ }, [filters, setFilters])
const handleSearchChange = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {
setSearchQuery(e.target.value)
@@ -216,7 +200,7 @@ const NodesPanel = memo(({ nodes, isLoading }: { nodes: GraphNode[]; isLoading?:
<DropdownMenuTrigger asChild>
<div>
<Button variant={"outline"} className="h-7 w-8 relative border-border border" size={"icon"}>
- <FilterIcon className={cn("opacity-60 h-3 w-3", filters && "opacity-100")} />
+ <FunnelPlus className={cn("opacity-60 h-3 w-3", filters && "opacity-100")} />
{filters && (
<Badge className="absolute -top-2 -right-1 text-xs rounded-full h-4 w-4 text-white" variant={"default"}>
{filters.length}
diff --git a/flowsint-app/src/renderer/src/components/graphs/toolbar.tsx b/flowsint-app/src/renderer/src/components/graphs/toolbar.tsx
index f4dd40c6..ceab9af4 100644
--- a/flowsint-app/src/renderer/src/components/graphs/toolbar.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/toolbar.tsx
@@ -12,17 +12,20 @@ import {
List,
SlidersHorizontal,
GitFork,
- ArrowRightLeft
+ ArrowRightLeft,
+ FunnelPlus
} from "lucide-react"
import { memo, useCallback } from "react"
import { toast } from "sonner"
import { cn } from "@/lib/utils"
import ForceControls from './force-controls'
+import Filters from "./filters"
+import { useGraphStore } from "@/stores/graph-store"
const isMac = navigator.platform.toUpperCase().indexOf('MAC') >= 0
// Tooltip wrapper component to avoid repetition
-const ToolbarButton = memo(function ToolbarButton({
+export const ToolbarButton = memo(function ToolbarButton({
icon,
tooltip,
onClick,
@@ -63,6 +66,7 @@ export const Toolbar = memo(function Toolbar({ isLoading }: { isLoading: boolean
const zoomOut = useGraphControls((s) => s.zoomOut);
const onLayout = useGraphControls((s) => s.onLayout);
const refetchGraph = useGraphControls((s) => s.refetchGraph)
+ const filters = useGraphStore(s => s.filters)
const handleRefresh = useCallback(() => {
try {
@@ -97,7 +101,7 @@ export const Toolbar = memo(function Toolbar({ isLoading }: { isLoading: boolean
return (
<div className="flex justify-start gap-2 items-center">
<TooltipProvider>
- < ToolbarButton
+ <ToolbarButton
icon={<ZoomIn className="h-4 w-4 opacity-70" />}
tooltip="Zoom In"
onClick={zoomIn}
@@ -119,26 +123,32 @@ export const Toolbar = memo(function Toolbar({ isLoading }: { isLoading: boolean
<ToolbarButton
icon={<GitFork className="h-4 w-4 opacity-70 rotate-180" />}
tooltip={`Hierarchy (${isMac ? '⌘' : 'ctrl'}+Y)`}
+ disabled={["hierarchy"].includes(view)}
onClick={handleDagreLayoutTB}
/>
+
<ToolbarButton
icon={<Waypoints className="h-4 w-4 opacity-70" />}
+ disabled={["force"].includes(view)}
tooltip={"Graph view"}
onClick={handleForceLayout}
/>
<ToolbarButton
icon={<List className="h-4 w-4 opacity-70" />}
tooltip={"Table view"}
+ disabled={["table"].includes(view)}
onClick={handleTableLayout}
/>
<ToolbarButton
icon={<ArrowRightLeft className="h-4 w-4 opacity-70" />}
tooltip={"Relationships view"}
+ disabled={["relationships"].includes(view)}
onClick={handleRelationshipsLayout}
/>
<ToolbarButton
icon={<MapPin className="h-4 w-4 opacity-70" />}
tooltip={"Map view"}
+ disabled={["map"].includes(view)}
onClick={handleMapLayout}
/>
<ToolbarButton
@@ -149,11 +159,19 @@ export const Toolbar = memo(function Toolbar({ isLoading }: { isLoading: boolean
/>
<ForceControls>
<ToolbarButton
- disabled={isLoading}
+ disabled={isLoading || !["force", "hierarchy"].includes(view)}
icon={<SlidersHorizontal className={cn("h-4 w-4 opacity-70")} />}
tooltip="Settings"
/>
</ForceControls>
+ <Filters>
+ <ToolbarButton
+ disabled={isLoading}
+ icon={<FunnelPlus className={cn("h-4 w-4 opacity-70")} />}
+ tooltip="Filters"
+ badge={filters && filters?.length > 0 ? filters?.length : null}
+ />
+ </Filters>
</TooltipProvider>
</div>
)
diff --git a/flowsint-app/src/renderer/src/components/graphs/wall/custom/custom-edge.tsx b/flowsint-app/src/renderer/src/components/graphs/wall/custom/custom-edge.tsx
deleted file mode 100644
index f2c24738..00000000
--- a/flowsint-app/src/renderer/src/components/graphs/wall/custom/custom-edge.tsx
+++ /dev/null
@@ -1,18 +0,0 @@
-import { CustomEdge } from "@/components/xyflow/custom-edge";
-import { memo } from "react";
-import { EdgeProps } from "@xyflow/react";
-
-const CustomEdgeLabel = memo((props: EdgeProps) => {
- return (
- <CustomEdge {...props}>
- <span className="block text-[9px] opacity-70 uppercase">
- {props.label}
- </span>
- </CustomEdge>
- );
-});
-
-// Add display name for debugging
-CustomEdgeLabel.displayName = 'CustomEdgeLabel';
-
-export default CustomEdgeLabel;
\ No newline at end of file
diff --git a/flowsint-app/src/renderer/src/components/graphs/wall/custom/custom-node.tsx b/flowsint-app/src/renderer/src/components/graphs/wall/custom/custom-node.tsx
deleted file mode 100644
index c86d9761..00000000
--- a/flowsint-app/src/renderer/src/components/graphs/wall/custom/custom-node.tsx
+++ /dev/null
@@ -1,56 +0,0 @@
-import { memo } from "react";
-import { NodeProps, Handle, Position } from "@xyflow/react";
-import { useNodesDisplaySettings } from "@/stores/node-display-settings";
-import { useGraphStore } from "@/stores/graph-store";
-import { useIcon } from "@/hooks/use-icon";
-
-const size = 32;
-
-
-export const CustomNode = memo(({ data, selected, id }: NodeProps) => {
- const isCurrent = useGraphStore(state => state.isCurrent);
- const color = useNodesDisplaySettings(state => state.colors[data.type as keyof typeof state.colors]);
- const IconComponent = useIcon(data.type as string, data?.src as string | null);
-
-
- return (
- <div className="xy-node -z-[5000] relative p-1 flex flex-col justify-center items-center !rounded-full h-20">
- <Handle type="target" position={Position.Top} id="a" className="opacity-0 group-hover:opacity-100" />
- <Handle type="target" position={Position.Right} id="b" className="opacity-0 group-hover:opacity-100" />
- <Handle type="source" position={Position.Bottom} id="c" className="opacity-0 group-hover:opacity-100" />
- <Handle type="source" position={Position.Left} id="d" className="opacity-0 group-hover:opacity-100" />
- <div style={{ width: size, height: size, borderColor: (selected || isCurrent(id)) ? "#f97316" : color }} className="!rounded-full gap-0 flex items-center justify-center border-primary/70 p-1 overflow-hidden">
- <IconComponent
- size={size}
- style={{
- pointerEvents: "none",
- opacity: selected ? 1 : 0.85,
- display: "block",
- margin: "0 auto",
- }}
- />
- </div>
- <div
- className="text-nowrap text-center max-w-[160px] text-ellipsis truncate"
- style={{
- color: (selected || isCurrent(id)) ? "#f97316" : undefined,
- marginTop: 2,
- pointerEvents: "none",
- }}
- >
- {data?.label as string ?? "N/A"}
- </div>
- <div
- className="text-nowrap text-center opacity-70"
- style={{
- fontSize: 10,
- color: selected ? "#f97316" : undefined,
- marginTop: 1,
- pointerEvents: "none",
- }}
- >
- {data?.type as string ?? "N/A"}
- </div>
- </div>
- );
-});
diff --git a/flowsint-app/src/renderer/src/components/graphs/wall/wall.tsx b/flowsint-app/src/renderer/src/components/graphs/wall/wall.tsx
deleted file mode 100644
index b9b94b2d..00000000
--- a/flowsint-app/src/renderer/src/components/graphs/wall/wall.tsx
+++ /dev/null
@@ -1,331 +0,0 @@
-
-import type React from "react"
-import { useState, useCallback, useRef, useEffect, memo, useMemo } from "react"
-import {
- ReactFlow,
- ReactFlowProvider,
- // Background,
- type Node,
- useReactFlow,
- type NodeMouseHandler,
- type ColorMode,
- MiniMap,
- // BackgroundVariant,
- EdgeTypes,
- MarkerType,
- ConnectionMode,
-} from "@xyflow/react"
-import "@xyflow/react/dist/style.css"
-import { getDagreLayoutedElements, getForceLayoutedElements } from "@/lib/utils"
-import { useTheme } from "@/components/theme-provider"
-import { useGraphStore, type GraphNode, type GraphEdge } from "@/stores/graph-store"
-import { CustomNode } from "./custom/custom-node"
-// import FloatingEdge from "./custom/floating-edge"
-// import FloatingConnectionLine from "./custom/connection-line"
-import GraphLoader from "../graph-loader"
-import { useGraphControls } from "@/stores/graph-controls-store"
-import { useNodesDisplaySettings } from "@/stores/node-display-settings"
-import ContextMenu from '../context-menu';
-import { useLayoutStore } from "@/stores/layout-store"
-import CustomEdge from "./custom/custom-edge"
-
-const nodeTypes = {
- custom: CustomNode,
-};
-
-const edgeTypes = {
- custom: CustomEdge,
-};
-
-const Wall = memo(({ theme, edges }: { theme: ColorMode, edges: GraphEdge[] }) => {
- const { fitView, zoomIn, zoomOut, setCenter } = useReactFlow()
- const reactFlowWrapper = useRef<HTMLDivElement>(null)
- const colors = useNodesDisplaySettings(state => state.colors)
- const [reactFlowInstance, setReactFlowInstance] = useState<any>(null)
- const setActions = useGraphControls(state => state.setActions)
- // Memoize store selectors
- const nodes = useGraphStore(state => state.nodes)
- const toggleNodeSelection = useGraphStore(state => state.toggleNodeSelection)
- const currentNode = useGraphStore(state => state.currentNode)
- const setCurrentNode = useGraphStore(state => state.setCurrentNode)
- const clearSelectedNodes = useGraphStore(state => state.clearSelectedNodes)
- const setNodes = useGraphStore(state => state.setNodes)
- const setEdges = useGraphStore(state => state.setEdges)
- const addNode = useGraphStore(state => state.addNode)
- const onNodesChange = useGraphStore(state => state.onNodesChange)
- const onEdgesChange = useGraphStore(state => state.onEdgesChange)
- const onConnect = useGraphStore(state => state.onConnect)
- const setOpenNodeEditorModal = useGraphStore(state => state.setOpenNodeEditorModal)
- const setActiveTab = useLayoutStore((s) => s.setActiveTab)
-
- const [menu, setMenu] = useState<{
- node: GraphNode;
- top?: number;
- left?: number;
- right?: number;
- bottom?: number;
- rawTop?: number;
- rawLeft?: number;
- wrapperWidth: number;
- wrapperHeight: number;
- setMenu: (menu: any | null) => void;
- } | null>(null);
-
- // Memoize filtered nodes and edges
- const visibleNodes = useMemo(() => nodes.filter(n => !n.hidden), [nodes])
- const visibleEdges = useMemo(() => edges.filter(e => !e.hidden), [edges])
-
- // Center view on selected node
- useEffect(() => {
- if (currentNode && reactFlowInstance) {
- const nodeWidth = currentNode.measured?.width ?? 0
- const nodeHeight = currentNode.measured?.height ?? 0
- setCenter(
- currentNode.position.x + nodeWidth / 2,
- currentNode.position.y + nodeHeight / 2 + 20,
- { duration: 500, zoom: 1.5 }
- )
- }
- }, [currentNode, reactFlowInstance, setCenter])
-
- useEffect(() => {
- if (reactFlowInstance) {
- const actions = {
- zoomIn: zoomIn,
- zoomToFit: fitView,
- zoomOut: zoomOut,
- onLayout: onLayout,
- }
- setActions(actions)
- }
- }, [reactFlowInstance, setActions])
-
- const onDragOver = useCallback((event: React.DragEvent<HTMLDivElement>) => {
- event.preventDefault()
- event.dataTransfer.dropEffect = "move"
- }, [])
-
- const onDrop = useCallback(
- (event: React.DragEvent<HTMLDivElement>) => {
- event.preventDefault()
- if (!reactFlowWrapper.current || !reactFlowInstance) return
- const reactFlowBounds = reactFlowWrapper.current.getBoundingClientRect()
- const newNodeData = JSON.parse(event.dataTransfer.getData("application/json")) as any & {
- type: string
- }
- const position = reactFlowInstance.screenToFlowPosition({
- x: event.clientX - reactFlowBounds.left,
- y: event.clientY - reactFlowBounds.top,
- })
- const id = `${newNodeData.type}-${Date.now()}`
- const newNode: GraphNode = {
- id: id,
- position,
- data: {
- id: id,
- type: newNodeData.type,
- caption: newNodeData.caption,
- label: newNodeData.label,
- created_at: new Date().toISOString(),
- },
- }
- addNode(newNode)
- const nodeWidth = newNode.measured?.width ?? 0
- const nodeHeight = newNode.measured?.height ?? 0
- setCenter(newNode.position.x + nodeWidth / 2, newNode.position.y + nodeHeight / 2 + 20, {
- duration: 500,
- zoom: 1.5,
- })
- },
- [reactFlowInstance, addNode, setCenter],
- )
-
- const onNodeClick: NodeMouseHandler = useCallback(
- (e: React.MouseEvent, node: Node) => {
- if (e.shiftKey) {
- toggleNodeSelection(node as GraphNode, true)
- return
- }
- const typedNode = node as GraphNode
- setCurrentNode(typedNode)
- setActiveTab("entities")
- const nodeWidth = typedNode.measured?.width ?? 0
- const nodeHeight = typedNode.measured?.height ?? 0
- setCenter(typedNode.position.x + nodeWidth / 2, typedNode.position.y + nodeHeight / 2 + 20, {
- duration: 500,
- zoom: 1.5,
- })
- },
- [setCenter, setCurrentNode],
- )
-
- const onNodeDoubleClick: NodeMouseHandler = useCallback(
- (_: React.MouseEvent, node: Node) => {
- const typedNode = node as GraphNode
- setCurrentNode(typedNode)
- setOpenNodeEditorModal(true)
- },
- [setCurrentNode],
- )
-
- const onPaneClick = useCallback(() => {
- setCurrentNode(null)
- clearSelectedNodes()
- setMenu(null)
- }, [setCurrentNode, clearSelectedNodes, setMenu])
-
- const onLayout = useCallback((type: "dagre-tb" | "dagre-lr" | "force") => {
- // Wait for nodes to be measured before running layout
- setTimeout(() => {
- let layouted: any;
- if (type === "dagre-tb") {
- layouted = getDagreLayoutedElements(
- nodes,
- edges,
- { direction: "TB" }
- );
- } else if (type === "dagre-lr") {
- layouted = getDagreLayoutedElements(
- nodes,
- edges,
- { direction: "LR" }
- );
- } else {
- layouted = getForceLayoutedElements(
- nodes,
- edges,
- {
- direction: "TB",
- strength: -300,
- distance: 100,
- iterations: 300
- }
- );
- }
- setNodes(layouted.nodes as GraphNode[])
- setEdges(layouted.edges as GraphEdge[])
- window.requestAnimationFrame(() => {
- fitView()
- })
- }, 100)
- }, [nodes, edges, setNodes, setEdges, fitView])
-
- // Run dagre layout on first render
- useEffect(() => {
- if (reactFlowInstance && nodes.length) {
- onLayout("dagre-tb")
- }
- }, [reactFlowInstance, nodes.length])
-
- const onNodeContextMenu = useCallback(
- (event: React.MouseEvent, node: Node) => {
- // Prevent native context menu from showing
- event.preventDefault();
-
- // Calculate position of the context menu. We want to make sure it
- // doesn't get positioned off-screen.
- if (!reactFlowWrapper.current) return;
-
- const pane = reactFlowWrapper.current.getBoundingClientRect();
- const relativeX = event.clientX - pane.left;
- const relativeY = event.clientY - pane.top;
-
- setMenu({
- node: node as GraphNode,
- rawTop: relativeY,
- rawLeft: relativeX,
- wrapperWidth: pane.width,
- wrapperHeight: pane.height,
- setMenu: setMenu,
- });
- },
- [setMenu],
- );
-
- // Memoize ReactFlow props
- const reactFlowProps = useMemo(() => ({
- nodes: visibleNodes,
- edges: visibleEdges,
- nodeTypes,
- edgeTypes: edgeTypes as EdgeTypes,
- onInit: setReactFlowInstance,
- onNodeContextMenu,
- onDrop,
- onDragOver,
- onNodeClick,
- onNodeDoubleClick,
- onPaneClick,
- onNodesChange,
- onEdgesChange,
- onConnect,
- fitView: true,
- proOptions: { hideAttribution: true },
- colorMode: theme,
- minZoom: 0.1,
- maxZoom: 2.5,
- onlyRenderVisibleElements: nodes.length > 500,
- connectionMode: ConnectionMode.Loose
- }), [
- visibleNodes,
- visibleEdges,
- onDrop,
- onNodeContextMenu,
- onDragOver,
- onNodeClick,
- onNodeDoubleClick,
- onPaneClick,
- onNodesChange,
- onEdgesChange,
- onConnect,
- theme,
- nodes.length
- ])
-
- return (
- <>
- <div ref={reactFlowWrapper} className="w-full h-full bg-background">
- <ReactFlow {...reactFlowProps} className="!bg-background">
- {/* <Background
- variant={"dots" as BackgroundVariant}
- bgColor="var(--background)" /> */}
- {menu && <ContextMenu
- {...menu}
- />}
- <MiniMap nodeBorderRadius={80} nodeClassName={"h-16 w-16 text-center"} nodeColor={(node) => colors[node.data.type as keyof typeof colors]} className="bg-background" position="bottom-right" pannable zoomable />
- </ReactFlow>
- </div>
- </>
- )
-})
-
-Wall.displayName = "Wall"
-
-const WallEditor = memo(({ isLoading, isRefetching }: { isLoading: boolean, isRefetching: boolean }) => {
- const { theme } = useTheme()
- const edges = useGraphStore(state => state.edges)
-
- const edgesWithMarkers = useMemo(() => edges.map(edge => ({
- ...edge,
- markerEnd: {
- type: MarkerType.ArrowClosed,
- },
- // markerStart: {
- // type: MarkerType.ArrowClosed,
- // orient: 'auto-start-reverse',
- // },
- })), [edges])
-
- if (isLoading || isRefetching) {
- return <GraphLoader />
- }
-
- return (
- <ReactFlowProvider>
- <Wall edges={edgesWithMarkers} theme={theme as ColorMode} />
- </ReactFlowProvider>
- )
-})
-
-WallEditor.displayName = "WallEditor"
-
-export default WallEditor
diff --git a/flowsint-app/src/renderer/src/components/table/index.tsx b/flowsint-app/src/renderer/src/components/table/index.tsx
index c7e192ae..cbd6586b 100644
--- a/flowsint-app/src/renderer/src/components/table/index.tsx
+++ b/flowsint-app/src/renderer/src/components/table/index.tsx
@@ -2,7 +2,7 @@ import { useGraphStore } from "@/stores/graph-store"
import NodesTable from "./nodes-view"
export default function NodesViewTable() {
- const nodes = useGraphStore(s => s.nodes)
+ const nodes = useGraphStore(s => s.filteredNodes)
return (
<NodesTable nodes={nodes} />
diff --git a/flowsint-app/src/renderer/src/hooks/use-events.ts b/flowsint-app/src/renderer/src/hooks/use-events.ts
index 8fa160fc..d1d9d8e0 100644
--- a/flowsint-app/src/renderer/src/hooks/use-events.ts
+++ b/flowsint-app/src/renderer/src/hooks/use-events.ts
@@ -31,7 +31,7 @@ export function useEvents(sketch_id: string | undefined) {
try {
const raw = JSON.parse(e.data) as any
const event = JSON.parse(raw.data) as Event
- if (event.type === EventLevel.GRAPH_APPEND) {
+ if (event.type === EventLevel.COMPLETED) {
refetchGraph()
// const nodes = event.payload.nodes as GraphNode[]
// const edges = event.payload.edges as GraphEdge[]
diff --git a/flowsint-app/src/renderer/src/lib/utils.ts b/flowsint-app/src/renderer/src/lib/utils.ts
index 6cd40364..5202eb54 100644
--- a/flowsint-app/src/renderer/src/lib/utils.ts
+++ b/flowsint-app/src/renderer/src/lib/utils.ts
@@ -335,4 +335,18 @@ export function capitalizeFirstLetter(string: string) {
}
-
+export const getAllNodeTypes = (actionItems: any[]) => {
+ const types: string[] = []
+ actionItems.forEach(item => {
+ if (item.children) {
+ item.children.forEach(child => {
+ if (child.type && !types.includes(child.type)) {
+ types.push(child.type)
+ }
+ })
+ } else if (item.type && !types.includes(item.type)) {
+ types.push(item.type)
+ }
+ })
+ return types.sort()
+}
\ No newline at end of file
diff --git a/flowsint-app/src/renderer/src/routes/_auth.dashboard.investigations.$investigationId.$type.$id.tsx b/flowsint-app/src/renderer/src/routes/_auth.dashboard.investigations.$investigationId.$type.$id.tsx
index b6d37bc7..49adf2a8 100644
--- a/flowsint-app/src/renderer/src/routes/_auth.dashboard.investigations.$investigationId.$type.$id.tsx
+++ b/flowsint-app/src/renderer/src/routes/_auth.dashboard.investigations.$investigationId.$type.$id.tsx
@@ -92,16 +92,16 @@ export const Route = createFileRoute('/_auth/dashboard/investigations/$investiga
</svg>
</div>
</div>
-
+
<h2 className="text-xl font-semibold text-foreground mb-2">
Oops! Something went wrong
</h2>
-
+
<p className="text-muted-foreground mb-6 text-sm leading-relaxed">
- We couldn't load this investigation page. This might be due to a network issue,
+ We couldn't load this investigation page. This might be due to a network issue,
invalid investigation ID, or the resource may not exist.
</p>
-
+
<div className="space-y-3">
<button
onClick={() => window.location.reload()}
@@ -124,7 +124,7 @@ export const Route = createFileRoute('/_auth/dashboard/investigations/$investiga
</svg>
Reload Page
</button>
-
+
<button
onClick={() => window.history.back()}
className="w-full bg-muted hover:bg-muted/80 text-muted-foreground hover:text-foreground
@@ -133,7 +133,7 @@ export const Route = createFileRoute('/_auth/dashboard/investigations/$investiga
Go Back
</button>
</div>
-
+
<div className="mt-6 pt-4 border-t">
<p className="text-xs text-muted-foreground">
If the problem persists, try:
diff --git a/flowsint-app/src/renderer/src/stores/graph-store.ts b/flowsint-app/src/renderer/src/stores/graph-store.ts
index 36edf3ae..eebc2fad 100644
--- a/flowsint-app/src/renderer/src/stores/graph-store.ts
+++ b/flowsint-app/src/renderer/src/stores/graph-store.ts
@@ -1,4 +1,5 @@
import { create } from "zustand"
+import { persist } from "zustand/middleware"
import type { EdgeData, NodeData } from "@/types"
import {
type Node,
@@ -24,6 +25,8 @@ interface GraphState {
// === Graph ===
nodes: GraphNode[]
edges: GraphEdge[]
+ filteredNodes: GraphNode[]
+ filteredEdges: GraphEdge[]
setNodes: (nodes: GraphNode[]) => void
setEdges: (edges: GraphEdge[]) => void
addNode: (newNode: Partial<GraphNode>) => GraphNode
@@ -71,8 +74,8 @@ interface GraphState {
handleEdit: (node: GraphNode) => void
// === Filters ===
- filters: Record<string, unknown>
- setFilters: (filters: Record<string, unknown>) => void
+ filters: string[] | null
+ setFilters: (filters: string[] | null) => void
// === Collapse/Expand logic ===
toggleCollapse: (nodeId: string) => void
@@ -84,210 +87,289 @@ interface GraphState {
getEdgesLength: () => number
}
-export const useGraphStore = create<GraphState>()((set, get) => ({
- // === Graph ===
- nodes: [],
- edges: [],
- setNodes: (nodes) => set({ nodes }),
- setEdges: (edges) => set({ edges }),
- addNode: (newNode) => {
- const { nodes } = get()
- const nodeWithId: GraphNode = {
- id: newNode.id || `node-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`,
- position: {
- x: 0,
- y: 0,
- },
- ...newNode,
- } as GraphNode
- set({ nodes: [...nodes, nodeWithId], currentNode: nodeWithId })
- return nodeWithId
- },
- addEdge: (newEdge) => {
- const { edges } = get()
- const edgeWithId: GraphEdge = {
- id: newEdge.id || `node-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`,
- ...newEdge,
- } as GraphEdge
- set({ edges: [...edges, edgeWithId] })
- return edgeWithId
- },
- removeNodes: (nodeIds: string[]) => {
- const { nodes, edges } = get();
- const newNodes = nodes.filter((n) => !nodeIds.includes(n.id));
- const newEdges = edges.filter(
- (e) => !nodeIds.includes(e.source) && !nodeIds.includes(e.target)
- );
- set({ nodes: newNodes, edges: newEdges });
- },
- removeEdges: (edgeIds: string[]) => {
- const { edges } = get()
- const newEdges = edges.filter((e) => !edgeIds.includes(e.id))
- set({ edges: newEdges })
- },
- updateGraphData: (nodes: GraphNode[], edges: GraphEdge[]) => {
- set({ nodes, edges })
- },
- updateNode: (nodeId, updates) => {
- const { nodes } = get()
- const updatedNodes = nodes.map(node =>
- node.id === nodeId
- ? { ...node, data: { ...node.data, ...updates } }
- : node
- )
- set({ nodes: updatedNodes })
- },
- updateEdge: (edgeId, updates) => {
- const { edges } = get()
- const updatedEdges = edges.map(edge =>
- edge.id === edgeId
- ? { ...edge, data: { ...edge.data, ...updates } as EdgeData }
- : edge
- )
- set({ edges: updatedEdges })
- },
- onNodesChange: (changes) => {
- set({
- nodes: applyNodeChanges(changes, get().nodes) as GraphNode[],
- })
- },
- onEdgesChange: (changes) => {
- set({
- edges: applyEdgeChanges(changes, get().edges) as GraphEdge[],
- })
- },
- onConnect: (connection: Connection) => {
- const edge: GraphEdge = {
- id: `${connection.source}-${connection.target}`,
- source: connection.source!,
- target: connection.target!,
- sourceHandle: connection.sourceHandle,
- targetHandle: connection.targetHandle,
+// --- Helpers ---
+const computeFilteredNodes = (nodes: GraphNode[], filters: string[] | null): GraphNode[] => {
+ if (!filters || Object.keys(filters).length === 0) return nodes
+ return nodes.filter((node) => {
+ if (filters && Array.isArray(filters)) {
+ return filters.includes(node.data.type)
}
- set({
- edges: [...get().edges, edge],
- })
- },
- reset: () => {
- set({
- // nodes: [],
- // edges: [],
+ return true
+ })
+}
+
+const computeFilteredEdges = (edges: GraphEdge[], filteredNodes: GraphNode[]): GraphEdge[] => {
+ const nodeIds = new Set(filteredNodes.map(n => n.id))
+ return edges.filter(e => nodeIds.has(e.source) && nodeIds.has(e.target))
+}
+
+// --- Store ---
+export const useGraphStore = create<GraphState>()(
+ persist(
+ (set, get) => ({
+ // === Graph ===
+ nodes: [],
+ edges: [],
+ filteredNodes: [],
+ filteredEdges: [],
+ getNodesLength: () => get().nodes.length,
+ getEdgesLength: () => get().edges.length,
+
+ setNodes: (nodes) => {
+ const { filters, edges } = get()
+ const filteredNodes = computeFilteredNodes(nodes, filters)
+ const filteredEdges = computeFilteredEdges(edges, filteredNodes)
+ set({ nodes, filteredNodes, filteredEdges })
+ },
+ setEdges: (edges) => {
+ const { filters, nodes } = get()
+ const filteredNodes = computeFilteredNodes(nodes, filters)
+ const filteredEdges = computeFilteredEdges(edges, filteredNodes)
+ set({ edges, filteredNodes, filteredEdges })
+ },
+
+ addNode: (newNode) => {
+ const { nodes, edges, filters } = get()
+ const nodeWithId: GraphNode = {
+ id: newNode.id || `node-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`,
+ position: { x: 0, y: 0 },
+ ...newNode,
+ } as GraphNode
+ const newNodes = [...nodes, nodeWithId]
+ const filteredNodes = computeFilteredNodes(newNodes, filters)
+ const filteredEdges = computeFilteredEdges(edges, filteredNodes)
+ set({ nodes: newNodes, currentNode: nodeWithId, filteredNodes, filteredEdges })
+ return nodeWithId
+ },
+
+ addEdge: (newEdge) => {
+ const { edges, nodes, filters } = get()
+ const edgeWithId: GraphEdge = {
+ id: newEdge.id || `edge-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`,
+ ...newEdge,
+ } as GraphEdge
+ const newEdges = [...edges, edgeWithId]
+ const filteredNodes = computeFilteredNodes(nodes, filters)
+ const filteredEdges = computeFilteredEdges(newEdges, filteredNodes)
+ set({ edges: newEdges, filteredNodes, filteredEdges })
+ return edgeWithId
+ },
+
+ removeNodes: (nodeIds: string[]) => {
+ const { nodes, edges, filters } = get()
+ const newNodes = nodes.filter(n => !nodeIds.includes(n.id))
+ const newEdges = edges.filter(e => !nodeIds.includes(e.source) && !nodeIds.includes(e.target))
+ const filteredNodes = computeFilteredNodes(newNodes, filters)
+ const filteredEdges = computeFilteredEdges(newEdges, filteredNodes)
+ set({ nodes: newNodes, edges: newEdges, filteredNodes, filteredEdges })
+ },
+
+ removeEdges: (edgeIds: string[]) => {
+ const { edges, nodes, filters } = get()
+ const newEdges = edges.filter(e => !edgeIds.includes(e.id))
+ const filteredNodes = computeFilteredNodes(nodes, filters)
+ const filteredEdges = computeFilteredEdges(newEdges, filteredNodes)
+ set({ edges: newEdges, filteredNodes, filteredEdges })
+ },
+
+ updateGraphData: (nodes, edges) => {
+ const { filters } = get()
+ const filteredNodes = computeFilteredNodes(nodes, filters)
+ const filteredEdges = computeFilteredEdges(edges, filteredNodes)
+ set({ nodes, edges, filteredNodes, filteredEdges })
+ },
+
+ updateNode: (nodeId, updates) => {
+ const { nodes, edges, filters } = get()
+ const updatedNodes = nodes.map(node =>
+ node.id === nodeId ? { ...node, data: { ...node.data, ...updates } } : node
+ )
+ const filteredNodes = computeFilteredNodes(updatedNodes, filters)
+ const filteredEdges = computeFilteredEdges(edges, filteredNodes)
+ set({ nodes: updatedNodes, filteredNodes, filteredEdges })
+ },
+
+ updateEdge: (edgeId, updates) => {
+ const { edges, nodes, filters } = get()
+ const updatedEdges = edges.map(edge =>
+ edge.id === edgeId ? { ...edge, data: { ...edge.data, ...updates } as EdgeData } : edge
+ )
+ const filteredNodes = computeFilteredNodes(nodes, filters)
+ const filteredEdges = computeFilteredEdges(updatedEdges, filteredNodes)
+ set({ edges: updatedEdges, filteredNodes, filteredEdges })
+ },
+
+ onNodesChange: (changes) => {
+ const { nodes, edges, filters } = get()
+ const updatedNodes = applyNodeChanges(changes, nodes) as GraphNode[]
+ const filteredNodes = computeFilteredNodes(updatedNodes, filters)
+ const filteredEdges = computeFilteredEdges(edges, filteredNodes)
+ set({ nodes: updatedNodes, filteredNodes, filteredEdges })
+ },
+
+ onEdgesChange: (changes) => {
+ const { edges, nodes, filters } = get()
+ const updatedEdges = applyEdgeChanges(changes, edges) as GraphEdge[]
+ const filteredNodes = computeFilteredNodes(nodes, filters)
+ const filteredEdges = computeFilteredEdges(updatedEdges, filteredNodes)
+ set({ edges: updatedEdges, filteredNodes, filteredEdges })
+ },
+
+ onConnect: (connection: Connection) => {
+ const { edges, nodes, filters } = get()
+ const edge: GraphEdge = {
+ id: `${connection.source}-${connection.target}`,
+ source: connection.source!,
+ target: connection.target!,
+ sourceHandle: connection.sourceHandle,
+ targetHandle: connection.targetHandle,
+ }
+ const newEdges = [...edges, edge]
+ const filteredNodes = computeFilteredNodes(nodes, filters)
+ const filteredEdges = computeFilteredEdges(newEdges, filteredNodes)
+ set({ edges: newEdges, filteredNodes, filteredEdges })
+ },
+
+ reset: () => {
+ set({
+ currentNode: null,
+ selectedNodes: [],
+ relatedNodeToAdd: null,
+ openMainDialog: false,
+ openFormDialog: false,
+ openAddRelationDialog: false,
+ openNodeEditorModal: false,
+ currentNodeType: null,
+ // filters: {},
+ filteredNodes: get().nodes,
+ filteredEdges: get().edges,
+ })
+ },
+
+ // === Selection & Current ===
currentNode: null,
selectedNodes: [],
+ isCurrent: (nodeId) => get().currentNode?.id === nodeId,
+ isSelected: (nodeId) => {
+ const { selectedNodes, isCurrent } = get()
+ return selectedNodes.some((node) => node.id === nodeId) || isCurrent(nodeId)
+ },
+ setCurrentNode: (node) => set({ currentNode: node }),
+ setSelectedNodes: (nodes) => set({ selectedNodes: nodes }),
+ clearSelectedNodes: () => set({ selectedNodes: [], currentNode: null }),
+ toggleNodeSelection: (node, multiSelect = false) => {
+ const { selectedNodes, currentNode } = get()
+ const isSelected = selectedNodes.some((n) => n.id === node.id)
+ let newSelected: GraphNode[]
+
+ if (multiSelect) {
+ newSelected = isSelected
+ ? selectedNodes.filter((n) => n.id !== node.id)
+ : [...selectedNodes, node]
+ } else {
+ newSelected = isSelected && selectedNodes.length === 1 ? [] : [node]
+ }
+
+ set({
+ selectedNodes: newSelected,
+ currentNode: multiSelect ? currentNode : (isSelected ? null : node),
+ })
+ },
+
+ // === Relation ===
relatedNodeToAdd: null,
+ setRelatedNodeToAdd: (node) => set({ relatedNodeToAdd: node }),
+
+ // === Dialogs ===
openMainDialog: false,
openFormDialog: false,
openAddRelationDialog: false,
openNodeEditorModal: false,
- currentNodeType: null,
- filters: {},
- })
- },
+ setOpenMainDialog: (open) => set({ openMainDialog: open }),
+ setOpenFormDialog: (open) => set({ openFormDialog: open }),
+ setOpenAddRelationDialog: (open) => set({ openAddRelationDialog: open }),
+ setOpenNodeEditorModal: (open) => set({ openNodeEditorModal: open }),
- // === Selection & Current ===
- currentNode: null,
- selectedNodes: [],
- isCurrent: (nodeId) => get().currentNode?.id === nodeId,
- isSelected: (nodeId) => {
- const { selectedNodes, isCurrent } = get()
- return selectedNodes.some((node) => node.id === nodeId) || isCurrent(nodeId)
- },
- setCurrentNode: (node) => set({ currentNode: node }),
- setSelectedNodes: (nodes) => set({ selectedNodes: nodes }),
- clearSelectedNodes: () => set({ selectedNodes: [], currentNode: null }),
- toggleNodeSelection: (node, multiSelect = false) => {
- const { selectedNodes, currentNode } = get()
- const isSelected = selectedNodes.some((n) => n.id === node.id)
- let newSelected: GraphNode[]
-
- if (multiSelect) {
- newSelected = isSelected
- ? selectedNodes.filter((n) => n.id !== node.id)
- : [...selectedNodes, node]
- } else {
- newSelected = isSelected && selectedNodes.length === 1 ? [] : [node]
- }
+ // === Action Type for Edit form ===
+ handleEdit: (node) => set({ currentNode: node, openNodeEditorModal: true }),
- set({
- selectedNodes: newSelected,
- currentNode: multiSelect ? currentNode : (isSelected ? null : node),
- })
- },
-
- // === Relation ===
- relatedNodeToAdd: null,
- setRelatedNodeToAdd: (node) => set({ relatedNodeToAdd: node }),
+ // === Action Type for Form ===
+ currentNodeType: null,
+ setCurrentNodeType: (nodeType) => set({ currentNodeType: nodeType }),
+ handleOpenFormModal: (selectedItem) => {
+ if (!selectedItem) return
+ set({
+ currentNodeType: selectedItem,
+ openMainDialog: false,
+ openFormDialog: true,
+ })
+ },
- // === Dialogs ===
- openMainDialog: false,
- openFormDialog: false,
- openAddRelationDialog: false,
- openNodeEditorModal: false,
- setOpenMainDialog: (open) => set({ openMainDialog: open }),
- setOpenFormDialog: (open) => set({ openFormDialog: open }),
- setOpenAddRelationDialog: (open) => set({ openAddRelationDialog: open }),
- setOpenNodeEditorModal: (open) => set({ openNodeEditorModal: open }),
+ // === Filters ===
+ filters: [],
+ setFilters: (filters) => {
+ const { nodes, edges } = get()
+ const filteredNodes = computeFilteredNodes(nodes, filters)
+ const filteredEdges = computeFilteredEdges(edges, filteredNodes)
+ set({ filters, filteredNodes, filteredEdges })
+ },
- // === Action Type for Edit form ===
- handleEdit: (node) => set({ currentNode: node, openNodeEditorModal: true }),
+ // === Collapse/Expand logic ===
+ toggleCollapse: (nodeId) => {
+ const { nodes, edges, filters } = get()
+ const node = nodes.find(n => n.id === nodeId)
+ if (!node) return
+ const isCollapsing = !node.collapsed
- // === Action Type for Form ===
- currentNodeType: null,
- setCurrentNodeType: (nodeType) => set({ currentNodeType: nodeType }),
- handleOpenFormModal: (selectedItem) => {
- if (!selectedItem) return
- set({
- currentNodeType: selectedItem,
- openMainDialog: false,
- openFormDialog: true,
- })
- },
+ const getDescendants = (parentId: string, accNodes: Set<string>, accEdges: Set<string>) => {
+ edges.forEach(edge => {
+ if (edge.source === parentId) {
+ accEdges.add(edge.id)
+ accNodes.add(edge.target)
+ getDescendants(edge.target, accNodes, accEdges)
+ }
+ })
+ }
- // === Filters ===
- filters: {},
- setFilters: (filters) => set({ filters }),
+ const descendantNodeIds = new Set<string>()
+ const descendantEdgeIds = new Set<string>()
+ getDescendants(nodeId, descendantNodeIds, descendantEdgeIds)
- // === Collapse/Expand logic ===
- toggleCollapse: (nodeId) => {
- const { nodes, edges } = get();
- // Find the node
- const node = nodes.find(n => n.id === nodeId);
- if (!node) return;
- const isCollapsing = !node.collapsed;
- // Recursively find all descendants
- const getDescendants = (parentId: string, accNodes: Set<string>, accEdges: Set<string>) => {
- edges.forEach(edge => {
- if (edge.source === parentId) {
- accEdges.add(edge.id);
- accNodes.add(edge.target);
- getDescendants(edge.target, accNodes, accEdges);
- }
- });
- };
- const descendantNodeIds = new Set<string>();
- const descendantEdgeIds = new Set<string>();
- getDescendants(nodeId, descendantNodeIds, descendantEdgeIds);
- // Update nodes and edges
- set({
- nodes: nodes.map(n =>
- n.id === nodeId
- ? { ...n, collapsed: isCollapsing }
- : descendantNodeIds.has(n.id)
- ? { ...n, hidden: isCollapsing }
- : n
- ),
- edges: edges.map(e =>
- descendantEdgeIds.has(e.id)
- ? { ...e, hidden: isCollapsing }
- : e
- ),
- });
- },
+ const newNodes = nodes.map(n =>
+ n.id === nodeId
+ ? { ...n, collapsed: isCollapsing }
+ : descendantNodeIds.has(n.id)
+ ? { ...n, hidden: isCollapsing }
+ : n
+ )
+ const newEdges = edges.map(e =>
+ descendantEdgeIds.has(e.id)
+ ? { ...e, hidden: isCollapsing }
+ : e
+ )
- // === Utils ===
- nodesLength: 0,
- edgesLength: 0,
+ const filteredNodes = computeFilteredNodes(newNodes, filters)
+ const filteredEdges = computeFilteredEdges(newEdges, filteredNodes)
+ set({ nodes: newNodes, edges: newEdges, filteredNodes, filteredEdges })
+ },
- getNodesLength: () => get().nodes.length,
- getEdgesLength: () => get().edges.length,
-}))
+ // === Utils ===
+ nodesLength: 0,
+ edgesLength: 0,
+ }),
+ {
+ name: "graph-store",
+ partialize: (state) => ({ filters: state.filters }),
+ onRehydrateStorage: () => (state) => {
+ if (state) {
+ // recalcul des nodes/edges filtrés après rehydratation
+ const { nodes, edges, filters } = state
+ const filteredNodes = computeFilteredNodes(nodes, filters)
+ const filteredEdges = computeFilteredEdges(edges, filteredNodes)
+ state.filteredNodes = filteredNodes
+ state.filteredEdges = filteredEdges
+ }
+ },
+ }
+ )
+)
diff --git a/flowsint-core/src/flowsint_core/core/registry.py b/flowsint-core/src/flowsint_core/core/registry.py
index c27a57ed..156b8e4c 100644
--- a/flowsint-core/src/flowsint_core/core/registry.py
+++ b/flowsint-core/src/flowsint_core/core/registry.py
@@ -1,32 +1,57 @@
import inspect
-from typing import Dict, Type, List
+from typing import Dict, Optional, Type, List
from flowsint_core.core.scanner_base import Scanner
-from flowsint_transforms.domains.subdomains import SubdomainScanner
-from flowsint_transforms.domains.whois import WhoisScanner
-from flowsint_transforms.domains.resolve import ResolveScanner
-from flowsint_transforms.ips.reverse_resolve import ReverseResolveScanner
-from flowsint_transforms.ips.geolocation import GeolocationScanner
-from flowsint_transforms.socials.maigret import MaigretScanner
-from flowsint_transforms.ips.ip_to_asn import IpToAsnScanner
-from flowsint_transforms.ips.asn_to_cidrs import AsnToCidrsScanner
-from flowsint_transforms.ips.cidr_to_ips import CidrToIpsScanner
-from flowsint_transforms.organizations.org_to_asn import OrgToAsnScanner
-from flowsint_transforms.domains.domain_to_asn import DomainToAsnScanner
-from flowsint_transforms.crypto.wallet_to_transactions import (
+
+# Domain-related scanners
+from flowsint_transforms.domain.to_subdomains import SubdomainScanner
+from flowsint_transforms.domain.to_whois import WhoisScanner
+from flowsint_transforms.domain.to_ip import ResolveScanner
+from flowsint_transforms.domain.to_website import DomainToWebsiteScanner
+from flowsint_transforms.domain.to_root_domain import DomainToRootDomain
+from flowsint_transforms.domain.to_asn import DomainToAsnScanner
+
+# IP-related scanners
+from flowsint_transforms.ip.to_domain import ReverseResolveScanner
+from flowsint_transforms.ip.to_geolocation import GeolocationScanner
+
+
+from flowsint_transforms.ip.to_asn import IpToAsnScanner
+
+# ASN-related scanners
+from flowsint_transforms.asn.to_cidrs import AsnToCidrsScanner
+
+# CIDR-related scanners
+from flowsint_transforms.cidr.to_ips import CidrToIpsScanner
+
+# Social media scanners
+from flowsint_transforms.social.to_maigret import MaigretScanner
+
+# Organization-related scanners
+from flowsint_transforms.organization.to_asn import OrgToAsnScanner
+from flowsint_transforms.organization.to_infos import OrgToInfosScanner
+
+# Cryptocurrency scanners
+from flowsint_transforms.crypto.to_transactions import (
CryptoWalletAddressToTransactions,
)
-from flowsint_transforms.crypto.wallet_to_nfts import CryptoWalletAddressToNFTs
-from flowsint_transforms.domains.to_website import DomainToWebsiteScanner
-from flowsint_transforms.websites.to_crawler import WebsiteToCrawler
-from flowsint_transforms.websites.to_links import WebsiteToLinks
-from flowsint_transforms.websites.to_domain import WebsiteToDomainScanner
-from flowsint_transforms.emails.to_gravatar import EmailToGravatarScanner
-from flowsint_transforms.emails.to_leaks import EmailToBreachesScanner
-from flowsint_transforms.individuals.to_org import IndividualToOrgScanner
-from flowsint_transforms.organizations.to_infos import OrgToInfosScanner
-from flowsint_transforms.websites.to_webtrackers import WebsiteToWebtrackersScanner
+from flowsint_transforms.crypto.to_nfts import CryptoWalletAddressToNFTs
+
+# Website-related scanners
+from flowsint_transforms.website.to_crawler import WebsiteToCrawler
+from flowsint_transforms.website.to_links import WebsiteToLinks
+from flowsint_transforms.website.to_domain import WebsiteToDomainScanner
+from flowsint_transforms.website.to_text import WebsiteToText
+from flowsint_transforms.website.to_webtrackers import WebsiteToWebtrackersScanner
+
+# Email-related scanners
+from flowsint_transforms.email.to_gravatar import EmailToGravatarScanner
+from flowsint_transforms.email.to_leaks import EmailToBreachesScanner
+
+# Individual-related scanners
+from flowsint_transforms.individual.to_org import IndividualToOrgScanner
+
+# Integration scanners
from flowsint_transforms.n8n.connector import N8nConnector
-from flowsint_transforms.domains.to_root_domain import DomainToRootDomain
class TransformRegistry:
@@ -66,10 +91,11 @@ class TransformRegistry:
}
@classmethod
- def list(cls) -> Dict[str, Dict[str, str]]:
+ def list(cls, exclude: Optional[List[str]] = []) -> Dict[str, Dict[str, str]]:
return {
name: cls._create_scanner_metadata(scanner)
for name, scanner in cls._scanners.items()
+ if name not in exclude
}
@classmethod
@@ -83,7 +109,9 @@ class TransformRegistry:
return scanners_by_category
@classmethod
- def list_by_input_type(cls, input_type: str) -> List[Dict[str, str]]:
+ def list_by_input_type(
+ cls, input_type: str, exclude: Optional[List[str]] = []
+ ) -> List[Dict[str, str]]:
input_type_lower = input_type.lower()
if input_type_lower == "any":
@@ -96,31 +124,55 @@ class TransformRegistry:
cls._create_scanner_metadata(scanner)
for scanner in cls._scanners.values()
if scanner.input_schema()["type"].lower() in ["any", input_type_lower]
+ and scanner.name not in exclude
]
# Register all scanners
+
+# Domain-related scanners
TransformRegistry.register(ReverseResolveScanner)
TransformRegistry.register(ResolveScanner)
TransformRegistry.register(SubdomainScanner)
TransformRegistry.register(WhoisScanner)
+TransformRegistry.register(DomainToWebsiteScanner)
+TransformRegistry.register(DomainToRootDomain)
+TransformRegistry.register(DomainToAsnScanner)
+
+# IP-related scanners
TransformRegistry.register(GeolocationScanner)
-TransformRegistry.register(MaigretScanner)
TransformRegistry.register(IpToAsnScanner)
+
+# ASN-related scanners
TransformRegistry.register(AsnToCidrsScanner)
+
+# CIDR-related scanners
TransformRegistry.register(CidrToIpsScanner)
+
+# Social media scanners
+TransformRegistry.register(MaigretScanner)
+
+# Organization-related scanners
TransformRegistry.register(OrgToAsnScanner)
-TransformRegistry.register(DomainToAsnScanner)
+TransformRegistry.register(OrgToInfosScanner)
+
+# Cryptocurrency scanners
TransformRegistry.register(CryptoWalletAddressToTransactions)
TransformRegistry.register(CryptoWalletAddressToNFTs)
-TransformRegistry.register(DomainToWebsiteScanner)
+
+# Website-related scanners
TransformRegistry.register(WebsiteToCrawler)
TransformRegistry.register(WebsiteToLinks)
TransformRegistry.register(WebsiteToDomainScanner)
+TransformRegistry.register(WebsiteToWebtrackersScanner)
+TransformRegistry.register(WebsiteToText)
+
+# Email-related scanners
TransformRegistry.register(EmailToGravatarScanner)
TransformRegistry.register(EmailToBreachesScanner)
+
+# Individual-related scanners
TransformRegistry.register(IndividualToOrgScanner)
-TransformRegistry.register(OrgToInfosScanner)
-TransformRegistry.register(WebsiteToWebtrackersScanner)
-TransformRegistry.register(DomainToRootDomain)
+
+# Integration scanners
TransformRegistry.register(N8nConnector)
diff --git a/flowsint-transforms/src/flowsint_transforms/ips/asn_to_cidrs.py b/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.py
similarity index 100%
rename from flowsint-transforms/src/flowsint_transforms/ips/asn_to_cidrs.py
rename to flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.py
diff --git a/flowsint-transforms/src/flowsint_transforms/ips/cidr_to_ips.py b/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.py
similarity index 100%
rename from flowsint-transforms/src/flowsint_transforms/ips/cidr_to_ips.py
rename to flowsint-transforms/src/flowsint_transforms/cidr/to_ips.py
diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_nfts.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py
similarity index 98%
rename from flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_nfts.py
rename to flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py
index 014cdf39..242dbbe9 100644
--- a/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_nfts.py
+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py
@@ -90,7 +90,7 @@ class CryptoWalletAddressToNFTs(Scanner):
results: OutputType = []
params = self.get_params()
Logger.debug(self.sketch_id, {"message": f"{str(params)}"})
- api_key = params.get("ETHERSCAN_API_KEY", None)
+ api_key = params.get("ETHERSCAN_API_KEY", os.getenv("ETHERSCAN_API_KEY"))
api_url = params.get("ETHERSCAN_API_URL", None)
if not api_key:
Logger.error(self.sketch_id, {"message": "ETHERSCAN_API_KEY is required"})
diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_transactions.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py
similarity index 98%
rename from flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_transactions.py
rename to flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py
index 2a03ad8e..6e2e31d0 100644
--- a/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_transactions.py
+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py
@@ -94,8 +94,8 @@ class CryptoWalletAddressToTransactions(Scanner):
async def scan(self, data: InputType) -> OutputType:
results: OutputType = []
params = self.get_params()
- api_key = params["ETHERSCAN_API_KEY"]
- api_url = params["ETHERSCAN_API_URL"]
+ api_key = params.get("ETHERSCAN_API_KEY", os.getenv("ETHERSCAN_API_KEY"))
+ api_url = params.get("ETHERSCAN_API_URL", ETHERSCAN_API_URL)
if not api_key:
Logger.error(self.sketch_id, {"message": "ETHERSCAN_API_KEY is required"})
raise ValueError("ETHERSCAN_API_KEY is required")
diff --git a/flowsint-transforms/src/flowsint_transforms/domains/domain_to_asn.py b/flowsint-transforms/src/flowsint_transforms/domain/to_asn.py
similarity index 100%
rename from flowsint-transforms/src/flowsint_transforms/domains/domain_to_asn.py
rename to flowsint-transforms/src/flowsint_transforms/domain/to_asn.py
diff --git a/flowsint-transforms/src/flowsint_transforms/domains/resolve.py b/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py
similarity index 98%
rename from flowsint-transforms/src/flowsint_transforms/domains/resolve.py
rename to flowsint-transforms/src/flowsint_transforms/domain/to_ip.py
index a9aa53c2..ee65289f 100644
--- a/flowsint-transforms/src/flowsint_transforms/domains/resolve.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py
@@ -15,7 +15,7 @@ class ResolveScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "domain_resolve_scanner"
+ return "domain_to_ip"
@classmethod
def category(cls) -> str:
@@ -292,7 +292,7 @@ Check FlowSint logs for detailed resolution information.
"domain",
"domain",
domain_obj.domain,
- type="domain" if domain_obj.root else "subdomain",
+ type="domain",
)
self.create_node("ip", "address", ip_obj.address, type="ip")
self.create_relationship(
diff --git a/flowsint-transforms/src/flowsint_transforms/domains/to_root_domain.py b/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py
similarity index 98%
rename from flowsint-transforms/src/flowsint_transforms/domains/to_root_domain.py
rename to flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py
index 2e66da4e..a0fcbc59 100644
--- a/flowsint-transforms/src/flowsint_transforms/domains/to_root_domain.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py
@@ -18,7 +18,7 @@ class DomainToRootDomain(Scanner):
@classmethod
def name(cls) -> str:
- return "to_root_domain"
+ return "domain_to_root_domain"
@classmethod
def category(cls) -> str:
diff --git a/flowsint-transforms/src/flowsint_transforms/domains/subdomains.py b/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py
similarity index 98%
rename from flowsint-transforms/src/flowsint_transforms/domains/subdomains.py
rename to flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py
index 83a11a79..9010729d 100644
--- a/flowsint-transforms/src/flowsint_transforms/domains/subdomains.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py
@@ -18,7 +18,7 @@ class SubdomainScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "domain_subdomains_scanner"
+ return "domain_to_subdomains"
@classmethod
def category(cls) -> str:
@@ -45,7 +45,7 @@ class SubdomainScanner(Scanner):
async def scan(self, data: InputType) -> OutputType:
"""Find subdomains using subfinder (if available) or fallback to crt.sh."""
domains: OutputType = []
- use_subfinder = self.__is_subfinder_installed()
+ use_subfinder = not self.__is_subfinder_installed()
for md in data:
d = Domain(domain=md.domain)
diff --git a/flowsint-transforms/src/flowsint_transforms/domains/to_website.py b/flowsint-transforms/src/flowsint_transforms/domain/to_website.py
similarity index 99%
rename from flowsint-transforms/src/flowsint_transforms/domains/to_website.py
rename to flowsint-transforms/src/flowsint_transforms/domain/to_website.py
index 70116968..3194d4a6 100644
--- a/flowsint-transforms/src/flowsint_transforms/domains/to_website.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_website.py
@@ -16,7 +16,7 @@ class DomainToWebsiteScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "to_website"
+ return "domain_to_website"
@classmethod
def category(cls) -> str:
diff --git a/flowsint-transforms/src/flowsint_transforms/domains/whois.py b/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py
similarity index 99%
rename from flowsint-transforms/src/flowsint_transforms/domains/whois.py
rename to flowsint-transforms/src/flowsint_transforms/domain/to_whois.py
index 162887c4..615bacd0 100644
--- a/flowsint-transforms/src/flowsint_transforms/domains/whois.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py
@@ -17,7 +17,7 @@ class WhoisScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "to_whois"
+ return "domain_to_whois"
@classmethod
def category(cls) -> str:
diff --git a/flowsint-transforms/src/flowsint_transforms/emails/to_gravatar.py b/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py
similarity index 100%
rename from flowsint-transforms/src/flowsint_transforms/emails/to_gravatar.py
rename to flowsint-transforms/src/flowsint_transforms/email/to_gravatar.py
diff --git a/flowsint-transforms/src/flowsint_transforms/emails/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py
similarity index 100%
rename from flowsint-transforms/src/flowsint_transforms/emails/to_leaks.py
rename to flowsint-transforms/src/flowsint_transforms/email/to_leaks.py
diff --git a/flowsint-transforms/src/flowsint_transforms/individuals/to_org.py b/flowsint-transforms/src/flowsint_transforms/individual/to_org.py
similarity index 100%
rename from flowsint-transforms/src/flowsint_transforms/individuals/to_org.py
rename to flowsint-transforms/src/flowsint_transforms/individual/to_org.py
diff --git a/flowsint-transforms/src/flowsint_transforms/ips/ip_to_asn.py b/flowsint-transforms/src/flowsint_transforms/ip/to_asn.py
similarity index 100%
rename from flowsint-transforms/src/flowsint_transforms/ips/ip_to_asn.py
rename to flowsint-transforms/src/flowsint_transforms/ip/to_asn.py
diff --git a/flowsint-transforms/src/flowsint_transforms/ips/reverse_resolve.py b/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py
similarity index 98%
rename from flowsint-transforms/src/flowsint_transforms/ips/reverse_resolve.py
rename to flowsint-transforms/src/flowsint_transforms/ip/to_domain.py
index 1f9542bb..0989b673 100644
--- a/flowsint-transforms/src/flowsint_transforms/ips/reverse_resolve.py
+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py
@@ -20,7 +20,7 @@ class ReverseResolveScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "ip_reverse_resolve_scanner"
+ return "ip_to_domain_scanner"
@classmethod
def category(cls) -> str:
diff --git a/flowsint-transforms/src/flowsint_transforms/ips/geolocation.py b/flowsint-transforms/src/flowsint_transforms/ip/to_geolocation.py
similarity index 99%
rename from flowsint-transforms/src/flowsint_transforms/ips/geolocation.py
rename to flowsint-transforms/src/flowsint_transforms/ip/to_geolocation.py
index 332b2060..da1509cf 100644
--- a/flowsint-transforms/src/flowsint_transforms/ips/geolocation.py
+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_geolocation.py
@@ -18,7 +18,7 @@ class GeolocationScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "ip_geolocation_scanner"
+ return "ip_to_geolocation_scanner"
@classmethod
def category(cls) -> str:
diff --git a/flowsint-transforms/src/flowsint_transforms/leaks/hibp.py b/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.py
similarity index 100%
rename from flowsint-transforms/src/flowsint_transforms/leaks/hibp.py
rename to flowsint-transforms/src/flowsint_transforms/leak/to_leaks.py
diff --git a/flowsint-transforms/src/flowsint_transforms/organizations/__init__.py b/flowsint-transforms/src/flowsint_transforms/organization/__init__.py
similarity index 100%
rename from flowsint-transforms/src/flowsint_transforms/organizations/__init__.py
rename to flowsint-transforms/src/flowsint_transforms/organization/__init__.py
diff --git a/flowsint-transforms/src/flowsint_transforms/organizations/org_to_asn.py b/flowsint-transforms/src/flowsint_transforms/organization/to_asn.py
similarity index 100%
rename from flowsint-transforms/src/flowsint_transforms/organizations/org_to_asn.py
rename to flowsint-transforms/src/flowsint_transforms/organization/to_asn.py
diff --git a/flowsint-transforms/src/flowsint_transforms/organizations/to_infos.py b/flowsint-transforms/src/flowsint_transforms/organization/to_infos.py
similarity index 99%
rename from flowsint-transforms/src/flowsint_transforms/organizations/to_infos.py
rename to flowsint-transforms/src/flowsint_transforms/organization/to_infos.py
index a9be4cf6..db3dd152 100644
--- a/flowsint-transforms/src/flowsint_transforms/organizations/to_infos.py
+++ b/flowsint-transforms/src/flowsint_transforms/organization/to_infos.py
@@ -14,7 +14,7 @@ class OrgToInfosScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "to_infos"
+ return "org_to_infos"
@classmethod
def category(cls) -> str:
diff --git a/flowsint-transforms/src/flowsint_transforms/phones/ignorant.py b/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py
similarity index 99%
rename from flowsint-transforms/src/flowsint_transforms/phones/ignorant.py
rename to flowsint-transforms/src/flowsint_transforms/phone/to_infos.py
index 9c5cb65c..c4cfb3d4 100644
--- a/flowsint-transforms/src/flowsint_transforms/phones/ignorant.py
+++ b/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py
@@ -14,7 +14,7 @@ class IgnorantScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "ignorant_scanner"
+ return "phone_to_infos_scanner"
@classmethod
def category(cls) -> str:
diff --git a/flowsint-transforms/src/flowsint_transforms/socials/maigret.py b/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py
similarity index 100%
rename from flowsint-transforms/src/flowsint_transforms/socials/maigret.py
rename to flowsint-transforms/src/flowsint_transforms/social/to_maigret.py
diff --git a/flowsint-transforms/src/flowsint_transforms/socials/sherlock.py b/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py
similarity index 100%
rename from flowsint-transforms/src/flowsint_transforms/socials/sherlock.py
rename to flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py
diff --git a/flowsint-transforms/src/flowsint_transforms/websites/to_crawler.py b/flowsint-transforms/src/flowsint_transforms/website/to_crawler.py
similarity index 99%
rename from flowsint-transforms/src/flowsint_transforms/websites/to_crawler.py
rename to flowsint-transforms/src/flowsint_transforms/website/to_crawler.py
index 72d35300..ab2fb295 100644
--- a/flowsint-transforms/src/flowsint_transforms/websites/to_crawler.py
+++ b/flowsint-transforms/src/flowsint_transforms/website/to_crawler.py
@@ -24,7 +24,7 @@ class WebsiteToCrawler(Scanner):
@classmethod
def name(cls) -> str:
- return "to_crawler"
+ return "website_to_crawler"
@classmethod
def category(cls) -> str:
diff --git a/flowsint-transforms/src/flowsint_transforms/websites/to_domain.py b/flowsint-transforms/src/flowsint_transforms/website/to_domain.py
similarity index 63%
rename from flowsint-transforms/src/flowsint_transforms/websites/to_domain.py
rename to flowsint-transforms/src/flowsint_transforms/website/to_domain.py
index 0d1403e5..a9e0e5a5 100644
--- a/flowsint-transforms/src/flowsint_transforms/websites/to_domain.py
+++ b/flowsint-transforms/src/flowsint_transforms/website/to_domain.py
@@ -15,7 +15,7 @@ class WebsiteToDomainScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "to_domain"
+ return "website_to_domain"
@classmethod
def category(cls) -> str:
@@ -43,7 +43,7 @@ class WebsiteToDomainScanner(Scanner):
results: OutputType = []
for website in data:
try:
- parsed_url = urlparse(website.url)
+ parsed_url = urlparse(str(website.url))
domain_name = parsed_url.netloc
# Remove port if present
@@ -62,7 +62,7 @@ class WebsiteToDomainScanner(Scanner):
Logger.error(
self.sketch_id,
{
- "message": f"Error extracting domain from website {website.url}: {e}"
+ "message": f"Error extracting domain from website {str(website.url)}: {e}"
},
)
continue
@@ -72,6 +72,40 @@ class WebsiteToDomainScanner(Scanner):
def postprocess(
self, results: OutputType, input_data: InputType = None
) -> OutputType:
+ # Create Neo4j relationships between websites and their corresponding domains
+ if input_data and self.neo4j_conn:
+ for input_website, result in zip(input_data, results):
+ website_url = str(input_website.url)
+ domain_name = result.domain
+
+ self.create_node(
+ "website",
+ "url",
+ website_url,
+ caption=website_url,
+ type="website",
+ )
+
+ # Create relationship with the specific domain for this website
+ self.create_node(
+ "domain",
+ "domain",
+ domain_name,
+ caption=domain_name,
+ type="domain",
+ )
+ self.create_relationship(
+ "website",
+ "url",
+ website_url,
+ "domain",
+ "domain",
+ domain_name,
+ "HAS_DOMAIN",
+ )
+ self.log_graph_message(
+ f"Extracted domain {domain_name} from website {website_url}."
+ )
return results
diff --git a/flowsint-transforms/src/flowsint_transforms/websites/to_links.py b/flowsint-transforms/src/flowsint_transforms/website/to_links.py
similarity index 99%
rename from flowsint-transforms/src/flowsint_transforms/websites/to_links.py
rename to flowsint-transforms/src/flowsint_transforms/website/to_links.py
index fa465eaf..bc90ee31 100644
--- a/flowsint-transforms/src/flowsint_transforms/websites/to_links.py
+++ b/flowsint-transforms/src/flowsint_transforms/website/to_links.py
@@ -15,7 +15,7 @@ class WebsiteToLinks(Scanner):
@classmethod
def name(cls) -> str:
- return "to_links"
+ return "website_to_links"
@classmethod
def category(cls) -> str:
diff --git a/flowsint-transforms/src/flowsint_transforms/website/to_text.py b/flowsint-transforms/src/flowsint_transforms/website/to_text.py
new file mode 100644
index 00000000..1913ac1b
--- /dev/null
+++ b/flowsint-transforms/src/flowsint_transforms/website/to_text.py
@@ -0,0 +1,103 @@
+from typing import List, Union
+from flowsint_core.core.scanner_base import Scanner
+from flowsint_types.phrase import Phrase
+from flowsint_types.website import Website
+import requests
+from bs4 import BeautifulSoup
+
+
+class WebsiteToText(Scanner):
+ """Extracts the texts in a webpage."""
+
+ InputType = List[Website]
+ OutputType = List[Phrase]
+
+ @classmethod
+ def name(cls) -> str:
+ return "website_to_text"
+
+ @classmethod
+ def category(cls) -> str:
+ return "Website"
+
+ @classmethod
+ def key(cls) -> str:
+ return "website"
+
+ def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:
+ cleaned: InputType = []
+ for item in data:
+ website_obj = None
+ if isinstance(item, str):
+ website_obj = Website(url=item)
+ elif isinstance(item, dict) and "url" in item:
+ website_obj = Website(url=item["url"])
+ elif isinstance(item, Website):
+ website_obj = item
+ if website_obj:
+ cleaned.append(website_obj)
+ return cleaned
+
+ async def scan(self, data: InputType) -> OutputType:
+ results: OutputType = []
+ for website in data:
+ text_data = self._extract_text(website.url)
+ if text_data:
+ phrase_obj = Phrase(text=text_data)
+ results.append(phrase_obj)
+ return results
+
+ def _extract_text(self, website_url: str) -> str:
+ try:
+ response = requests.get(website_url)
+ response.raise_for_status()
+ soup = BeautifulSoup(response.text, "html.parser")
+ text = soup.get_text()
+ return text
+ except requests.exceptions.RequestException as e:
+ print(f"Error fetching the URL: {e}")
+ return None
+ except Exception as e:
+ print(f"An error occurred: {e}")
+ return None
+
+ def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ # Create Neo4j relationships between websites and their corresponding phrases
+ for input_website, result in zip(original_input, results):
+ website_url = str(input_website.url)
+
+ if self.neo4j_conn:
+ self.create_node(
+ "website",
+ "url",
+ str(website_url),
+ caption=str(website_url),
+ type="website",
+ )
+
+ # Create relationship with the specific phrase for this website
+ self.create_node(
+ "phrase",
+ "text",
+ result.text,
+ caption=result.text,
+ type="phrase",
+ )
+ self.create_relationship(
+ "website",
+ "url",
+ website_url,
+ "phrase",
+ "text",
+ result.text,
+ "HAS_INNER_TEXT",
+ )
+ self.log_graph_message(
+ f"Extracted some text from the website {website_url}."
+ )
+ return results
+
+
+# Make types available at module level for easy access
+InputType = WebsiteToText.InputType
+OutputType = WebsiteToText.OutputType
diff --git a/flowsint-transforms/src/flowsint_transforms/websites/to_webtrackers.py b/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py
similarity index 99%
rename from flowsint-transforms/src/flowsint_transforms/websites/to_webtrackers.py
rename to flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py
index 5e5a98f2..4a7e16ad 100644
--- a/flowsint-transforms/src/flowsint_transforms/websites/to_webtrackers.py
+++ b/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py
@@ -29,7 +29,7 @@ class WebsiteToWebtrackersScanner(Scanner):
@classmethod
def name(cls) -> str:
- return "to_webtrackers"
+ return "website_to_webtrackers"
@classmethod
def category(cls) -> str:
diff --git a/flowsint-transforms/tests/scanners/domains/__init__.py b/flowsint-transforms/tests/scanners/domain/__init__.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/domains/__init__.py
rename to flowsint-transforms/tests/scanners/domain/__init__.py
diff --git a/flowsint-transforms/tests/scanners/domains/resolve.py b/flowsint-transforms/tests/scanners/domain/resolve.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/domains/resolve.py
rename to flowsint-transforms/tests/scanners/domain/resolve.py
diff --git a/flowsint-transforms/tests/scanners/domains/subdomains.py b/flowsint-transforms/tests/scanners/domain/subdomains.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/domains/subdomains.py
rename to flowsint-transforms/tests/scanners/domain/subdomains.py
diff --git a/flowsint-transforms/tests/scanners/domains/whois.py b/flowsint-transforms/tests/scanners/domain/whois.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/domains/whois.py
rename to flowsint-transforms/tests/scanners/domain/whois.py
diff --git a/flowsint-transforms/tests/scanners/emails/__init__.py b/flowsint-transforms/tests/scanners/email/__init__.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/emails/__init__.py
rename to flowsint-transforms/tests/scanners/email/__init__.py
diff --git a/flowsint-transforms/tests/scanners/emails/to_gravatar.py b/flowsint-transforms/tests/scanners/email/to_gravatar.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/emails/to_gravatar.py
rename to flowsint-transforms/tests/scanners/email/to_gravatar.py
diff --git a/flowsint-transforms/tests/scanners/emails/to_leaks.py b/flowsint-transforms/tests/scanners/email/to_leaks.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/emails/to_leaks.py
rename to flowsint-transforms/tests/scanners/email/to_leaks.py
diff --git a/flowsint-transforms/tests/scanners/ips/__init__.py b/flowsint-transforms/tests/scanners/ip/__init__.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/ips/__init__.py
rename to flowsint-transforms/tests/scanners/ip/__init__.py
diff --git a/flowsint-transforms/tests/scanners/ips/asn_to_cidrs.py b/flowsint-transforms/tests/scanners/ip/asn_to_cidrs.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/ips/asn_to_cidrs.py
rename to flowsint-transforms/tests/scanners/ip/asn_to_cidrs.py
diff --git a/flowsint-transforms/tests/scanners/ips/cidr_to_ips.py b/flowsint-transforms/tests/scanners/ip/cidr_to_ips.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/ips/cidr_to_ips.py
rename to flowsint-transforms/tests/scanners/ip/cidr_to_ips.py
diff --git a/flowsint-transforms/tests/scanners/ips/geolocation.py b/flowsint-transforms/tests/scanners/ip/geolocation.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/ips/geolocation.py
rename to flowsint-transforms/tests/scanners/ip/geolocation.py
diff --git a/flowsint-transforms/tests/scanners/ips/ip_to_asn.py b/flowsint-transforms/tests/scanners/ip/ip_to_asn.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/ips/ip_to_asn.py
rename to flowsint-transforms/tests/scanners/ip/ip_to_asn.py
diff --git a/flowsint-transforms/tests/scanners/organizations/__init__.py b/flowsint-transforms/tests/scanners/organization/__init__.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/organizations/__init__.py
rename to flowsint-transforms/tests/scanners/organization/__init__.py
diff --git a/flowsint-transforms/tests/scanners/organizations/to_infos.py b/flowsint-transforms/tests/scanners/organization/to_infos.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/organizations/to_infos.py
rename to flowsint-transforms/tests/scanners/organization/to_infos.py
diff --git a/flowsint-transforms/tests/scanners/socials/__init__.py b/flowsint-transforms/tests/scanners/social/__init__.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/socials/__init__.py
rename to flowsint-transforms/tests/scanners/social/__init__.py
diff --git a/flowsint-transforms/tests/scanners/socials/maigret.py b/flowsint-transforms/tests/scanners/social/maigret.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/socials/maigret.py
rename to flowsint-transforms/tests/scanners/social/maigret.py
diff --git a/flowsint-transforms/tests/scanners/websites/__init__.py b/flowsint-transforms/tests/scanners/website/__init__.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/websites/__init__.py
rename to flowsint-transforms/tests/scanners/website/__init__.py
diff --git a/flowsint-transforms/tests/scanners/websites/to_links.py b/flowsint-transforms/tests/scanners/website/to_links.py
similarity index 100%
rename from flowsint-transforms/tests/scanners/websites/to_links.py
rename to flowsint-transforms/tests/scanners/website/to_links.py
diff --git a/flowsint-types/src/flowsint_types/organization.py b/flowsint-types/src/flowsint_types/organization.py
index 33302bbb..e7562227 100644
--- a/flowsint-types/src/flowsint_types/organization.py
+++ b/flowsint-types/src/flowsint_types/organization.py
@@ -1,5 +1,5 @@
from pydantic import BaseModel, Field
-from typing import Optional, List
+from typing import Any, Optional, List
from .individual import Individual
from .address import PhysicalAddress
@@ -8,15 +8,15 @@ class Organization(BaseModel):
"""Represents an organization with detailed business and administrative information."""
# Basic information
- name: str = Field(..., description="Organization name", title="Organization Name")
- siren: Optional[str] = Field(None, description="SIREN number", title="SIREN Number")
- nom_complet: Optional[str] = Field(
+ name: Any = Field(..., description="Organization name", title="Organization Name")
+ siren: Optional[Any] = Field(None, description="SIREN number", title="SIREN Number")
+ nom_complet: Optional[Any] = Field(
None, description="Complete name", title="Complete Name"
)
- nom_raison_sociale: Optional[str] = Field(
+ nom_raison_sociale: Optional[Any] = Field(
None, description="Company name", title="Company Name"
)
- sigle: Optional[str] = Field(None, description="Acronym", title="Acronym")
+ sigle: Optional[Any] = Field(None, description="Acronym", title="Acronym")
# Establishment information
nombre_etablissements: Optional[int] = Field(
None, description="Number of establishments", title="Number of Establishments"
@@ -27,228 +27,228 @@ class Organization(BaseModel):
title="Number of Open Establishments",
)
# Activity information
- activite_principale: Optional[str] = Field(
+ activite_principale: Optional[Any] = Field(
None, description="Main activity code", title="Main Activity Code"
)
- section_activite_principale: Optional[str] = Field(
+ section_activite_principale: Optional[Any] = Field(
None, description="Main activity section", title="Main Activity Section"
)
# Company category
- categorie_entreprise: Optional[str] = Field(
+ categorie_entreprise: Optional[Any] = Field(
None, description="Company category", title="Company Category"
)
- annee_categorie_entreprise: Optional[str] = Field(
+ annee_categorie_entreprise: Optional[Any] = Field(
None, description="Year of company category", title="Company Category Year"
)
# Employment information
- caractere_employeur: Optional[str] = Field(
+ caractere_employeur: Optional[Any] = Field(
None, description="Employer character", title="Employer Character"
)
- tranche_effectif_salarie: Optional[str] = Field(
+ tranche_effectif_salarie: Optional[Any] = Field(
None, description="Employee size range", title="Employee Size Range"
)
- annee_tranche_effectif_salarie: Optional[str] = Field(
+ annee_tranche_effectif_salarie: Optional[Any] = Field(
None,
description="Year of employee size range",
title="Employee Size Range Year",
)
# Dates
- date_creation: Optional[str] = Field(
+ date_creation: Optional[Any] = Field(
None, description="Creation date", title="Creation Date"
)
- date_fermeture: Optional[str] = Field(
+ date_fermeture: Optional[Any] = Field(
None, description="Closing date", title="Closing Date"
)
- date_mise_a_jour: Optional[str] = Field(
+ date_mise_a_jour: Optional[Any] = Field(
None, description="Update date", title="Update Date"
)
- date_mise_a_jour_insee: Optional[str] = Field(
+ date_mise_a_jour_insee: Optional[Any] = Field(
None, description="INSEE update date", title="INSEE Update Date"
)
- date_mise_a_jour_rne: Optional[str] = Field(
+ date_mise_a_jour_rne: Optional[Any] = Field(
None, description="RNE update date", title="RNE Update Date"
)
# Legal information
- nature_juridique: Optional[str] = Field(
+ nature_juridique: Optional[Any] = Field(
None, description="Legal nature", title="Legal Nature"
)
- etat_administratif: Optional[str] = Field(
- None, description="Administrative status", title="Administrative Status"
+ etat_adminiAnyatif: Optional[Any] = Field(
+ None, description="AdminiAnyative status", title="AdminiAnyative Status"
)
- statut_diffusion: Optional[str] = Field(
+ statut_diffusion: Optional[Any] = Field(
None, description="Diffusion status", title="Diffusion Status"
)
# Siege (Headquarters) information
- siege_activite_principale: Optional[str] = Field(
+ siege_activite_principale: Optional[Any] = Field(
None, description="Siege main activity code", title="Headquarters Main Activity"
)
- siege_activite_principale_registre_metier: Optional[str] = Field(
+ siege_activite_principale_regiAnye_metier: Optional[Any] = Field(
None,
description="Siege main activity in trade register",
title="Headquarters Trade Register Activity",
)
- siege_annee_tranche_effectif_salarie: Optional[str] = Field(
+ siege_annee_tranche_effectif_salarie: Optional[Any] = Field(
None,
description="Siege year of employee size range",
title="Headquarters Employee Size Year",
)
- siege_adresse: Optional[str] = Field(
+ siege_adresse: Optional[Any] = Field(
None, description="Siege address", title="Headquarters Address"
)
- siege_caractere_employeur: Optional[str] = Field(
+ siege_caractere_employeur: Optional[Any] = Field(
None,
description="Siege employer character",
title="Headquarters Employer Character",
)
- siege_cedex: Optional[str] = Field(
+ siege_cedex: Optional[Any] = Field(
None, description="Siege CEDEX code", title="Headquarters CEDEX"
)
- siege_code_pays_etranger: Optional[str] = Field(
+ siege_code_pays_etranger: Optional[Any] = Field(
None,
description="Siege foreign country code",
title="Headquarters Foreign Country Code",
)
- siege_code_postal: Optional[str] = Field(
+ siege_code_postal: Optional[Any] = Field(
None, description="Siege postal code", title="Headquarters Postal Code"
)
- siege_commune: Optional[str] = Field(
+ siege_commune: Optional[Any] = Field(
None,
description="Siege municipality code",
title="Headquarters Municipality Code",
)
- siege_complement_adresse: Optional[str] = Field(
+ siege_complement_adresse: Optional[Any] = Field(
None,
description="Siege address complement",
title="Headquarters Address Complement",
)
- siege_coordonnees: Optional[str] = Field(
+ siege_coordonnees: Optional[Any] = Field(
None, description="Siege coordinates", title="Headquarters Coordinates"
)
- siege_date_creation: Optional[str] = Field(
+ siege_date_creation: Optional[Any] = Field(
None, description="Siege creation date", title="Headquarters Creation Date"
)
- siege_date_debut_activite: Optional[str] = Field(
+ siege_date_debut_activite: Optional[Any] = Field(
None,
description="Siege activity start date",
title="Headquarters Activity Start Date",
)
- siege_date_fermeture: Optional[str] = Field(
+ siege_date_fermeture: Optional[Any] = Field(
None, description="Siege closing date", title="Headquarters Closing Date"
)
- siege_date_mise_a_jour: Optional[str] = Field(
+ siege_date_mise_a_jour: Optional[Any] = Field(
None, description="Siege update date", title="Headquarters Update Date"
)
- siege_date_mise_a_jour_insee: Optional[str] = Field(
+ siege_date_mise_a_jour_insee: Optional[Any] = Field(
None,
description="Siege INSEE update date",
title="Headquarters INSEE Update Date",
)
- siege_departement: Optional[str] = Field(
+ siege_departement: Optional[Any] = Field(
None, description="Siege department code", title="Headquarters Department"
)
- siege_distribution_speciale: Optional[str] = Field(
+ siege_diAnyibution_speciale: Optional[Any] = Field(
None,
- description="Siege special distribution",
- title="Headquarters Special Distribution",
+ description="Siege special diAnyibution",
+ title="Headquarters Special DiAnyibution",
)
- siege_epci: Optional[str] = Field(
+ siege_epci: Optional[Any] = Field(
None, description="Siege EPCI code", title="Headquarters EPCI Code"
)
siege_est_siege: Optional[bool] = Field(
None, description="Siege is headquarters", title="Is Headquarters"
)
- siege_etat_administratif: Optional[str] = Field(
+ siege_etat_adminiAnyatif: Optional[Any] = Field(
None,
- description="Siege administrative status",
- title="Headquarters Administrative Status",
+ description="Siege adminiAnyative status",
+ title="Headquarters AdminiAnyative Status",
)
siege_geo_adresse: Optional[PhysicalAddress] = Field(
None,
description="Siege geocoded address",
title="Headquarters Geocoded Address",
)
- siege_geo_id: Optional[str] = Field(
+ siege_geo_id: Optional[Any] = Field(
None, description="Siege geographic ID", title="Headquarters Geographic ID"
)
- siege_indice_repetition: Optional[str] = Field(
+ siege_indice_repetition: Optional[Any] = Field(
None,
description="Siege repetition index",
title="Headquarters Repetition Index",
)
- siege_latitude: Optional[str] = Field(
+ siege_latitude: Optional[Any] = Field(
None, description="Siege latitude", title="Headquarters Latitude"
)
- siege_libelle_cedex: Optional[str] = Field(
+ siege_libelle_cedex: Optional[Any] = Field(
None, description="Siege CEDEX label", title="Headquarters CEDEX Label"
)
- siege_libelle_commune: Optional[str] = Field(
+ siege_libelle_commune: Optional[Any] = Field(
None,
description="Siege municipality label",
title="Headquarters Municipality Label",
)
- siege_libelle_commune_etranger: Optional[str] = Field(
+ siege_libelle_commune_etranger: Optional[Any] = Field(
None,
description="Siege foreign municipality label",
title="Headquarters Foreign Municipality Label",
)
- siege_libelle_pays_etranger: Optional[str] = Field(
+ siege_libelle_pays_etranger: Optional[Any] = Field(
None,
description="Siege foreign country label",
title="Headquarters Foreign Country Label",
)
- siege_libelle_voie: Optional[str] = Field(
- None, description="Siege street label", title="Headquarters Street Label"
+ siege_libelle_voie: Optional[Any] = Field(
+ None, description="Siege Anyeet label", title="Headquarters Anyeet Label"
)
- siege_liste_enseignes: Optional[List[str]] = Field(
+ siege_liste_enseignes: Optional[List[Any]] = Field(
None, description="Siege list of brands", title="Headquarters Brands List"
)
- siege_liste_finess: Optional[List[str]] = Field(
+ siege_liste_finess: Optional[List[Any]] = Field(
None, description="Siege list of FINESS", title="Headquarters FINESS List"
)
- siege_liste_id_bio: Optional[List[str]] = Field(
+ siege_liste_id_bio: Optional[List[Any]] = Field(
None, description="Siege list of bio IDs", title="Headquarters Bio IDs List"
)
- siege_liste_idcc: Optional[List[str]] = Field(
+ siege_liste_idcc: Optional[List[Any]] = Field(
None, description="Siege list of IDCC", title="Headquarters IDCC List"
)
- siege_liste_id_organisme_formation: Optional[List[str]] = Field(
+ siege_liste_id_organisme_formation: Optional[List[Any]] = Field(
None,
description="Siege list of training organization IDs",
title="Headquarters Training Organization IDs",
)
- siege_liste_rge: Optional[List[str]] = Field(
+ siege_liste_rge: Optional[List[Any]] = Field(
None, description="Siege list of RGE", title="Headquarters RGE List"
)
- siege_liste_uai: Optional[List[str]] = Field(
+ siege_liste_uai: Optional[List[Any]] = Field(
None, description="Siege list of UAI", title="Headquarters UAI List"
)
- siege_longitude: Optional[str] = Field(
+ siege_longitude: Optional[Any] = Field(
None, description="Siege longitude", title="Headquarters Longitude"
)
- siege_nom_commercial: Optional[str] = Field(
+ siege_nom_commercial: Optional[Any] = Field(
None, description="Siege commercial name", title="Headquarters Commercial Name"
)
- siege_numero_voie: Optional[str] = Field(
- None, description="Siege street number", title="Headquarters Street Number"
+ siege_numero_voie: Optional[Any] = Field(
+ None, description="Siege Anyeet number", title="Headquarters Anyeet Number"
)
- siege_region: Optional[str] = Field(
+ siege_region: Optional[Any] = Field(
None, description="Siege region code", title="Headquarters Region"
)
- siege_siret: Optional[str] = Field(
+ siege_siret: Optional[Any] = Field(
None, description="Siege SIRET number", title="Headquarters SIRET"
)
- siege_statut_diffusion_etablissement: Optional[str] = Field(
+ siege_statut_diffusion_etablissement: Optional[Any] = Field(
None,
description="Siege establishment diffusion status",
title="Headquarters Establishment Diffusion Status",
)
- siege_tranche_effectif_salarie: Optional[str] = Field(
+ siege_tranche_effectif_salarie: Optional[Any] = Field(
None,
description="Siege employee size range",
title="Headquarters Employee Size Range",
)
- siege_type_voie: Optional[str] = Field(
- None, description="Siege street type", title="Headquarters Street Type"
+ siege_type_voie: Optional[Any] = Field(
+ None, description="Siege Anyeet type", title="Headquarters Anyeet Type"
)
# Dirigeants (Leaders) - using Individual type
dirigeants: Optional[List[Individual]] = Field(
@@ -263,7 +263,7 @@ class Organization(BaseModel):
None, description="Financial information", title="Financial Information"
)
# Complements (Additional information)
- complements_collectivite_territoriale: Optional[str] = Field(
+ complements_collectivite_territoriale: Optional[Any] = Field(
None,
description="Complements territorial community",
title="Territorial Community",
@@ -273,7 +273,7 @@ class Organization(BaseModel):
description="Complements collective agreement informed",
title="Collective Agreement Informed",
)
- complements_liste_idcc: Optional[List[str]] = Field(
+ complements_liste_idcc: Optional[List[Any]] = Field(
None, description="Complements list of IDCC", title="IDCC List"
)
complements_egapro_renseignee: Optional[bool] = Field(
@@ -317,7 +317,7 @@ class Organization(BaseModel):
complements_est_qualiopi: Optional[bool] = Field(
None, description="Complements is Qualiopi", title="Is Qualiopi"
)
- complements_liste_id_organisme_formation: Optional[List[str]] = Field(
+ complements_liste_id_organisme_formation: Optional[List[Any]] = Field(
None,
description="Complements list of training organization IDs",
title="Training Organization IDs",
@@ -348,16 +348,16 @@ class Organization(BaseModel):
description="Complements GES balance informed",
title="GES Balance Informed",
)
- complements_identifiant_association: Optional[str] = Field(
+ complements_identifiant_association: Optional[Any] = Field(
None,
description="Complements association identifier",
title="Association Identifier",
)
- complements_statut_entrepreneur_spectacle: Optional[str] = Field(
+ complements_statut_entrepreneur_spectacle: Optional[Any] = Field(
None,
description="Complements show entrepreneur status",
title="Show Entrepreneur Status",
)
- complements_type_siae: Optional[str] = Field(
+ complements_type_siae: Optional[Any] = Field(
None, description="Complements SIAE type", title="SIAE Type"
)
diff --git a/test.py b/test.py
new file mode 100644
index 00000000..fe862cc3
--- /dev/null
+++ b/test.py
@@ -0,0 +1,30 @@
+import requests
+from bs4 import BeautifulSoup
+
+def extract_text_from_url(url):
+ try:
+ # Send a GET request to the URL
+ response = requests.get(url)
+ response.raise_for_status() # Raise an exception for HTTP errors
+
+ # Parse the HTML content using BeautifulSoup
+ soup = BeautifulSoup(response.text, 'html.parser')
+
+ # Extract all the text from the page
+ text = soup.get_text()
+
+ return text
+ except requests.exceptions.RequestException as e:
+ print(f"Error fetching the URL: {e}")
+ return None
+ except Exception as e:
+ print(f"An error occurred: {e}")
+ return None
+
+# Example usage
+if __name__ == "__main__":
+ url = input("Enter the URL of the webpage: ")
+ extracted_text = extract_text_from_url(url)
+ if extracted_text:
+ print("Extracted Text:")
+ print(extracted_text)
\ No newline at end of file