A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat: filters
dextmorgn49c455c47780
diff --git a/.env.example b/.env.exampleindex a66fd592..fe25c03b 100644--- a/.env.example+++ b/.env.example@@ -1,6 +1,7 @@NODE_ENV=productionNEXT_PUBLIC_AUTH_REDIRECT=http://app.flowsint.localhost/auth/callbackHIBP_API_KEY=+ETHERSCAN_API_KEY=NEXT_PUBLIC_FLOWSINT_API=https://api.flowsint.localhost/apiNEXT_PUBLIC_DOCKER_FLOWSINT_API=http://flowsint-api:5000/apiAUTH_SECRET="superscretchangeitplz"diff --git a/flowsint-api/app/api/routes/transforms.py b/flowsint-api/app/api/routes/transforms.pyindex d44d3caf..56b55c29 100644--- a/flowsint-api/app/api/routes/transforms.py+++ b/flowsint-api/app/api/routes/transforms.py@@ -39,9 +39,11 @@ def get_transforms(# current_user: Profile = Depends(get_current_user),):if category is not None and category != "undefined":- transforms = TransformRegistry.list_by_input_type(category)+ transforms = TransformRegistry.list_by_input_type(+ category, exclude=["n8n_connector"]+ )else:- transforms = TransformRegistry.list()+ transforms = TransformRegistry.list(exclude=["n8n_conector"])return transformsdiff --git a/flowsint-api/app/api/routes/types.py b/flowsint-api/app/api/routes/types.pyindex e98e5f02..3f5dab46 100644--- a/flowsint-api/app/api/routes/types.py+++ b/flowsint-api/app/api/routes/types.py@@ -1,5 +1,5 @@from typing import Any, Dict, Optional, Type-from uuid import UUID, uuid4+from uuid import uuid4from fastapi import APIRouterfrom pydantic import BaseModel, TypeAdapterfrom flowsint_types import (diff --git a/flowsint-app/src/renderer/src/components/flows/flow-sheet.tsx b/flowsint-app/src/renderer/src/components/flows/flow-sheet.tsxindex eb411bee..0a9f1511 100644--- a/flowsint-app/src/renderer/src/components/flows/flow-sheet.tsx+++ b/flowsint-app/src/renderer/src/components/flows/flow-sheet.tsx@@ -13,11 +13,11 @@ import { TooltipProvider, Tooltip, TooltipContent, TooltipTrigger } from "@/compimport { useNodesDisplaySettings } from "@/stores/node-display-settings"import { Scanner } from "@/types/transform"import { useQuery } from "@tanstack/react-query"-import { transformService } from "@/api/transfrom-service"import { Input } from "../ui/input"import { Alert, AlertDescription } from "../ui/alert"import { categoryColors } from "./scanner-data"import { useIcon } from "@/hooks/use-icon"+import { flowService } from "@/api/flow-service"const FlowSheet = ({ onLayout }: { onLayout: () => void }) => {const openFlowSheet = useFlowStore(state => state.openFlowSheet)@@ -29,7 +29,7 @@ const FlowSheet = ({ onLayout }: { onLayout: () => void }) => {const { data: materials, isLoading, error } = useQuery({queryKey: ["raw_material", selectedNode?.data.outputs.type],enabled: !!selectedNode?.data.outputs.type,- queryFn: () => transformService.getFlowsRawMaterialForType(selectedNode?.data.outputs.type.toLowerCase() || ""),+ queryFn: () => flowService.getRawMaterialForType(selectedNode?.data.outputs.type.toLowerCase() || ""),})const [searchTerm, setSearchTerm] = useState<string>("")diff --git a/flowsint-app/src/renderer/src/components/flows/new-flow.tsx b/flowsint-app/src/renderer/src/components/flows/new-flow.tsxindex c9213f79..96d10148 100644--- a/flowsint-app/src/renderer/src/components/flows/new-flow.tsx+++ b/flowsint-app/src/renderer/src/components/flows/new-flow.tsx@@ -1,8 +1,8 @@-import { transformService } from '@/api/transfrom-service'import { useCallback, ReactNode, cloneElement, isValidElement } from 'react'import { toast } from 'sonner'import { useNavigate } from '@tanstack/react-router'import { useFlowStore } from '@/stores/flow-store'+import { flowService } from '@/api/flow-service'const NewFlow = ({ children }: { children: ReactNode }) => {const navigate = useNavigate()@@ -15,7 +15,7 @@ const NewFlow = ({ children }: { children: ReactNode }) => {// Reset nodes and edgessetNodes([])setEdges([])- const response = await transformService.create(JSON.stringify({+ const response = await flowService.create(JSON.stringify({name: "New flow",description: "A new example flow.",category: [],@@ -27,7 +27,7 @@ const NewFlow = ({ children }: { children: ReactNode }) => {{loading: 'Creating transform...',success: 'Transform created successfully.',- error: 'Failed to create transform.'+ error: 'Failed to create flow.'})diff --git a/flowsint-app/src/renderer/src/components/graphs/filters.tsx b/flowsint-app/src/renderer/src/components/graphs/filters.tsxnew file mode 100644index 00000000..41828b1d--- /dev/null+++ b/flowsint-app/src/renderer/src/components/graphs/filters.tsx@@ -0,0 +1,70 @@+import { useCallback } from 'react';+import { useGraphStore } from '@/stores/graph-store';+import { useActionItems } from '@/hooks/use-action-items';+import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuLabel, DropdownMenuSeparator, DropdownMenuTrigger } from '../ui/dropdown-menu';+import { Button } from '../ui/button';+import { XIcon } from 'lucide-react';+import { cn, getAllNodeTypes } from '@/lib/utils';+import { Badge } from '../ui/badge';++const Filters = ({ children }: { children: React.ReactNode }) => {+ const filters = useGraphStore(s => s.filters);+ const setFilters = useGraphStore(s => s.setFilters);+ const { actionItems } = useActionItems()++ const clearFilters = useCallback(() => {+ setFilters(null)+ }, [setFilters, filters])++ const toggleFilter = useCallback((type: string | null) => {+ if (type === null) {+ clearFilters()+ } else {+ const currentTypeFilters = filters || []+ const isChecked = currentTypeFilters.includes(type)+ const newTypeFilters = isChecked ? currentTypeFilters.filter((t: string) => t !== type) : [...currentTypeFilters, type.toLowerCase()]+ setFilters(newTypeFilters)+ }+ }, [setFilters, filters, clearFilters])++ return (+ <DropdownMenu>+ <div>+ <DropdownMenuTrigger asChild>+ <div>+ {children}+ </div>+ </DropdownMenuTrigger>+ </div>+ <DropdownMenuContent className="w-48 max-h-[50vh] space-y-1 overflow-y-auto">+ <DropdownMenuLabel>+ {filters?.length ? (+ <Badge variant={"outline"} className="flex items-center gap-1 pr-1">+ {filters?.length} filter(s)+ <Button size={"icon"} variant={"ghost"} className="h-5 w-5 rounded-full" onClick={clearFilters}>+ <XIcon className="h-3 w-3" />+ </Button>+ </Badge>+ ) : (+ "filters"+ )}+ </DropdownMenuLabel>+ <DropdownMenuSeparator />+ <DropdownMenuItem className={cn(filters == null && "bg-primary")} onClick={() => toggleFilter(null)}>+ All+ </DropdownMenuItem>+ {getAllNodeTypes(actionItems || []).map((type) => (+ <DropdownMenuItem+ className={cn(filters?.includes(type.toLowerCase()) && "bg-primary/30")}+ key={type}+ onClick={() => toggleFilter(type)}+ >+ {type || "unknown"}+ </DropdownMenuItem>+ ))}+ </DropdownMenuContent>+ </DropdownMenu>+ );+};++export default Filters;\ No newline at end of filediff --git a/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-main.tsxindex a87d3433..dab6878a 100644--- a/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx+++ b/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx@@ -4,9 +4,9 @@ import GraphViewer from './graph-viewer'import ContextMenu from './context-menu'const GraphMain = () => {- const nodes = useGraphStore(s => s.nodes)- const edges = useGraphStore(s => s.edges)- const currentNode = useGraphStore(s => s.currentNode)+ const filteredNodes = useGraphStore(s => s.filteredNodes)+ const filteredEdges = useGraphStore(s => s.filteredEdges)+ // const currentNode = useGraphStore(s => s.currentNode)const toggleNodeSelection = useGraphStore(s => s.toggleNodeSelection)const clearSelectedNodes = useGraphStore(s => s.clearSelectedNodes)@@ -14,12 +14,13 @@ const GraphMain = () => {const containerRef = useRef<HTMLDivElement>(null)const [menu, setMenu] = React.useState<any>(null)- // Handle current node centering- useEffect(() => {- if (!currentNode || !graphRef.current) return- graphRef.current.centerAt(currentNode.x, currentNode.y, 500)- graphRef.current.zoom(5, 500)- }, [currentNode])++ // // Handle current node centering+ // useEffect(() => {+ // if (!currentNode || !graphRef.current) return+ // graphRef.current.centerAt(currentNode.position.x, currentNode.position.x, 500)+ // graphRef.current.zoom(5, 500)+ // }, [currentNode])const handleNodeClick = useCallback((node: any) => {toggleNodeSelection(node, false)@@ -59,8 +60,8 @@ const GraphMain = () => {return (<div ref={containerRef} className="relative h-full w-full bg-background"><GraphViewer- nodes={nodes}- edges={edges}+ nodes={filteredNodes}+ edges={filteredEdges}onNodeClick={handleNodeClick}onNodeRightClick={onNodeContextMenu}onBackgroundClick={handleBackgroundClick}diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsxindex 7838f010..945be343 100644--- a/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx+++ b/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx@@ -1,5 +1,5 @@import { useLoaderData } from '@tanstack/react-router'-import { useEffect, useRef, memo, useState, lazy, Suspense } from 'react'+import { useEffect, useRef, memo, useState, lazy, Suspense, useMemo } from 'react'import { useGraphStore, type GraphNode, type GraphEdge } from '@/stores/graph-store'import { Toolbar } from './toolbar'import { cn } from '@/lib/utils'@@ -21,7 +21,7 @@ const RelationshipsTable = lazy(() => import('@/components/table/relationships-vconst Graph = lazy(() => import('./graph'))// const Wall = lazy(() => import('./wall/wall'))-const NODE_COUNT_THRESHOLD = 4500;+const NODE_COUNT_THRESHOLD = 1500;// Separate component for the drag overlayconst DragOverlay = memo(({ isDragging }: { isDragging: boolean }) => (@@ -56,6 +56,48 @@ const GraphPanel = ({ graphData, isLoading }: GraphPanelProps) => {})const [isDraggingOver, setIsDraggingOver] = useState(false)+ // Determine which graph component to use based on node count with debouncing+ const [stableGraphType, setStableGraphType] = useState<'cosmograph' | 'force-graph' | null>(null)++ const shouldUseCosmograph = useMemo(() => {+ if (!nodes || nodes.length === 0) return false+ return nodes.length > NODE_COUNT_THRESHOLD+ }, [nodes?.length])++ // Debounce graph type changes to prevent rapid switching+ useEffect(() => {+ if (!nodes || nodes.length === 0) {+ setStableGraphType(null)+ return+ }++ const newType = shouldUseCosmograph ? 'cosmograph' : 'force-graph'++ // Only update if the type actually changed+ if (stableGraphType !== newType) {+ const timer = setTimeout(() => {+ setStableGraphType(newType)+ }, 500) // 500ms delay to prevent rapid switching++ return () => clearTimeout(timer)+ }++ // Return empty cleanup function for the case when type hasn't changed+ return () => { }+ }, [shouldUseCosmograph, stableGraphType, nodes?.length])++ const graphComponentKey = stableGraphType || 'loading'++ // Memoize the graph component to prevent unnecessary re-renders+ const graphComponent = useMemo(() => {+ if (stableGraphType === 'cosmograph') {+ return <Graph />+ } else if (stableGraphType === 'force-graph') {+ return <GraphMain />+ }+ return null+ }, [stableGraphType])+useEffect(() => {if (graphData?.nds && graphData?.rls) {updateGraphData(graphData.nds, graphData.rls)@@ -128,21 +170,28 @@ const GraphPanel = ({ graphData, isLoading }: GraphPanelProps) => {</div></div>}>- {nodes?.length > NODE_COUNT_THRESHOLD ? (- <>{view === "table" && <NodesTable />}- {["force", "hierarchy"].includes(view) && <Graph />}- {view === "map" && <MapPanel />}- {view === "relationships" && <RelationshipsTable />}- </>- ) : (- <>- {/* {view === "hierarchy" && <Wall isLoading={false} isRefetching={false} />} */}- {/* {view === "force" && <GraphMain />} */}- {["force", "hierarchy"].includes(view) && <GraphMain />}+ {/* Only render graph components when we have actual data */}+ {nodes && nodes.length > 0 && stableGraphType ? (+ <div key={graphComponentKey} className="h-full w-full flex relative outline-2 outline-transparent bg-background"+ >{view === "table" && <NodesTable />}{view === "map" && <MapPanel />}{view === "relationships" && <RelationshipsTable />}- </>+ {["force", "hierarchy"].includes(view) && graphComponent}+ </div>+ ) : (+ // Show loading state while waiting for data or graph type to stabilize+ <div className="h-full w-full flex items-center justify-center">+ <div className="text-center flex flex-col items-center gap-2">+ <Loader />+ <span className="text-muted-foreground">+ {!nodes || nodes.length === 0+ ? 'Loading data...'+ : 'Preparing data...'+ }+ </span>+ </div>+ </div>)}</Suspense><DragOverlay isDragging={isDraggingOver} />diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsxindex 86c02e42..7714afca 100644--- a/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx+++ b/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx@@ -9,6 +9,12 @@ import { Button } from '../ui/button';import { useTheme } from '@/components/theme-provider'import { GRAPH_COLORS } from '../flows/scanner-data';+function truncateText(text: string, limit: number = 16) {+ if (text.length <= limit)+ return text+ return text.substring(0, limit) + "..."+}+interface GraphViewerProps {nodes: GraphNode[];edges: GraphEdge[];@@ -388,7 +394,7 @@ const GraphViewer: React.FC<GraphViewerProps> = ({// Optimized label renderingif (showLabels && globalScale > 3) {- const label = node.nodeLabel || node.label || node.id;+ const label = truncateText(node.nodeLabel || node.label || node.id, 58);if (label) {// Only show labels for highlighted nodes when there's any highlighting// or show all labels when there's no highlightingdiff --git a/flowsint-app/src/renderer/src/components/graphs/graph.tsx b/flowsint-app/src/renderer/src/components/graphs/graph.tsxindex 6d5c014d..11da00da 100644--- a/flowsint-app/src/renderer/src/components/graphs/graph.tsx+++ b/flowsint-app/src/renderer/src/components/graphs/graph.tsx@@ -236,8 +236,8 @@ GraphProvider.displayName = "GraphProvider"// Composant principal avec optimisations maximalesconst Graph = memo(() => {- const nodes = useGraphStore(s => s.nodes)- const edges = useGraphStore(s => s.edges)+ const nodes = useGraphStore(s => s.filteredNodes)+ const edges = useGraphStore(s => s.filteredEdges)if (!nodes.length) {return <EmptyState />diff --git a/flowsint-app/src/renderer/src/components/graphs/nodes-panel.tsx b/flowsint-app/src/renderer/src/components/graphs/nodes-panel.tsxindex 57857767..b3964ffc 100644--- a/flowsint-app/src/renderer/src/components/graphs/nodes-panel.tsx+++ b/flowsint-app/src/renderer/src/components/graphs/nodes-panel.tsx@@ -5,10 +5,10 @@ import { useGraphStore } from "@/stores/graph-store"import { Button } from "@/components/ui/button"import { Badge } from "@/components/ui/badge"import { TypeBadge } from "@/components/type-badge"-import { Search, FilterIcon, XIcon } from "lucide-react"+import { Search, FunnelPlus, XIcon } from "lucide-react"import { Input } from "@/components/ui/input"import { useActionItems } from "@/hooks/use-action-items"-import { cn } from "@/lib/utils"+import { cn, getAllNodeTypes } from "@/lib/utils"import {DropdownMenu,DropdownMenuContent,@@ -21,22 +21,6 @@ import type { GraphNode } from "@/stores/graph-store"import { Checkbox } from "@/components/ui/checkbox"const ITEM_HEIGHT = 40--const getAllNodeTypes = (actionItems: any[]) => {- const types: string[] = []- actionItems.forEach(item => {- if (item.children) {- item.children.forEach(child => {- if (child.type && !types.includes(child.type)) {- types.push(child.type)- }- })- } else if (item.type && !types.includes(item.type)) {- types.push(item.type)- }- })- return types.sort()-}// Mémoiser le composant NodeRendererconst NodeRenderer = memo(({@@ -106,7 +90,8 @@ const NodesPanel = memo(({ nodes, isLoading }: { nodes: GraphNode[]; isLoading?:const setSelectedNodes = useGraphStore((state) => state.setSelectedNodes)const selectedNodes = useGraphStore((state) => state.selectedNodes || [])const [searchQuery, setSearchQuery] = useState<string>("")- const [filters, setFilters] = useState<null | string[]>(null)+ const filters = useGraphStore(s => s.filters);+ const setFilters = useGraphStore(s => s.setFilters);const { actionItems } = useActionItems()// Ref pour le conteneur parent du virtualizer@@ -139,13 +124,12 @@ const NodesPanel = memo(({ nodes, isLoading }: { nodes: GraphNode[]; isLoading?:if (filter === null) {setFilters(null)} else {- setFilters((prev) => {- if (prev === null) return [filter]- const next = prev.includes(filter) ? prev.filter((f) => f !== filter) : [...prev, filter]- return next.length === 0 ? null : next- })+ const currentFilters = filters || []+ const isChecked = currentFilters.includes(filter)+ const newFilters = isChecked ? currentFilters.filter((f) => f !== filter) : [...currentFilters, filter.toLowerCase()]+ setFilters(newFilters.length === 0 ? null : newFilters)}- }, [])+ }, [filters, setFilters])const handleSearchChange = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {setSearchQuery(e.target.value)@@ -216,7 +200,7 @@ const NodesPanel = memo(({ nodes, isLoading }: { nodes: GraphNode[]; isLoading?:<DropdownMenuTrigger asChild><div><Button variant={"outline"} className="h-7 w-8 relative border-border border" size={"icon"}>- <FilterIcon className={cn("opacity-60 h-3 w-3", filters && "opacity-100")} />+ <FunnelPlus className={cn("opacity-60 h-3 w-3", filters && "opacity-100")} />{filters && (<Badge className="absolute -top-2 -right-1 text-xs rounded-full h-4 w-4 text-white" variant={"default"}>{filters.length}diff --git a/flowsint-app/src/renderer/src/components/graphs/toolbar.tsx b/flowsint-app/src/renderer/src/components/graphs/toolbar.tsxindex f4dd40c6..ceab9af4 100644--- a/flowsint-app/src/renderer/src/components/graphs/toolbar.tsx+++ b/flowsint-app/src/renderer/src/components/graphs/toolbar.tsx@@ -12,17 +12,20 @@ import {List,SlidersHorizontal,GitFork,- ArrowRightLeft+ ArrowRightLeft,+ FunnelPlus} from "lucide-react"import { memo, useCallback } from "react"import { toast } from "sonner"import { cn } from "@/lib/utils"import ForceControls from './force-controls'+import Filters from "./filters"+import { useGraphStore } from "@/stores/graph-store"const isMac = navigator.platform.toUpperCase().indexOf('MAC') >= 0// Tooltip wrapper component to avoid repetition-const ToolbarButton = memo(function ToolbarButton({+export const ToolbarButton = memo(function ToolbarButton({icon,tooltip,onClick,@@ -63,6 +66,7 @@ export const Toolbar = memo(function Toolbar({ isLoading }: { isLoading: booleanconst zoomOut = useGraphControls((s) => s.zoomOut);const onLayout = useGraphControls((s) => s.onLayout);const refetchGraph = useGraphControls((s) => s.refetchGraph)+ const filters = useGraphStore(s => s.filters)const handleRefresh = useCallback(() => {try {@@ -97,7 +101,7 @@ export const Toolbar = memo(function Toolbar({ isLoading }: { isLoading: booleanreturn (<div className="flex justify-start gap-2 items-center"><TooltipProvider>- < ToolbarButton+ <ToolbarButtonicon={<ZoomIn className="h-4 w-4 opacity-70" />}tooltip="Zoom In"onClick={zoomIn}@@ -119,26 +123,32 @@ export const Toolbar = memo(function Toolbar({ isLoading }: { isLoading: boolean<ToolbarButtonicon={<GitFork className="h-4 w-4 opacity-70 rotate-180" />}tooltip={`Hierarchy (${isMac ? '⌘' : 'ctrl'}+Y)`}+ disabled={["hierarchy"].includes(view)}onClick={handleDagreLayoutTB}/>+<ToolbarButtonicon={<Waypoints className="h-4 w-4 opacity-70" />}+ disabled={["force"].includes(view)}tooltip={"Graph view"}onClick={handleForceLayout}/><ToolbarButtonicon={<List className="h-4 w-4 opacity-70" />}tooltip={"Table view"}+ disabled={["table"].includes(view)}onClick={handleTableLayout}/><ToolbarButtonicon={<ArrowRightLeft className="h-4 w-4 opacity-70" />}tooltip={"Relationships view"}+ disabled={["relationships"].includes(view)}onClick={handleRelationshipsLayout}/><ToolbarButtonicon={<MapPin className="h-4 w-4 opacity-70" />}tooltip={"Map view"}+ disabled={["map"].includes(view)}onClick={handleMapLayout}/><ToolbarButton@@ -149,11 +159,19 @@ export const Toolbar = memo(function Toolbar({ isLoading }: { isLoading: boolean/><ForceControls><ToolbarButton- disabled={isLoading}+ disabled={isLoading || !["force", "hierarchy"].includes(view)}icon={<SlidersHorizontal className={cn("h-4 w-4 opacity-70")} />}tooltip="Settings"/></ForceControls>+ <Filters>+ <ToolbarButton+ disabled={isLoading}+ icon={<FunnelPlus className={cn("h-4 w-4 opacity-70")} />}+ tooltip="Filters"+ badge={filters && filters?.length > 0 ? filters?.length : null}+ />+ </Filters></TooltipProvider></div>)diff --git a/flowsint-app/src/renderer/src/components/graphs/wall/custom/custom-edge.tsx b/flowsint-app/src/renderer/src/components/graphs/wall/custom/custom-edge.tsxdeleted file mode 100644index f2c24738..00000000--- a/flowsint-app/src/renderer/src/components/graphs/wall/custom/custom-edge.tsx+++ /dev/null@@ -1,18 +0,0 @@-import { CustomEdge } from "@/components/xyflow/custom-edge";-import { memo } from "react";-import { EdgeProps } from "@xyflow/react";--const CustomEdgeLabel = memo((props: EdgeProps) => {- return (- <CustomEdge {...props}>- <span className="block text-[9px] opacity-70 uppercase">- {props.label}- </span>- </CustomEdge>- );-});--// Add display name for debugging-CustomEdgeLabel.displayName = 'CustomEdgeLabel';--export default CustomEdgeLabel;\ No newline at end of filediff --git a/flowsint-app/src/renderer/src/components/graphs/wall/custom/custom-node.tsx b/flowsint-app/src/renderer/src/components/graphs/wall/custom/custom-node.tsxdeleted file mode 100644index c86d9761..00000000--- a/flowsint-app/src/renderer/src/components/graphs/wall/custom/custom-node.tsx+++ /dev/null@@ -1,56 +0,0 @@-import { memo } from "react";-import { NodeProps, Handle, Position } from "@xyflow/react";-import { useNodesDisplaySettings } from "@/stores/node-display-settings";-import { useGraphStore } from "@/stores/graph-store";-import { useIcon } from "@/hooks/use-icon";--const size = 32;---export const CustomNode = memo(({ data, selected, id }: NodeProps) => {- const isCurrent = useGraphStore(state => state.isCurrent);- const color = useNodesDisplaySettings(state => state.colors[data.type as keyof typeof state.colors]);- const IconComponent = useIcon(data.type as string, data?.src as string | null);--- return (- <div className="xy-node -z-[5000] relative p-1 flex flex-col justify-center items-center !rounded-full h-20">- <Handle type="target" position={Position.Top} id="a" className="opacity-0 group-hover:opacity-100" />- <Handle type="target" position={Position.Right} id="b" className="opacity-0 group-hover:opacity-100" />- <Handle type="source" position={Position.Bottom} id="c" className="opacity-0 group-hover:opacity-100" />- <Handle type="source" position={Position.Left} id="d" className="opacity-0 group-hover:opacity-100" />- <div style={{ width: size, height: size, borderColor: (selected || isCurrent(id)) ? "#f97316" : color }} className="!rounded-full gap-0 flex items-center justify-center border-primary/70 p-1 overflow-hidden">- <IconComponent- size={size}- style={{- pointerEvents: "none",- opacity: selected ? 1 : 0.85,- display: "block",- margin: "0 auto",- }}- />- </div>- <div- className="text-nowrap text-center max-w-[160px] text-ellipsis truncate"- style={{- color: (selected || isCurrent(id)) ? "#f97316" : undefined,- marginTop: 2,- pointerEvents: "none",- }}- >- {data?.label as string ?? "N/A"}- </div>- <div- className="text-nowrap text-center opacity-70"- style={{- fontSize: 10,- color: selected ? "#f97316" : undefined,- marginTop: 1,- pointerEvents: "none",- }}- >- {data?.type as string ?? "N/A"}- </div>- </div>- );-});diff --git a/flowsint-app/src/renderer/src/components/graphs/wall/wall.tsx b/flowsint-app/src/renderer/src/components/graphs/wall/wall.tsxdeleted file mode 100644index b9b94b2d..00000000--- a/flowsint-app/src/renderer/src/components/graphs/wall/wall.tsx+++ /dev/null@@ -1,331 +0,0 @@--import type React from "react"-import { useState, useCallback, useRef, useEffect, memo, useMemo } from "react"-import {- ReactFlow,- ReactFlowProvider,- // Background,- type Node,- useReactFlow,- type NodeMouseHandler,- type ColorMode,- MiniMap,- // BackgroundVariant,- EdgeTypes,- MarkerType,- ConnectionMode,-} from "@xyflow/react"-import "@xyflow/react/dist/style.css"-import { getDagreLayoutedElements, getForceLayoutedElements } from "@/lib/utils"-import { useTheme } from "@/components/theme-provider"-import { useGraphStore, type GraphNode, type GraphEdge } from "@/stores/graph-store"-import { CustomNode } from "./custom/custom-node"-// import FloatingEdge from "./custom/floating-edge"-// import FloatingConnectionLine from "./custom/connection-line"-import GraphLoader from "../graph-loader"-import { useGraphControls } from "@/stores/graph-controls-store"-import { useNodesDisplaySettings } from "@/stores/node-display-settings"-import ContextMenu from '../context-menu';-import { useLayoutStore } from "@/stores/layout-store"-import CustomEdge from "./custom/custom-edge"--const nodeTypes = {- custom: CustomNode,-};--const edgeTypes = {- custom: CustomEdge,-};--const Wall = memo(({ theme, edges }: { theme: ColorMode, edges: GraphEdge[] }) => {- const { fitView, zoomIn, zoomOut, setCenter } = useReactFlow()- const reactFlowWrapper = useRef<HTMLDivElement>(null)- const colors = useNodesDisplaySettings(state => state.colors)- const [reactFlowInstance, setReactFlowInstance] = useState<any>(null)- const setActions = useGraphControls(state => state.setActions)- // Memoize store selectors- const nodes = useGraphStore(state => state.nodes)- const toggleNodeSelection = useGraphStore(state => state.toggleNodeSelection)- const currentNode = useGraphStore(state => state.currentNode)- const setCurrentNode = useGraphStore(state => state.setCurrentNode)- const clearSelectedNodes = useGraphStore(state => state.clearSelectedNodes)- const setNodes = useGraphStore(state => state.setNodes)- const setEdges = useGraphStore(state => state.setEdges)- const addNode = useGraphStore(state => state.addNode)- const onNodesChange = useGraphStore(state => state.onNodesChange)- const onEdgesChange = useGraphStore(state => state.onEdgesChange)- const onConnect = useGraphStore(state => state.onConnect)- const setOpenNodeEditorModal = useGraphStore(state => state.setOpenNodeEditorModal)- const setActiveTab = useLayoutStore((s) => s.setActiveTab)-- const [menu, setMenu] = useState<{- node: GraphNode;- top?: number;- left?: number;- right?: number;- bottom?: number;- rawTop?: number;- rawLeft?: number;- wrapperWidth: number;- wrapperHeight: number;- setMenu: (menu: any | null) => void;- } | null>(null);-- // Memoize filtered nodes and edges- const visibleNodes = useMemo(() => nodes.filter(n => !n.hidden), [nodes])- const visibleEdges = useMemo(() => edges.filter(e => !e.hidden), [edges])-- // Center view on selected node- useEffect(() => {- if (currentNode && reactFlowInstance) {- const nodeWidth = currentNode.measured?.width ?? 0- const nodeHeight = currentNode.measured?.height ?? 0- setCenter(- currentNode.position.x + nodeWidth / 2,- currentNode.position.y + nodeHeight / 2 + 20,- { duration: 500, zoom: 1.5 }- )- }- }, [currentNode, reactFlowInstance, setCenter])-- useEffect(() => {- if (reactFlowInstance) {- const actions = {- zoomIn: zoomIn,- zoomToFit: fitView,- zoomOut: zoomOut,- onLayout: onLayout,- }- setActions(actions)- }- }, [reactFlowInstance, setActions])-- const onDragOver = useCallback((event: React.DragEvent<HTMLDivElement>) => {- event.preventDefault()- event.dataTransfer.dropEffect = "move"- }, [])-- const onDrop = useCallback(- (event: React.DragEvent<HTMLDivElement>) => {- event.preventDefault()- if (!reactFlowWrapper.current || !reactFlowInstance) return- const reactFlowBounds = reactFlowWrapper.current.getBoundingClientRect()- const newNodeData = JSON.parse(event.dataTransfer.getData("application/json")) as any & {- type: string- }- const position = reactFlowInstance.screenToFlowPosition({- x: event.clientX - reactFlowBounds.left,- y: event.clientY - reactFlowBounds.top,- })- const id = `${newNodeData.type}-${Date.now()}`- const newNode: GraphNode = {- id: id,- position,- data: {- id: id,- type: newNodeData.type,- caption: newNodeData.caption,- label: newNodeData.label,- created_at: new Date().toISOString(),- },- }- addNode(newNode)- const nodeWidth = newNode.measured?.width ?? 0- const nodeHeight = newNode.measured?.height ?? 0- setCenter(newNode.position.x + nodeWidth / 2, newNode.position.y + nodeHeight / 2 + 20, {- duration: 500,- zoom: 1.5,- })- },- [reactFlowInstance, addNode, setCenter],- )-- const onNodeClick: NodeMouseHandler = useCallback(- (e: React.MouseEvent, node: Node) => {- if (e.shiftKey) {- toggleNodeSelection(node as GraphNode, true)- return- }- const typedNode = node as GraphNode- setCurrentNode(typedNode)- setActiveTab("entities")- const nodeWidth = typedNode.measured?.width ?? 0- const nodeHeight = typedNode.measured?.height ?? 0- setCenter(typedNode.position.x + nodeWidth / 2, typedNode.position.y + nodeHeight / 2 + 20, {- duration: 500,- zoom: 1.5,- })- },- [setCenter, setCurrentNode],- )-- const onNodeDoubleClick: NodeMouseHandler = useCallback(- (_: React.MouseEvent, node: Node) => {- const typedNode = node as GraphNode- setCurrentNode(typedNode)- setOpenNodeEditorModal(true)- },- [setCurrentNode],- )-- const onPaneClick = useCallback(() => {- setCurrentNode(null)- clearSelectedNodes()- setMenu(null)- }, [setCurrentNode, clearSelectedNodes, setMenu])-- const onLayout = useCallback((type: "dagre-tb" | "dagre-lr" | "force") => {- // Wait for nodes to be measured before running layout- setTimeout(() => {- let layouted: any;- if (type === "dagre-tb") {- layouted = getDagreLayoutedElements(- nodes,- edges,- { direction: "TB" }- );- } else if (type === "dagre-lr") {- layouted = getDagreLayoutedElements(- nodes,- edges,- { direction: "LR" }- );- } else {- layouted = getForceLayoutedElements(- nodes,- edges,- {- direction: "TB",- strength: -300,- distance: 100,- iterations: 300- }- );- }- setNodes(layouted.nodes as GraphNode[])- setEdges(layouted.edges as GraphEdge[])- window.requestAnimationFrame(() => {- fitView()- })- }, 100)- }, [nodes, edges, setNodes, setEdges, fitView])-- // Run dagre layout on first render- useEffect(() => {- if (reactFlowInstance && nodes.length) {- onLayout("dagre-tb")- }- }, [reactFlowInstance, nodes.length])-- const onNodeContextMenu = useCallback(- (event: React.MouseEvent, node: Node) => {- // Prevent native context menu from showing- event.preventDefault();-- // Calculate position of the context menu. We want to make sure it- // doesn't get positioned off-screen.- if (!reactFlowWrapper.current) return;-- const pane = reactFlowWrapper.current.getBoundingClientRect();- const relativeX = event.clientX - pane.left;- const relativeY = event.clientY - pane.top;-- setMenu({- node: node as GraphNode,- rawTop: relativeY,- rawLeft: relativeX,- wrapperWidth: pane.width,- wrapperHeight: pane.height,- setMenu: setMenu,- });- },- [setMenu],- );-- // Memoize ReactFlow props- const reactFlowProps = useMemo(() => ({- nodes: visibleNodes,- edges: visibleEdges,- nodeTypes,- edgeTypes: edgeTypes as EdgeTypes,- onInit: setReactFlowInstance,- onNodeContextMenu,- onDrop,- onDragOver,- onNodeClick,- onNodeDoubleClick,- onPaneClick,- onNodesChange,- onEdgesChange,- onConnect,- fitView: true,- proOptions: { hideAttribution: true },- colorMode: theme,- minZoom: 0.1,- maxZoom: 2.5,- onlyRenderVisibleElements: nodes.length > 500,- connectionMode: ConnectionMode.Loose- }), [- visibleNodes,- visibleEdges,- onDrop,- onNodeContextMenu,- onDragOver,- onNodeClick,- onNodeDoubleClick,- onPaneClick,- onNodesChange,- onEdgesChange,- onConnect,- theme,- nodes.length- ])-- return (- <>- <div ref={reactFlowWrapper} className="w-full h-full bg-background">- <ReactFlow {...reactFlowProps} className="!bg-background">- {/* <Background- variant={"dots" as BackgroundVariant}- bgColor="var(--background)" /> */}- {menu && <ContextMenu- {...menu}- />}- <MiniMap nodeBorderRadius={80} nodeClassName={"h-16 w-16 text-center"} nodeColor={(node) => colors[node.data.type as keyof typeof colors]} className="bg-background" position="bottom-right" pannable zoomable />- </ReactFlow>- </div>- </>- )-})--Wall.displayName = "Wall"--const WallEditor = memo(({ isLoading, isRefetching }: { isLoading: boolean, isRefetching: boolean }) => {- const { theme } = useTheme()- const edges = useGraphStore(state => state.edges)-- const edgesWithMarkers = useMemo(() => edges.map(edge => ({- ...edge,- markerEnd: {- type: MarkerType.ArrowClosed,- },- // markerStart: {- // type: MarkerType.ArrowClosed,- // orient: 'auto-start-reverse',- // },- })), [edges])-- if (isLoading || isRefetching) {- return <GraphLoader />- }-- return (- <ReactFlowProvider>- <Wall edges={edgesWithMarkers} theme={theme as ColorMode} />- </ReactFlowProvider>- )-})--WallEditor.displayName = "WallEditor"--export default WallEditordiff --git a/flowsint-app/src/renderer/src/components/table/index.tsx b/flowsint-app/src/renderer/src/components/table/index.tsxindex c7e192ae..cbd6586b 100644--- a/flowsint-app/src/renderer/src/components/table/index.tsx+++ b/flowsint-app/src/renderer/src/components/table/index.tsx@@ -2,7 +2,7 @@ import { useGraphStore } from "@/stores/graph-store"import NodesTable from "./nodes-view"export default function NodesViewTable() {- const nodes = useGraphStore(s => s.nodes)+ const nodes = useGraphStore(s => s.filteredNodes)return (<NodesTable nodes={nodes} />diff --git a/flowsint-app/src/renderer/src/hooks/use-events.ts b/flowsint-app/src/renderer/src/hooks/use-events.tsindex 8fa160fc..d1d9d8e0 100644--- a/flowsint-app/src/renderer/src/hooks/use-events.ts+++ b/flowsint-app/src/renderer/src/hooks/use-events.ts@@ -31,7 +31,7 @@ export function useEvents(sketch_id: string | undefined) {try {const raw = JSON.parse(e.data) as anyconst event = JSON.parse(raw.data) as Event- if (event.type === EventLevel.GRAPH_APPEND) {+ if (event.type === EventLevel.COMPLETED) {refetchGraph()// const nodes = event.payload.nodes as GraphNode[]// const edges = event.payload.edges as GraphEdge[]diff --git a/flowsint-app/src/renderer/src/lib/utils.ts b/flowsint-app/src/renderer/src/lib/utils.tsindex 6cd40364..5202eb54 100644--- a/flowsint-app/src/renderer/src/lib/utils.ts+++ b/flowsint-app/src/renderer/src/lib/utils.ts@@ -335,4 +335,18 @@ export function capitalizeFirstLetter(string: string) {}-+export const getAllNodeTypes = (actionItems: any[]) => {+ const types: string[] = []+ actionItems.forEach(item => {+ if (item.children) {+ item.children.forEach(child => {+ if (child.type && !types.includes(child.type)) {+ types.push(child.type)+ }+ })+ } else if (item.type && !types.includes(item.type)) {+ types.push(item.type)+ }+ })+ return types.sort()+}\ No newline at end of filediff --git a/flowsint-app/src/renderer/src/routes/_auth.dashboard.investigations.$investigationId.$type.$id.tsx b/flowsint-app/src/renderer/src/routes/_auth.dashboard.investigations.$investigationId.$type.$id.tsxindex b6d37bc7..49adf2a8 100644--- a/flowsint-app/src/renderer/src/routes/_auth.dashboard.investigations.$investigationId.$type.$id.tsx+++ b/flowsint-app/src/renderer/src/routes/_auth.dashboard.investigations.$investigationId.$type.$id.tsx@@ -92,16 +92,16 @@ export const Route = createFileRoute('/_auth/dashboard/investigations/$investiga</svg></div></div>-+<h2 className="text-xl font-semibold text-foreground mb-2">Oops! Something went wrong</h2>-+<p className="text-muted-foreground mb-6 text-sm leading-relaxed">- We couldn't load this investigation page. This might be due to a network issue,+ We couldn't load this investigation page. This might be due to a network issue,invalid investigation ID, or the resource may not exist.</p>-+<div className="space-y-3"><buttononClick={() => window.location.reload()}@@ -124,7 +124,7 @@ export const Route = createFileRoute('/_auth/dashboard/investigations/$investiga</svg>Reload Page</button>-+<buttononClick={() => window.history.back()}className="w-full bg-muted hover:bg-muted/80 text-muted-foreground hover:text-foreground@@ -133,7 +133,7 @@ export const Route = createFileRoute('/_auth/dashboard/investigations/$investigaGo Back</button></div>-+<div className="mt-6 pt-4 border-t"><p className="text-xs text-muted-foreground">If the problem persists, try:diff --git a/flowsint-app/src/renderer/src/stores/graph-store.ts b/flowsint-app/src/renderer/src/stores/graph-store.tsindex 36edf3ae..eebc2fad 100644--- a/flowsint-app/src/renderer/src/stores/graph-store.ts+++ b/flowsint-app/src/renderer/src/stores/graph-store.ts@@ -1,4 +1,5 @@import { create } from "zustand"+import { persist } from "zustand/middleware"import type { EdgeData, NodeData } from "@/types"import {type Node,@@ -24,6 +25,8 @@ interface GraphState {// === Graph ===nodes: GraphNode[]edges: GraphEdge[]+ filteredNodes: GraphNode[]+ filteredEdges: GraphEdge[]setNodes: (nodes: GraphNode[]) => voidsetEdges: (edges: GraphEdge[]) => voidaddNode: (newNode: Partial<GraphNode>) => GraphNode@@ -71,8 +74,8 @@ interface GraphState {handleEdit: (node: GraphNode) => void// === Filters ===- filters: Record<string, unknown>- setFilters: (filters: Record<string, unknown>) => void+ filters: string[] | null+ setFilters: (filters: string[] | null) => void// === Collapse/Expand logic ===toggleCollapse: (nodeId: string) => void@@ -84,210 +87,289 @@ interface GraphState {getEdgesLength: () => number}-export const useGraphStore = create<GraphState>()((set, get) => ({- // === Graph ===- nodes: [],- edges: [],- setNodes: (nodes) => set({ nodes }),- setEdges: (edges) => set({ edges }),- addNode: (newNode) => {- const { nodes } = get()- const nodeWithId: GraphNode = {- id: newNode.id || `node-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`,- position: {- x: 0,- y: 0,- },- ...newNode,- } as GraphNode- set({ nodes: [...nodes, nodeWithId], currentNode: nodeWithId })- return nodeWithId- },- addEdge: (newEdge) => {- const { edges } = get()- const edgeWithId: GraphEdge = {- id: newEdge.id || `node-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`,- ...newEdge,- } as GraphEdge- set({ edges: [...edges, edgeWithId] })- return edgeWithId- },- removeNodes: (nodeIds: string[]) => {- const { nodes, edges } = get();- const newNodes = nodes.filter((n) => !nodeIds.includes(n.id));- const newEdges = edges.filter(- (e) => !nodeIds.includes(e.source) && !nodeIds.includes(e.target)- );- set({ nodes: newNodes, edges: newEdges });- },- removeEdges: (edgeIds: string[]) => {- const { edges } = get()- const newEdges = edges.filter((e) => !edgeIds.includes(e.id))- set({ edges: newEdges })- },- updateGraphData: (nodes: GraphNode[], edges: GraphEdge[]) => {- set({ nodes, edges })- },- updateNode: (nodeId, updates) => {- const { nodes } = get()- const updatedNodes = nodes.map(node =>- node.id === nodeId- ? { ...node, data: { ...node.data, ...updates } }- : node- )- set({ nodes: updatedNodes })- },- updateEdge: (edgeId, updates) => {- const { edges } = get()- const updatedEdges = edges.map(edge =>- edge.id === edgeId- ? { ...edge, data: { ...edge.data, ...updates } as EdgeData }- : edge- )- set({ edges: updatedEdges })- },- onNodesChange: (changes) => {- set({- nodes: applyNodeChanges(changes, get().nodes) as GraphNode[],- })- },- onEdgesChange: (changes) => {- set({- edges: applyEdgeChanges(changes, get().edges) as GraphEdge[],- })- },- onConnect: (connection: Connection) => {- const edge: GraphEdge = {- id: `${connection.source}-${connection.target}`,- source: connection.source!,- target: connection.target!,- sourceHandle: connection.sourceHandle,- targetHandle: connection.targetHandle,+// --- Helpers ---+const computeFilteredNodes = (nodes: GraphNode[], filters: string[] | null): GraphNode[] => {+ if (!filters || Object.keys(filters).length === 0) return nodes+ return nodes.filter((node) => {+ if (filters && Array.isArray(filters)) {+ return filters.includes(node.data.type)}- set({- edges: [...get().edges, edge],- })- },- reset: () => {- set({- // nodes: [],- // edges: [],+ return true+ })+}++const computeFilteredEdges = (edges: GraphEdge[], filteredNodes: GraphNode[]): GraphEdge[] => {+ const nodeIds = new Set(filteredNodes.map(n => n.id))+ return edges.filter(e => nodeIds.has(e.source) && nodeIds.has(e.target))+}++// --- Store ---+export const useGraphStore = create<GraphState>()(+ persist(+ (set, get) => ({+ // === Graph ===+ nodes: [],+ edges: [],+ filteredNodes: [],+ filteredEdges: [],+ getNodesLength: () => get().nodes.length,+ getEdgesLength: () => get().edges.length,++ setNodes: (nodes) => {+ const { filters, edges } = get()+ const filteredNodes = computeFilteredNodes(nodes, filters)+ const filteredEdges = computeFilteredEdges(edges, filteredNodes)+ set({ nodes, filteredNodes, filteredEdges })+ },+ setEdges: (edges) => {+ const { filters, nodes } = get()+ const filteredNodes = computeFilteredNodes(nodes, filters)+ const filteredEdges = computeFilteredEdges(edges, filteredNodes)+ set({ edges, filteredNodes, filteredEdges })+ },++ addNode: (newNode) => {+ const { nodes, edges, filters } = get()+ const nodeWithId: GraphNode = {+ id: newNode.id || `node-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`,+ position: { x: 0, y: 0 },+ ...newNode,+ } as GraphNode+ const newNodes = [...nodes, nodeWithId]+ const filteredNodes = computeFilteredNodes(newNodes, filters)+ const filteredEdges = computeFilteredEdges(edges, filteredNodes)+ set({ nodes: newNodes, currentNode: nodeWithId, filteredNodes, filteredEdges })+ return nodeWithId+ },++ addEdge: (newEdge) => {+ const { edges, nodes, filters } = get()+ const edgeWithId: GraphEdge = {+ id: newEdge.id || `edge-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`,+ ...newEdge,+ } as GraphEdge+ const newEdges = [...edges, edgeWithId]+ const filteredNodes = computeFilteredNodes(nodes, filters)+ const filteredEdges = computeFilteredEdges(newEdges, filteredNodes)+ set({ edges: newEdges, filteredNodes, filteredEdges })+ return edgeWithId+ },++ removeNodes: (nodeIds: string[]) => {+ const { nodes, edges, filters } = get()+ const newNodes = nodes.filter(n => !nodeIds.includes(n.id))+ const newEdges = edges.filter(e => !nodeIds.includes(e.source) && !nodeIds.includes(e.target))+ const filteredNodes = computeFilteredNodes(newNodes, filters)+ const filteredEdges = computeFilteredEdges(newEdges, filteredNodes)+ set({ nodes: newNodes, edges: newEdges, filteredNodes, filteredEdges })+ },++ removeEdges: (edgeIds: string[]) => {+ const { edges, nodes, filters } = get()+ const newEdges = edges.filter(e => !edgeIds.includes(e.id))+ const filteredNodes = computeFilteredNodes(nodes, filters)+ const filteredEdges = computeFilteredEdges(newEdges, filteredNodes)+ set({ edges: newEdges, filteredNodes, filteredEdges })+ },++ updateGraphData: (nodes, edges) => {+ const { filters } = get()+ const filteredNodes = computeFilteredNodes(nodes, filters)+ const filteredEdges = computeFilteredEdges(edges, filteredNodes)+ set({ nodes, edges, filteredNodes, filteredEdges })+ },++ updateNode: (nodeId, updates) => {+ const { nodes, edges, filters } = get()+ const updatedNodes = nodes.map(node =>+ node.id === nodeId ? { ...node, data: { ...node.data, ...updates } } : node+ )+ const filteredNodes = computeFilteredNodes(updatedNodes, filters)+ const filteredEdges = computeFilteredEdges(edges, filteredNodes)+ set({ nodes: updatedNodes, filteredNodes, filteredEdges })+ },++ updateEdge: (edgeId, updates) => {+ const { edges, nodes, filters } = get()+ const updatedEdges = edges.map(edge =>+ edge.id === edgeId ? { ...edge, data: { ...edge.data, ...updates } as EdgeData } : edge+ )+ const filteredNodes = computeFilteredNodes(nodes, filters)+ const filteredEdges = computeFilteredEdges(updatedEdges, filteredNodes)+ set({ edges: updatedEdges, filteredNodes, filteredEdges })+ },++ onNodesChange: (changes) => {+ const { nodes, edges, filters } = get()+ const updatedNodes = applyNodeChanges(changes, nodes) as GraphNode[]+ const filteredNodes = computeFilteredNodes(updatedNodes, filters)+ const filteredEdges = computeFilteredEdges(edges, filteredNodes)+ set({ nodes: updatedNodes, filteredNodes, filteredEdges })+ },++ onEdgesChange: (changes) => {+ const { edges, nodes, filters } = get()+ const updatedEdges = applyEdgeChanges(changes, edges) as GraphEdge[]+ const filteredNodes = computeFilteredNodes(nodes, filters)+ const filteredEdges = computeFilteredEdges(updatedEdges, filteredNodes)+ set({ edges: updatedEdges, filteredNodes, filteredEdges })+ },++ onConnect: (connection: Connection) => {+ const { edges, nodes, filters } = get()+ const edge: GraphEdge = {+ id: `${connection.source}-${connection.target}`,+ source: connection.source!,+ target: connection.target!,+ sourceHandle: connection.sourceHandle,+ targetHandle: connection.targetHandle,+ }+ const newEdges = [...edges, edge]+ const filteredNodes = computeFilteredNodes(nodes, filters)+ const filteredEdges = computeFilteredEdges(newEdges, filteredNodes)+ set({ edges: newEdges, filteredNodes, filteredEdges })+ },++ reset: () => {+ set({+ currentNode: null,+ selectedNodes: [],+ relatedNodeToAdd: null,+ openMainDialog: false,+ openFormDialog: false,+ openAddRelationDialog: false,+ openNodeEditorModal: false,+ currentNodeType: null,+ // filters: {},+ filteredNodes: get().nodes,+ filteredEdges: get().edges,+ })+ },++ // === Selection & Current ===currentNode: null,selectedNodes: [],+ isCurrent: (nodeId) => get().currentNode?.id === nodeId,+ isSelected: (nodeId) => {+ const { selectedNodes, isCurrent } = get()+ return selectedNodes.some((node) => node.id === nodeId) || isCurrent(nodeId)+ },+ setCurrentNode: (node) => set({ currentNode: node }),+ setSelectedNodes: (nodes) => set({ selectedNodes: nodes }),+ clearSelectedNodes: () => set({ selectedNodes: [], currentNode: null }),+ toggleNodeSelection: (node, multiSelect = false) => {+ const { selectedNodes, currentNode } = get()+ const isSelected = selectedNodes.some((n) => n.id === node.id)+ let newSelected: GraphNode[]++ if (multiSelect) {+ newSelected = isSelected+ ? selectedNodes.filter((n) => n.id !== node.id)+ : [...selectedNodes, node]+ } else {+ newSelected = isSelected && selectedNodes.length === 1 ? [] : [node]+ }++ set({+ selectedNodes: newSelected,+ currentNode: multiSelect ? currentNode : (isSelected ? null : node),+ })+ },++ // === Relation ===relatedNodeToAdd: null,+ setRelatedNodeToAdd: (node) => set({ relatedNodeToAdd: node }),++ // === Dialogs ===openMainDialog: false,openFormDialog: false,openAddRelationDialog: false,openNodeEditorModal: false,- currentNodeType: null,- filters: {},- })- },+ setOpenMainDialog: (open) => set({ openMainDialog: open }),+ setOpenFormDialog: (open) => set({ openFormDialog: open }),+ setOpenAddRelationDialog: (open) => set({ openAddRelationDialog: open }),+ setOpenNodeEditorModal: (open) => set({ openNodeEditorModal: open }),- // === Selection & Current ===- currentNode: null,- selectedNodes: [],- isCurrent: (nodeId) => get().currentNode?.id === nodeId,- isSelected: (nodeId) => {- const { selectedNodes, isCurrent } = get()- return selectedNodes.some((node) => node.id === nodeId) || isCurrent(nodeId)- },- setCurrentNode: (node) => set({ currentNode: node }),- setSelectedNodes: (nodes) => set({ selectedNodes: nodes }),- clearSelectedNodes: () => set({ selectedNodes: [], currentNode: null }),- toggleNodeSelection: (node, multiSelect = false) => {- const { selectedNodes, currentNode } = get()- const isSelected = selectedNodes.some((n) => n.id === node.id)- let newSelected: GraphNode[]-- if (multiSelect) {- newSelected = isSelected- ? selectedNodes.filter((n) => n.id !== node.id)- : [...selectedNodes, node]- } else {- newSelected = isSelected && selectedNodes.length === 1 ? [] : [node]- }+ // === Action Type for Edit form ===+ handleEdit: (node) => set({ currentNode: node, openNodeEditorModal: true }),- set({- selectedNodes: newSelected,- currentNode: multiSelect ? currentNode : (isSelected ? null : node),- })- },-- // === Relation ===- relatedNodeToAdd: null,- setRelatedNodeToAdd: (node) => set({ relatedNodeToAdd: node }),+ // === Action Type for Form ===+ currentNodeType: null,+ setCurrentNodeType: (nodeType) => set({ currentNodeType: nodeType }),+ handleOpenFormModal: (selectedItem) => {+ if (!selectedItem) return+ set({+ currentNodeType: selectedItem,+ openMainDialog: false,+ openFormDialog: true,+ })+ },- // === Dialogs ===- openMainDialog: false,- openFormDialog: false,- openAddRelationDialog: false,- openNodeEditorModal: false,- setOpenMainDialog: (open) => set({ openMainDialog: open }),- setOpenFormDialog: (open) => set({ openFormDialog: open }),- setOpenAddRelationDialog: (open) => set({ openAddRelationDialog: open }),- setOpenNodeEditorModal: (open) => set({ openNodeEditorModal: open }),+ // === Filters ===+ filters: [],+ setFilters: (filters) => {+ const { nodes, edges } = get()+ const filteredNodes = computeFilteredNodes(nodes, filters)+ const filteredEdges = computeFilteredEdges(edges, filteredNodes)+ set({ filters, filteredNodes, filteredEdges })+ },- // === Action Type for Edit form ===- handleEdit: (node) => set({ currentNode: node, openNodeEditorModal: true }),+ // === Collapse/Expand logic ===+ toggleCollapse: (nodeId) => {+ const { nodes, edges, filters } = get()+ const node = nodes.find(n => n.id === nodeId)+ if (!node) return+ const isCollapsing = !node.collapsed- // === Action Type for Form ===- currentNodeType: null,- setCurrentNodeType: (nodeType) => set({ currentNodeType: nodeType }),- handleOpenFormModal: (selectedItem) => {- if (!selectedItem) return- set({- currentNodeType: selectedItem,- openMainDialog: false,- openFormDialog: true,- })- },+ const getDescendants = (parentId: string, accNodes: Set<string>, accEdges: Set<string>) => {+ edges.forEach(edge => {+ if (edge.source === parentId) {+ accEdges.add(edge.id)+ accNodes.add(edge.target)+ getDescendants(edge.target, accNodes, accEdges)+ }+ })+ }- // === Filters ===- filters: {},- setFilters: (filters) => set({ filters }),+ const descendantNodeIds = new Set<string>()+ const descendantEdgeIds = new Set<string>()+ getDescendants(nodeId, descendantNodeIds, descendantEdgeIds)- // === Collapse/Expand logic ===- toggleCollapse: (nodeId) => {- const { nodes, edges } = get();- // Find the node- const node = nodes.find(n => n.id === nodeId);- if (!node) return;- const isCollapsing = !node.collapsed;- // Recursively find all descendants- const getDescendants = (parentId: string, accNodes: Set<string>, accEdges: Set<string>) => {- edges.forEach(edge => {- if (edge.source === parentId) {- accEdges.add(edge.id);- accNodes.add(edge.target);- getDescendants(edge.target, accNodes, accEdges);- }- });- };- const descendantNodeIds = new Set<string>();- const descendantEdgeIds = new Set<string>();- getDescendants(nodeId, descendantNodeIds, descendantEdgeIds);- // Update nodes and edges- set({- nodes: nodes.map(n =>- n.id === nodeId- ? { ...n, collapsed: isCollapsing }- : descendantNodeIds.has(n.id)- ? { ...n, hidden: isCollapsing }- : n- ),- edges: edges.map(e =>- descendantEdgeIds.has(e.id)- ? { ...e, hidden: isCollapsing }- : e- ),- });- },+ const newNodes = nodes.map(n =>+ n.id === nodeId+ ? { ...n, collapsed: isCollapsing }+ : descendantNodeIds.has(n.id)+ ? { ...n, hidden: isCollapsing }+ : n+ )+ const newEdges = edges.map(e =>+ descendantEdgeIds.has(e.id)+ ? { ...e, hidden: isCollapsing }+ : e+ )- // === Utils ===- nodesLength: 0,- edgesLength: 0,+ const filteredNodes = computeFilteredNodes(newNodes, filters)+ const filteredEdges = computeFilteredEdges(newEdges, filteredNodes)+ set({ nodes: newNodes, edges: newEdges, filteredNodes, filteredEdges })+ },- getNodesLength: () => get().nodes.length,- getEdgesLength: () => get().edges.length,-}))+ // === Utils ===+ nodesLength: 0,+ edgesLength: 0,+ }),+ {+ name: "graph-store",+ partialize: (state) => ({ filters: state.filters }),+ onRehydrateStorage: () => (state) => {+ if (state) {+ // recalcul des nodes/edges filtrés après rehydratation+ const { nodes, edges, filters } = state+ const filteredNodes = computeFilteredNodes(nodes, filters)+ const filteredEdges = computeFilteredEdges(edges, filteredNodes)+ state.filteredNodes = filteredNodes+ state.filteredEdges = filteredEdges+ }+ },+ }+ )+)diff --git a/flowsint-core/src/flowsint_core/core/registry.py b/flowsint-core/src/flowsint_core/core/registry.pyindex c27a57ed..156b8e4c 100644--- a/flowsint-core/src/flowsint_core/core/registry.py+++ b/flowsint-core/src/flowsint_core/core/registry.py@@ -1,32 +1,57 @@import inspect-from typing import Dict, Type, List+from typing import Dict, Optional, Type, Listfrom flowsint_core.core.scanner_base import Scanner-from flowsint_transforms.domains.subdomains import SubdomainScanner-from flowsint_transforms.domains.whois import WhoisScanner-from flowsint_transforms.domains.resolve import ResolveScanner-from flowsint_transforms.ips.reverse_resolve import ReverseResolveScanner-from flowsint_transforms.ips.geolocation import GeolocationScanner-from flowsint_transforms.socials.maigret import MaigretScanner-from flowsint_transforms.ips.ip_to_asn import IpToAsnScanner-from flowsint_transforms.ips.asn_to_cidrs import AsnToCidrsScanner-from flowsint_transforms.ips.cidr_to_ips import CidrToIpsScanner-from flowsint_transforms.organizations.org_to_asn import OrgToAsnScanner-from flowsint_transforms.domains.domain_to_asn import DomainToAsnScanner-from flowsint_transforms.crypto.wallet_to_transactions import (++# Domain-related scanners+from flowsint_transforms.domain.to_subdomains import SubdomainScanner+from flowsint_transforms.domain.to_whois import WhoisScanner+from flowsint_transforms.domain.to_ip import ResolveScanner+from flowsint_transforms.domain.to_website import DomainToWebsiteScanner+from flowsint_transforms.domain.to_root_domain import DomainToRootDomain+from flowsint_transforms.domain.to_asn import DomainToAsnScanner++# IP-related scanners+from flowsint_transforms.ip.to_domain import ReverseResolveScanner+from flowsint_transforms.ip.to_geolocation import GeolocationScanner+++from flowsint_transforms.ip.to_asn import IpToAsnScanner++# ASN-related scanners+from flowsint_transforms.asn.to_cidrs import AsnToCidrsScanner++# CIDR-related scanners+from flowsint_transforms.cidr.to_ips import CidrToIpsScanner++# Social media scanners+from flowsint_transforms.social.to_maigret import MaigretScanner++# Organization-related scanners+from flowsint_transforms.organization.to_asn import OrgToAsnScanner+from flowsint_transforms.organization.to_infos import OrgToInfosScanner++# Cryptocurrency scanners+from flowsint_transforms.crypto.to_transactions import (CryptoWalletAddressToTransactions,)-from flowsint_transforms.crypto.wallet_to_nfts import CryptoWalletAddressToNFTs-from flowsint_transforms.domains.to_website import DomainToWebsiteScanner-from flowsint_transforms.websites.to_crawler import WebsiteToCrawler-from flowsint_transforms.websites.to_links import WebsiteToLinks-from flowsint_transforms.websites.to_domain import WebsiteToDomainScanner-from flowsint_transforms.emails.to_gravatar import EmailToGravatarScanner-from flowsint_transforms.emails.to_leaks import EmailToBreachesScanner-from flowsint_transforms.individuals.to_org import IndividualToOrgScanner-from flowsint_transforms.organizations.to_infos import OrgToInfosScanner-from flowsint_transforms.websites.to_webtrackers import WebsiteToWebtrackersScanner+from flowsint_transforms.crypto.to_nfts import CryptoWalletAddressToNFTs++# Website-related scanners+from flowsint_transforms.website.to_crawler import WebsiteToCrawler+from flowsint_transforms.website.to_links import WebsiteToLinks+from flowsint_transforms.website.to_domain import WebsiteToDomainScanner+from flowsint_transforms.website.to_text import WebsiteToText+from flowsint_transforms.website.to_webtrackers import WebsiteToWebtrackersScanner++# Email-related scanners+from flowsint_transforms.email.to_gravatar import EmailToGravatarScanner+from flowsint_transforms.email.to_leaks import EmailToBreachesScanner++# Individual-related scanners+from flowsint_transforms.individual.to_org import IndividualToOrgScanner++# Integration scannersfrom flowsint_transforms.n8n.connector import N8nConnector-from flowsint_transforms.domains.to_root_domain import DomainToRootDomainclass TransformRegistry:@@ -66,10 +91,11 @@ class TransformRegistry:}@classmethod- def list(cls) -> Dict[str, Dict[str, str]]:+ def list(cls, exclude: Optional[List[str]] = []) -> Dict[str, Dict[str, str]]:return {name: cls._create_scanner_metadata(scanner)for name, scanner in cls._scanners.items()+ if name not in exclude}@classmethod@@ -83,7 +109,9 @@ class TransformRegistry:return scanners_by_category@classmethod- def list_by_input_type(cls, input_type: str) -> List[Dict[str, str]]:+ def list_by_input_type(+ cls, input_type: str, exclude: Optional[List[str]] = []+ ) -> List[Dict[str, str]]:input_type_lower = input_type.lower()if input_type_lower == "any":@@ -96,31 +124,55 @@ class TransformRegistry:cls._create_scanner_metadata(scanner)for scanner in cls._scanners.values()if scanner.input_schema()["type"].lower() in ["any", input_type_lower]+ and scanner.name not in exclude]# Register all scanners++# Domain-related scannersTransformRegistry.register(ReverseResolveScanner)TransformRegistry.register(ResolveScanner)TransformRegistry.register(SubdomainScanner)TransformRegistry.register(WhoisScanner)+TransformRegistry.register(DomainToWebsiteScanner)+TransformRegistry.register(DomainToRootDomain)+TransformRegistry.register(DomainToAsnScanner)++# IP-related scannersTransformRegistry.register(GeolocationScanner)-TransformRegistry.register(MaigretScanner)TransformRegistry.register(IpToAsnScanner)++# ASN-related scannersTransformRegistry.register(AsnToCidrsScanner)++# CIDR-related scannersTransformRegistry.register(CidrToIpsScanner)++# Social media scanners+TransformRegistry.register(MaigretScanner)++# Organization-related scannersTransformRegistry.register(OrgToAsnScanner)-TransformRegistry.register(DomainToAsnScanner)+TransformRegistry.register(OrgToInfosScanner)++# Cryptocurrency scannersTransformRegistry.register(CryptoWalletAddressToTransactions)TransformRegistry.register(CryptoWalletAddressToNFTs)-TransformRegistry.register(DomainToWebsiteScanner)++# Website-related scannersTransformRegistry.register(WebsiteToCrawler)TransformRegistry.register(WebsiteToLinks)TransformRegistry.register(WebsiteToDomainScanner)+TransformRegistry.register(WebsiteToWebtrackersScanner)+TransformRegistry.register(WebsiteToText)++# Email-related scannersTransformRegistry.register(EmailToGravatarScanner)TransformRegistry.register(EmailToBreachesScanner)++# Individual-related scannersTransformRegistry.register(IndividualToOrgScanner)-TransformRegistry.register(OrgToInfosScanner)-TransformRegistry.register(WebsiteToWebtrackersScanner)-TransformRegistry.register(DomainToRootDomain)++# Integration scannersTransformRegistry.register(N8nConnector)diff --git a/flowsint-transforms/src/flowsint_transforms/ips/asn_to_cidrs.py b/flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.pysimilarity index 100%rename from flowsint-transforms/src/flowsint_transforms/ips/asn_to_cidrs.pyrename to flowsint-transforms/src/flowsint_transforms/asn/to_cidrs.pydiff --git a/flowsint-transforms/src/flowsint_transforms/ips/cidr_to_ips.py b/flowsint-transforms/src/flowsint_transforms/cidr/to_ips.pysimilarity index 100%rename from flowsint-transforms/src/flowsint_transforms/ips/cidr_to_ips.pyrename to flowsint-transforms/src/flowsint_transforms/cidr/to_ips.pydiff --git a/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_nfts.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.pysimilarity index 98%rename from flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_nfts.pyrename to flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.pyindex 014cdf39..242dbbe9 100644--- a/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_nfts.py+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_nfts.py@@ -90,7 +90,7 @@ class CryptoWalletAddressToNFTs(Scanner):results: OutputType = []params = self.get_params()Logger.debug(self.sketch_id, {"message": f"{str(params)}"})- api_key = params.get("ETHERSCAN_API_KEY", None)+ api_key = params.get("ETHERSCAN_API_KEY", os.getenv("ETHERSCAN_API_KEY"))api_url = params.get("ETHERSCAN_API_URL", None)if not api_key:Logger.error(self.sketch_id, {"message": "ETHERSCAN_API_KEY is required"})diff --git a/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_transactions.py b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.pysimilarity index 98%rename from flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_transactions.pyrename to flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.pyindex 2a03ad8e..6e2e31d0 100644--- a/flowsint-transforms/src/flowsint_transforms/crypto/wallet_to_transactions.py+++ b/flowsint-transforms/src/flowsint_transforms/crypto/to_transactions.py@@ -94,8 +94,8 @@ class CryptoWalletAddressToTransactions(Scanner):async def scan(self, data: InputType) -> OutputType:results: OutputType = []params = self.get_params()- api_key = params["ETHERSCAN_API_KEY"]- api_url = params["ETHERSCAN_API_URL"]+ api_key = params.get("ETHERSCAN_API_KEY", os.getenv("ETHERSCAN_API_KEY"))+ api_url = params.get("ETHERSCAN_API_URL", ETHERSCAN_API_URL)if not api_key:Logger.error(self.sketch_id, {"message": "ETHERSCAN_API_KEY is required"})raise ValueError("ETHERSCAN_API_KEY is required")diff --git a/flowsint-transforms/src/flowsint_transforms/domains/domain_to_asn.py b/flowsint-transforms/src/flowsint_transforms/domain/to_asn.pysimilarity index 100%rename from flowsint-transforms/src/flowsint_transforms/domains/domain_to_asn.pyrename to flowsint-transforms/src/flowsint_transforms/domain/to_asn.pydiff --git a/flowsint-transforms/src/flowsint_transforms/domains/resolve.py b/flowsint-transforms/src/flowsint_transforms/domain/to_ip.pysimilarity index 98%rename from flowsint-transforms/src/flowsint_transforms/domains/resolve.pyrename to flowsint-transforms/src/flowsint_transforms/domain/to_ip.pyindex a9aa53c2..ee65289f 100644--- a/flowsint-transforms/src/flowsint_transforms/domains/resolve.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_ip.py@@ -15,7 +15,7 @@ class ResolveScanner(Scanner):@classmethoddef name(cls) -> str:- return "domain_resolve_scanner"+ return "domain_to_ip"@classmethoddef category(cls) -> str:@@ -292,7 +292,7 @@ Check FlowSint logs for detailed resolution information."domain","domain",domain_obj.domain,- type="domain" if domain_obj.root else "subdomain",+ type="domain",)self.create_node("ip", "address", ip_obj.address, type="ip")self.create_relationship(diff --git a/flowsint-transforms/src/flowsint_transforms/domains/to_root_domain.py b/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.pysimilarity index 98%rename from flowsint-transforms/src/flowsint_transforms/domains/to_root_domain.pyrename to flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.pyindex 2e66da4e..a0fcbc59 100644--- a/flowsint-transforms/src/flowsint_transforms/domains/to_root_domain.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_root_domain.py@@ -18,7 +18,7 @@ class DomainToRootDomain(Scanner):@classmethoddef name(cls) -> str:- return "to_root_domain"+ return "domain_to_root_domain"@classmethoddef category(cls) -> str:diff --git a/flowsint-transforms/src/flowsint_transforms/domains/subdomains.py b/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.pysimilarity index 98%rename from flowsint-transforms/src/flowsint_transforms/domains/subdomains.pyrename to flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.pyindex 83a11a79..9010729d 100644--- a/flowsint-transforms/src/flowsint_transforms/domains/subdomains.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py@@ -18,7 +18,7 @@ class SubdomainScanner(Scanner):@classmethoddef name(cls) -> str:- return "domain_subdomains_scanner"+ return "domain_to_subdomains"@classmethoddef category(cls) -> str:@@ -45,7 +45,7 @@ class SubdomainScanner(Scanner):async def scan(self, data: InputType) -> OutputType:"""Find subdomains using subfinder (if available) or fallback to crt.sh."""domains: OutputType = []- use_subfinder = self.__is_subfinder_installed()+ use_subfinder = not self.__is_subfinder_installed()for md in data:d = Domain(domain=md.domain)diff --git a/flowsint-transforms/src/flowsint_transforms/domains/to_website.py b/flowsint-transforms/src/flowsint_transforms/domain/to_website.pysimilarity index 99%rename from flowsint-transforms/src/flowsint_transforms/domains/to_website.pyrename to flowsint-transforms/src/flowsint_transforms/domain/to_website.pyindex 70116968..3194d4a6 100644--- a/flowsint-transforms/src/flowsint_transforms/domains/to_website.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_website.py@@ -16,7 +16,7 @@ class DomainToWebsiteScanner(Scanner):@classmethoddef name(cls) -> str:- return "to_website"+ return "domain_to_website"@classmethoddef category(cls) -> str:diff --git a/flowsint-transforms/src/flowsint_transforms/domains/whois.py b/flowsint-transforms/src/flowsint_transforms/domain/to_whois.pysimilarity index 99%rename from flowsint-transforms/src/flowsint_transforms/domains/whois.pyrename to flowsint-transforms/src/flowsint_transforms/domain/to_whois.pyindex 162887c4..615bacd0 100644--- a/flowsint-transforms/src/flowsint_transforms/domains/whois.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_whois.py@@ -17,7 +17,7 @@ class WhoisScanner(Scanner):@classmethoddef name(cls) -> str:- return "to_whois"+ return "domain_to_whois"@classmethoddef category(cls) -> str:diff --git a/flowsint-transforms/src/flowsint_transforms/emails/to_gravatar.py b/flowsint-transforms/src/flowsint_transforms/email/to_gravatar.pysimilarity index 100%rename from flowsint-transforms/src/flowsint_transforms/emails/to_gravatar.pyrename to flowsint-transforms/src/flowsint_transforms/email/to_gravatar.pydiff --git a/flowsint-transforms/src/flowsint_transforms/emails/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.pysimilarity index 100%rename from flowsint-transforms/src/flowsint_transforms/emails/to_leaks.pyrename to flowsint-transforms/src/flowsint_transforms/email/to_leaks.pydiff --git a/flowsint-transforms/src/flowsint_transforms/individuals/to_org.py b/flowsint-transforms/src/flowsint_transforms/individual/to_org.pysimilarity index 100%rename from flowsint-transforms/src/flowsint_transforms/individuals/to_org.pyrename to flowsint-transforms/src/flowsint_transforms/individual/to_org.pydiff --git a/flowsint-transforms/src/flowsint_transforms/ips/ip_to_asn.py b/flowsint-transforms/src/flowsint_transforms/ip/to_asn.pysimilarity index 100%rename from flowsint-transforms/src/flowsint_transforms/ips/ip_to_asn.pyrename to flowsint-transforms/src/flowsint_transforms/ip/to_asn.pydiff --git a/flowsint-transforms/src/flowsint_transforms/ips/reverse_resolve.py b/flowsint-transforms/src/flowsint_transforms/ip/to_domain.pysimilarity index 98%rename from flowsint-transforms/src/flowsint_transforms/ips/reverse_resolve.pyrename to flowsint-transforms/src/flowsint_transforms/ip/to_domain.pyindex 1f9542bb..0989b673 100644--- a/flowsint-transforms/src/flowsint_transforms/ips/reverse_resolve.py+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_domain.py@@ -20,7 +20,7 @@ class ReverseResolveScanner(Scanner):@classmethoddef name(cls) -> str:- return "ip_reverse_resolve_scanner"+ return "ip_to_domain_scanner"@classmethoddef category(cls) -> str:diff --git a/flowsint-transforms/src/flowsint_transforms/ips/geolocation.py b/flowsint-transforms/src/flowsint_transforms/ip/to_geolocation.pysimilarity index 99%rename from flowsint-transforms/src/flowsint_transforms/ips/geolocation.pyrename to flowsint-transforms/src/flowsint_transforms/ip/to_geolocation.pyindex 332b2060..da1509cf 100644--- a/flowsint-transforms/src/flowsint_transforms/ips/geolocation.py+++ b/flowsint-transforms/src/flowsint_transforms/ip/to_geolocation.py@@ -18,7 +18,7 @@ class GeolocationScanner(Scanner):@classmethoddef name(cls) -> str:- return "ip_geolocation_scanner"+ return "ip_to_geolocation_scanner"@classmethoddef category(cls) -> str:diff --git a/flowsint-transforms/src/flowsint_transforms/leaks/hibp.py b/flowsint-transforms/src/flowsint_transforms/leak/to_leaks.pysimilarity index 100%rename from flowsint-transforms/src/flowsint_transforms/leaks/hibp.pyrename to flowsint-transforms/src/flowsint_transforms/leak/to_leaks.pydiff --git a/flowsint-transforms/src/flowsint_transforms/organizations/__init__.py b/flowsint-transforms/src/flowsint_transforms/organization/__init__.pysimilarity index 100%rename from flowsint-transforms/src/flowsint_transforms/organizations/__init__.pyrename to flowsint-transforms/src/flowsint_transforms/organization/__init__.pydiff --git a/flowsint-transforms/src/flowsint_transforms/organizations/org_to_asn.py b/flowsint-transforms/src/flowsint_transforms/organization/to_asn.pysimilarity index 100%rename from flowsint-transforms/src/flowsint_transforms/organizations/org_to_asn.pyrename to flowsint-transforms/src/flowsint_transforms/organization/to_asn.pydiff --git a/flowsint-transforms/src/flowsint_transforms/organizations/to_infos.py b/flowsint-transforms/src/flowsint_transforms/organization/to_infos.pysimilarity index 99%rename from flowsint-transforms/src/flowsint_transforms/organizations/to_infos.pyrename to flowsint-transforms/src/flowsint_transforms/organization/to_infos.pyindex a9be4cf6..db3dd152 100644--- a/flowsint-transforms/src/flowsint_transforms/organizations/to_infos.py+++ b/flowsint-transforms/src/flowsint_transforms/organization/to_infos.py@@ -14,7 +14,7 @@ class OrgToInfosScanner(Scanner):@classmethoddef name(cls) -> str:- return "to_infos"+ return "org_to_infos"@classmethoddef category(cls) -> str:diff --git a/flowsint-transforms/src/flowsint_transforms/phones/ignorant.py b/flowsint-transforms/src/flowsint_transforms/phone/to_infos.pysimilarity index 99%rename from flowsint-transforms/src/flowsint_transforms/phones/ignorant.pyrename to flowsint-transforms/src/flowsint_transforms/phone/to_infos.pyindex 9c5cb65c..c4cfb3d4 100644--- a/flowsint-transforms/src/flowsint_transforms/phones/ignorant.py+++ b/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py@@ -14,7 +14,7 @@ class IgnorantScanner(Scanner):@classmethoddef name(cls) -> str:- return "ignorant_scanner"+ return "phone_to_infos_scanner"@classmethoddef category(cls) -> str:diff --git a/flowsint-transforms/src/flowsint_transforms/socials/maigret.py b/flowsint-transforms/src/flowsint_transforms/social/to_maigret.pysimilarity index 100%rename from flowsint-transforms/src/flowsint_transforms/socials/maigret.pyrename to flowsint-transforms/src/flowsint_transforms/social/to_maigret.pydiff --git a/flowsint-transforms/src/flowsint_transforms/socials/sherlock.py b/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.pysimilarity index 100%rename from flowsint-transforms/src/flowsint_transforms/socials/sherlock.pyrename to flowsint-transforms/src/flowsint_transforms/social/to_sherlock.pydiff --git a/flowsint-transforms/src/flowsint_transforms/websites/to_crawler.py b/flowsint-transforms/src/flowsint_transforms/website/to_crawler.pysimilarity index 99%rename from flowsint-transforms/src/flowsint_transforms/websites/to_crawler.pyrename to flowsint-transforms/src/flowsint_transforms/website/to_crawler.pyindex 72d35300..ab2fb295 100644--- a/flowsint-transforms/src/flowsint_transforms/websites/to_crawler.py+++ b/flowsint-transforms/src/flowsint_transforms/website/to_crawler.py@@ -24,7 +24,7 @@ class WebsiteToCrawler(Scanner):@classmethoddef name(cls) -> str:- return "to_crawler"+ return "website_to_crawler"@classmethoddef category(cls) -> str:diff --git a/flowsint-transforms/src/flowsint_transforms/websites/to_domain.py b/flowsint-transforms/src/flowsint_transforms/website/to_domain.pysimilarity index 63%rename from flowsint-transforms/src/flowsint_transforms/websites/to_domain.pyrename to flowsint-transforms/src/flowsint_transforms/website/to_domain.pyindex 0d1403e5..a9e0e5a5 100644--- a/flowsint-transforms/src/flowsint_transforms/websites/to_domain.py+++ b/flowsint-transforms/src/flowsint_transforms/website/to_domain.py@@ -15,7 +15,7 @@ class WebsiteToDomainScanner(Scanner):@classmethoddef name(cls) -> str:- return "to_domain"+ return "website_to_domain"@classmethoddef category(cls) -> str:@@ -43,7 +43,7 @@ class WebsiteToDomainScanner(Scanner):results: OutputType = []for website in data:try:- parsed_url = urlparse(website.url)+ parsed_url = urlparse(str(website.url))domain_name = parsed_url.netloc# Remove port if present@@ -62,7 +62,7 @@ class WebsiteToDomainScanner(Scanner):Logger.error(self.sketch_id,{- "message": f"Error extracting domain from website {website.url}: {e}"+ "message": f"Error extracting domain from website {str(website.url)}: {e}"},)continue@@ -72,6 +72,40 @@ class WebsiteToDomainScanner(Scanner):def postprocess(self, results: OutputType, input_data: InputType = None) -> OutputType:+ # Create Neo4j relationships between websites and their corresponding domains+ if input_data and self.neo4j_conn:+ for input_website, result in zip(input_data, results):+ website_url = str(input_website.url)+ domain_name = result.domain++ self.create_node(+ "website",+ "url",+ website_url,+ caption=website_url,+ type="website",+ )++ # Create relationship with the specific domain for this website+ self.create_node(+ "domain",+ "domain",+ domain_name,+ caption=domain_name,+ type="domain",+ )+ self.create_relationship(+ "website",+ "url",+ website_url,+ "domain",+ "domain",+ domain_name,+ "HAS_DOMAIN",+ )+ self.log_graph_message(+ f"Extracted domain {domain_name} from website {website_url}."+ )return resultsdiff --git a/flowsint-transforms/src/flowsint_transforms/websites/to_links.py b/flowsint-transforms/src/flowsint_transforms/website/to_links.pysimilarity index 99%rename from flowsint-transforms/src/flowsint_transforms/websites/to_links.pyrename to flowsint-transforms/src/flowsint_transforms/website/to_links.pyindex fa465eaf..bc90ee31 100644--- a/flowsint-transforms/src/flowsint_transforms/websites/to_links.py+++ b/flowsint-transforms/src/flowsint_transforms/website/to_links.py@@ -15,7 +15,7 @@ class WebsiteToLinks(Scanner):@classmethoddef name(cls) -> str:- return "to_links"+ return "website_to_links"@classmethoddef category(cls) -> str:diff --git a/flowsint-transforms/src/flowsint_transforms/website/to_text.py b/flowsint-transforms/src/flowsint_transforms/website/to_text.pynew file mode 100644index 00000000..1913ac1b--- /dev/null+++ b/flowsint-transforms/src/flowsint_transforms/website/to_text.py@@ -0,0 +1,103 @@+from typing import List, Union+from flowsint_core.core.scanner_base import Scanner+from flowsint_types.phrase import Phrase+from flowsint_types.website import Website+import requests+from bs4 import BeautifulSoup+++class WebsiteToText(Scanner):+ """Extracts the texts in a webpage."""++ InputType = List[Website]+ OutputType = List[Phrase]++ @classmethod+ def name(cls) -> str:+ return "website_to_text"++ @classmethod+ def category(cls) -> str:+ return "Website"++ @classmethod+ def key(cls) -> str:+ return "website"++ def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:+ cleaned: InputType = []+ for item in data:+ website_obj = None+ if isinstance(item, str):+ website_obj = Website(url=item)+ elif isinstance(item, dict) and "url" in item:+ website_obj = Website(url=item["url"])+ elif isinstance(item, Website):+ website_obj = item+ if website_obj:+ cleaned.append(website_obj)+ return cleaned++ async def scan(self, data: InputType) -> OutputType:+ results: OutputType = []+ for website in data:+ text_data = self._extract_text(website.url)+ if text_data:+ phrase_obj = Phrase(text=text_data)+ results.append(phrase_obj)+ return results++ def _extract_text(self, website_url: str) -> str:+ try:+ response = requests.get(website_url)+ response.raise_for_status()+ soup = BeautifulSoup(response.text, "html.parser")+ text = soup.get_text()+ return text+ except requests.exceptions.RequestException as e:+ print(f"Error fetching the URL: {e}")+ return None+ except Exception as e:+ print(f"An error occurred: {e}")+ return None++ def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ # Create Neo4j relationships between websites and their corresponding phrases+ for input_website, result in zip(original_input, results):+ website_url = str(input_website.url)++ if self.neo4j_conn:+ self.create_node(+ "website",+ "url",+ str(website_url),+ caption=str(website_url),+ type="website",+ )++ # Create relationship with the specific phrase for this website+ self.create_node(+ "phrase",+ "text",+ result.text,+ caption=result.text,+ type="phrase",+ )+ self.create_relationship(+ "website",+ "url",+ website_url,+ "phrase",+ "text",+ result.text,+ "HAS_INNER_TEXT",+ )+ self.log_graph_message(+ f"Extracted some text from the website {website_url}."+ )+ return results+++# Make types available at module level for easy access+InputType = WebsiteToText.InputType+OutputType = WebsiteToText.OutputTypediff --git a/flowsint-transforms/src/flowsint_transforms/websites/to_webtrackers.py b/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.pysimilarity index 99%rename from flowsint-transforms/src/flowsint_transforms/websites/to_webtrackers.pyrename to flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.pyindex 5e5a98f2..4a7e16ad 100644--- a/flowsint-transforms/src/flowsint_transforms/websites/to_webtrackers.py+++ b/flowsint-transforms/src/flowsint_transforms/website/to_webtrackers.py@@ -29,7 +29,7 @@ class WebsiteToWebtrackersScanner(Scanner):@classmethoddef name(cls) -> str:- return "to_webtrackers"+ return "website_to_webtrackers"@classmethoddef category(cls) -> str:diff --git a/flowsint-transforms/tests/scanners/domains/__init__.py b/flowsint-transforms/tests/scanners/domain/__init__.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/domains/__init__.pyrename to flowsint-transforms/tests/scanners/domain/__init__.pydiff --git a/flowsint-transforms/tests/scanners/domains/resolve.py b/flowsint-transforms/tests/scanners/domain/resolve.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/domains/resolve.pyrename to flowsint-transforms/tests/scanners/domain/resolve.pydiff --git a/flowsint-transforms/tests/scanners/domains/subdomains.py b/flowsint-transforms/tests/scanners/domain/subdomains.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/domains/subdomains.pyrename to flowsint-transforms/tests/scanners/domain/subdomains.pydiff --git a/flowsint-transforms/tests/scanners/domains/whois.py b/flowsint-transforms/tests/scanners/domain/whois.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/domains/whois.pyrename to flowsint-transforms/tests/scanners/domain/whois.pydiff --git a/flowsint-transforms/tests/scanners/emails/__init__.py b/flowsint-transforms/tests/scanners/email/__init__.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/emails/__init__.pyrename to flowsint-transforms/tests/scanners/email/__init__.pydiff --git a/flowsint-transforms/tests/scanners/emails/to_gravatar.py b/flowsint-transforms/tests/scanners/email/to_gravatar.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/emails/to_gravatar.pyrename to flowsint-transforms/tests/scanners/email/to_gravatar.pydiff --git a/flowsint-transforms/tests/scanners/emails/to_leaks.py b/flowsint-transforms/tests/scanners/email/to_leaks.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/emails/to_leaks.pyrename to flowsint-transforms/tests/scanners/email/to_leaks.pydiff --git a/flowsint-transforms/tests/scanners/ips/__init__.py b/flowsint-transforms/tests/scanners/ip/__init__.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/ips/__init__.pyrename to flowsint-transforms/tests/scanners/ip/__init__.pydiff --git a/flowsint-transforms/tests/scanners/ips/asn_to_cidrs.py b/flowsint-transforms/tests/scanners/ip/asn_to_cidrs.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/ips/asn_to_cidrs.pyrename to flowsint-transforms/tests/scanners/ip/asn_to_cidrs.pydiff --git a/flowsint-transforms/tests/scanners/ips/cidr_to_ips.py b/flowsint-transforms/tests/scanners/ip/cidr_to_ips.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/ips/cidr_to_ips.pyrename to flowsint-transforms/tests/scanners/ip/cidr_to_ips.pydiff --git a/flowsint-transforms/tests/scanners/ips/geolocation.py b/flowsint-transforms/tests/scanners/ip/geolocation.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/ips/geolocation.pyrename to flowsint-transforms/tests/scanners/ip/geolocation.pydiff --git a/flowsint-transforms/tests/scanners/ips/ip_to_asn.py b/flowsint-transforms/tests/scanners/ip/ip_to_asn.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/ips/ip_to_asn.pyrename to flowsint-transforms/tests/scanners/ip/ip_to_asn.pydiff --git a/flowsint-transforms/tests/scanners/organizations/__init__.py b/flowsint-transforms/tests/scanners/organization/__init__.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/organizations/__init__.pyrename to flowsint-transforms/tests/scanners/organization/__init__.pydiff --git a/flowsint-transforms/tests/scanners/organizations/to_infos.py b/flowsint-transforms/tests/scanners/organization/to_infos.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/organizations/to_infos.pyrename to flowsint-transforms/tests/scanners/organization/to_infos.pydiff --git a/flowsint-transforms/tests/scanners/socials/__init__.py b/flowsint-transforms/tests/scanners/social/__init__.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/socials/__init__.pyrename to flowsint-transforms/tests/scanners/social/__init__.pydiff --git a/flowsint-transforms/tests/scanners/socials/maigret.py b/flowsint-transforms/tests/scanners/social/maigret.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/socials/maigret.pyrename to flowsint-transforms/tests/scanners/social/maigret.pydiff --git a/flowsint-transforms/tests/scanners/websites/__init__.py b/flowsint-transforms/tests/scanners/website/__init__.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/websites/__init__.pyrename to flowsint-transforms/tests/scanners/website/__init__.pydiff --git a/flowsint-transforms/tests/scanners/websites/to_links.py b/flowsint-transforms/tests/scanners/website/to_links.pysimilarity index 100%rename from flowsint-transforms/tests/scanners/websites/to_links.pyrename to flowsint-transforms/tests/scanners/website/to_links.pydiff --git a/flowsint-types/src/flowsint_types/organization.py b/flowsint-types/src/flowsint_types/organization.pyindex 33302bbb..e7562227 100644--- a/flowsint-types/src/flowsint_types/organization.py+++ b/flowsint-types/src/flowsint_types/organization.py@@ -1,5 +1,5 @@from pydantic import BaseModel, Field-from typing import Optional, List+from typing import Any, Optional, Listfrom .individual import Individualfrom .address import PhysicalAddress@@ -8,15 +8,15 @@ class Organization(BaseModel):"""Represents an organization with detailed business and administrative information."""# Basic information- name: str = Field(..., description="Organization name", title="Organization Name")- siren: Optional[str] = Field(None, description="SIREN number", title="SIREN Number")- nom_complet: Optional[str] = Field(+ name: Any = Field(..., description="Organization name", title="Organization Name")+ siren: Optional[Any] = Field(None, description="SIREN number", title="SIREN Number")+ nom_complet: Optional[Any] = Field(None, description="Complete name", title="Complete Name")- nom_raison_sociale: Optional[str] = Field(+ nom_raison_sociale: Optional[Any] = Field(None, description="Company name", title="Company Name")- sigle: Optional[str] = Field(None, description="Acronym", title="Acronym")+ sigle: Optional[Any] = Field(None, description="Acronym", title="Acronym")# Establishment informationnombre_etablissements: Optional[int] = Field(None, description="Number of establishments", title="Number of Establishments"@@ -27,228 +27,228 @@ class Organization(BaseModel):title="Number of Open Establishments",)# Activity information- activite_principale: Optional[str] = Field(+ activite_principale: Optional[Any] = Field(None, description="Main activity code", title="Main Activity Code")- section_activite_principale: Optional[str] = Field(+ section_activite_principale: Optional[Any] = Field(None, description="Main activity section", title="Main Activity Section")# Company category- categorie_entreprise: Optional[str] = Field(+ categorie_entreprise: Optional[Any] = Field(None, description="Company category", title="Company Category")- annee_categorie_entreprise: Optional[str] = Field(+ annee_categorie_entreprise: Optional[Any] = Field(None, description="Year of company category", title="Company Category Year")# Employment information- caractere_employeur: Optional[str] = Field(+ caractere_employeur: Optional[Any] = Field(None, description="Employer character", title="Employer Character")- tranche_effectif_salarie: Optional[str] = Field(+ tranche_effectif_salarie: Optional[Any] = Field(None, description="Employee size range", title="Employee Size Range")- annee_tranche_effectif_salarie: Optional[str] = Field(+ annee_tranche_effectif_salarie: Optional[Any] = Field(None,description="Year of employee size range",title="Employee Size Range Year",)# Dates- date_creation: Optional[str] = Field(+ date_creation: Optional[Any] = Field(None, description="Creation date", title="Creation Date")- date_fermeture: Optional[str] = Field(+ date_fermeture: Optional[Any] = Field(None, description="Closing date", title="Closing Date")- date_mise_a_jour: Optional[str] = Field(+ date_mise_a_jour: Optional[Any] = Field(None, description="Update date", title="Update Date")- date_mise_a_jour_insee: Optional[str] = Field(+ date_mise_a_jour_insee: Optional[Any] = Field(None, description="INSEE update date", title="INSEE Update Date")- date_mise_a_jour_rne: Optional[str] = Field(+ date_mise_a_jour_rne: Optional[Any] = Field(None, description="RNE update date", title="RNE Update Date")# Legal information- nature_juridique: Optional[str] = Field(+ nature_juridique: Optional[Any] = Field(None, description="Legal nature", title="Legal Nature")- etat_administratif: Optional[str] = Field(- None, description="Administrative status", title="Administrative Status"+ etat_adminiAnyatif: Optional[Any] = Field(+ None, description="AdminiAnyative status", title="AdminiAnyative Status")- statut_diffusion: Optional[str] = Field(+ statut_diffusion: Optional[Any] = Field(None, description="Diffusion status", title="Diffusion Status")# Siege (Headquarters) information- siege_activite_principale: Optional[str] = Field(+ siege_activite_principale: Optional[Any] = Field(None, description="Siege main activity code", title="Headquarters Main Activity")- siege_activite_principale_registre_metier: Optional[str] = Field(+ siege_activite_principale_regiAnye_metier: Optional[Any] = Field(None,description="Siege main activity in trade register",title="Headquarters Trade Register Activity",)- siege_annee_tranche_effectif_salarie: Optional[str] = Field(+ siege_annee_tranche_effectif_salarie: Optional[Any] = Field(None,description="Siege year of employee size range",title="Headquarters Employee Size Year",)- siege_adresse: Optional[str] = Field(+ siege_adresse: Optional[Any] = Field(None, description="Siege address", title="Headquarters Address")- siege_caractere_employeur: Optional[str] = Field(+ siege_caractere_employeur: Optional[Any] = Field(None,description="Siege employer character",title="Headquarters Employer Character",)- siege_cedex: Optional[str] = Field(+ siege_cedex: Optional[Any] = Field(None, description="Siege CEDEX code", title="Headquarters CEDEX")- siege_code_pays_etranger: Optional[str] = Field(+ siege_code_pays_etranger: Optional[Any] = Field(None,description="Siege foreign country code",title="Headquarters Foreign Country Code",)- siege_code_postal: Optional[str] = Field(+ siege_code_postal: Optional[Any] = Field(None, description="Siege postal code", title="Headquarters Postal Code")- siege_commune: Optional[str] = Field(+ siege_commune: Optional[Any] = Field(None,description="Siege municipality code",title="Headquarters Municipality Code",)- siege_complement_adresse: Optional[str] = Field(+ siege_complement_adresse: Optional[Any] = Field(None,description="Siege address complement",title="Headquarters Address Complement",)- siege_coordonnees: Optional[str] = Field(+ siege_coordonnees: Optional[Any] = Field(None, description="Siege coordinates", title="Headquarters Coordinates")- siege_date_creation: Optional[str] = Field(+ siege_date_creation: Optional[Any] = Field(None, description="Siege creation date", title="Headquarters Creation Date")- siege_date_debut_activite: Optional[str] = Field(+ siege_date_debut_activite: Optional[Any] = Field(None,description="Siege activity start date",title="Headquarters Activity Start Date",)- siege_date_fermeture: Optional[str] = Field(+ siege_date_fermeture: Optional[Any] = Field(None, description="Siege closing date", title="Headquarters Closing Date")- siege_date_mise_a_jour: Optional[str] = Field(+ siege_date_mise_a_jour: Optional[Any] = Field(None, description="Siege update date", title="Headquarters Update Date")- siege_date_mise_a_jour_insee: Optional[str] = Field(+ siege_date_mise_a_jour_insee: Optional[Any] = Field(None,description="Siege INSEE update date",title="Headquarters INSEE Update Date",)- siege_departement: Optional[str] = Field(+ siege_departement: Optional[Any] = Field(None, description="Siege department code", title="Headquarters Department")- siege_distribution_speciale: Optional[str] = Field(+ siege_diAnyibution_speciale: Optional[Any] = Field(None,- description="Siege special distribution",- title="Headquarters Special Distribution",+ description="Siege special diAnyibution",+ title="Headquarters Special DiAnyibution",)- siege_epci: Optional[str] = Field(+ siege_epci: Optional[Any] = Field(None, description="Siege EPCI code", title="Headquarters EPCI Code")siege_est_siege: Optional[bool] = Field(None, description="Siege is headquarters", title="Is Headquarters")- siege_etat_administratif: Optional[str] = Field(+ siege_etat_adminiAnyatif: Optional[Any] = Field(None,- description="Siege administrative status",- title="Headquarters Administrative Status",+ description="Siege adminiAnyative status",+ title="Headquarters AdminiAnyative Status",)siege_geo_adresse: Optional[PhysicalAddress] = Field(None,description="Siege geocoded address",title="Headquarters Geocoded Address",)- siege_geo_id: Optional[str] = Field(+ siege_geo_id: Optional[Any] = Field(None, description="Siege geographic ID", title="Headquarters Geographic ID")- siege_indice_repetition: Optional[str] = Field(+ siege_indice_repetition: Optional[Any] = Field(None,description="Siege repetition index",title="Headquarters Repetition Index",)- siege_latitude: Optional[str] = Field(+ siege_latitude: Optional[Any] = Field(None, description="Siege latitude", title="Headquarters Latitude")- siege_libelle_cedex: Optional[str] = Field(+ siege_libelle_cedex: Optional[Any] = Field(None, description="Siege CEDEX label", title="Headquarters CEDEX Label")- siege_libelle_commune: Optional[str] = Field(+ siege_libelle_commune: Optional[Any] = Field(None,description="Siege municipality label",title="Headquarters Municipality Label",)- siege_libelle_commune_etranger: Optional[str] = Field(+ siege_libelle_commune_etranger: Optional[Any] = Field(None,description="Siege foreign municipality label",title="Headquarters Foreign Municipality Label",)- siege_libelle_pays_etranger: Optional[str] = Field(+ siege_libelle_pays_etranger: Optional[Any] = Field(None,description="Siege foreign country label",title="Headquarters Foreign Country Label",)- siege_libelle_voie: Optional[str] = Field(- None, description="Siege street label", title="Headquarters Street Label"+ siege_libelle_voie: Optional[Any] = Field(+ None, description="Siege Anyeet label", title="Headquarters Anyeet Label")- siege_liste_enseignes: Optional[List[str]] = Field(+ siege_liste_enseignes: Optional[List[Any]] = Field(None, description="Siege list of brands", title="Headquarters Brands List")- siege_liste_finess: Optional[List[str]] = Field(+ siege_liste_finess: Optional[List[Any]] = Field(None, description="Siege list of FINESS", title="Headquarters FINESS List")- siege_liste_id_bio: Optional[List[str]] = Field(+ siege_liste_id_bio: Optional[List[Any]] = Field(None, description="Siege list of bio IDs", title="Headquarters Bio IDs List")- siege_liste_idcc: Optional[List[str]] = Field(+ siege_liste_idcc: Optional[List[Any]] = Field(None, description="Siege list of IDCC", title="Headquarters IDCC List")- siege_liste_id_organisme_formation: Optional[List[str]] = Field(+ siege_liste_id_organisme_formation: Optional[List[Any]] = Field(None,description="Siege list of training organization IDs",title="Headquarters Training Organization IDs",)- siege_liste_rge: Optional[List[str]] = Field(+ siege_liste_rge: Optional[List[Any]] = Field(None, description="Siege list of RGE", title="Headquarters RGE List")- siege_liste_uai: Optional[List[str]] = Field(+ siege_liste_uai: Optional[List[Any]] = Field(None, description="Siege list of UAI", title="Headquarters UAI List")- siege_longitude: Optional[str] = Field(+ siege_longitude: Optional[Any] = Field(None, description="Siege longitude", title="Headquarters Longitude")- siege_nom_commercial: Optional[str] = Field(+ siege_nom_commercial: Optional[Any] = Field(None, description="Siege commercial name", title="Headquarters Commercial Name")- siege_numero_voie: Optional[str] = Field(- None, description="Siege street number", title="Headquarters Street Number"+ siege_numero_voie: Optional[Any] = Field(+ None, description="Siege Anyeet number", title="Headquarters Anyeet Number")- siege_region: Optional[str] = Field(+ siege_region: Optional[Any] = Field(None, description="Siege region code", title="Headquarters Region")- siege_siret: Optional[str] = Field(+ siege_siret: Optional[Any] = Field(None, description="Siege SIRET number", title="Headquarters SIRET")- siege_statut_diffusion_etablissement: Optional[str] = Field(+ siege_statut_diffusion_etablissement: Optional[Any] = Field(None,description="Siege establishment diffusion status",title="Headquarters Establishment Diffusion Status",)- siege_tranche_effectif_salarie: Optional[str] = Field(+ siege_tranche_effectif_salarie: Optional[Any] = Field(None,description="Siege employee size range",title="Headquarters Employee Size Range",)- siege_type_voie: Optional[str] = Field(- None, description="Siege street type", title="Headquarters Street Type"+ siege_type_voie: Optional[Any] = Field(+ None, description="Siege Anyeet type", title="Headquarters Anyeet Type")# Dirigeants (Leaders) - using Individual typedirigeants: Optional[List[Individual]] = Field(@@ -263,7 +263,7 @@ class Organization(BaseModel):None, description="Financial information", title="Financial Information")# Complements (Additional information)- complements_collectivite_territoriale: Optional[str] = Field(+ complements_collectivite_territoriale: Optional[Any] = Field(None,description="Complements territorial community",title="Territorial Community",@@ -273,7 +273,7 @@ class Organization(BaseModel):description="Complements collective agreement informed",title="Collective Agreement Informed",)- complements_liste_idcc: Optional[List[str]] = Field(+ complements_liste_idcc: Optional[List[Any]] = Field(None, description="Complements list of IDCC", title="IDCC List")complements_egapro_renseignee: Optional[bool] = Field(@@ -317,7 +317,7 @@ class Organization(BaseModel):complements_est_qualiopi: Optional[bool] = Field(None, description="Complements is Qualiopi", title="Is Qualiopi")- complements_liste_id_organisme_formation: Optional[List[str]] = Field(+ complements_liste_id_organisme_formation: Optional[List[Any]] = Field(None,description="Complements list of training organization IDs",title="Training Organization IDs",@@ -348,16 +348,16 @@ class Organization(BaseModel):description="Complements GES balance informed",title="GES Balance Informed",)- complements_identifiant_association: Optional[str] = Field(+ complements_identifiant_association: Optional[Any] = Field(None,description="Complements association identifier",title="Association Identifier",)- complements_statut_entrepreneur_spectacle: Optional[str] = Field(+ complements_statut_entrepreneur_spectacle: Optional[Any] = Field(None,description="Complements show entrepreneur status",title="Show Entrepreneur Status",)- complements_type_siae: Optional[str] = Field(+ complements_type_siae: Optional[Any] = Field(None, description="Complements SIAE type", title="SIAE Type")diff --git a/test.py b/test.pynew file mode 100644index 00000000..fe862cc3--- /dev/null+++ b/test.py@@ -0,0 +1,30 @@+import requests+from bs4 import BeautifulSoup++def extract_text_from_url(url):+ try:+ # Send a GET request to the URL+ response = requests.get(url)+ response.raise_for_status() # Raise an exception for HTTP errors++ # Parse the HTML content using BeautifulSoup+ soup = BeautifulSoup(response.text, 'html.parser')++ # Extract all the text from the page+ text = soup.get_text()++ return text+ except requests.exceptions.RequestException as e:+ print(f"Error fetching the URL: {e}")+ return None+ except Exception as e:+ print(f"An error occurred: {e}")+ return None++# Example usage+if __name__ == "__main__":+ url = input("Enter the URL of the webpage: ")+ extracted_text = extract_text_from_url(url)+ if extracted_text:+ print("Extracted Text:")+ print(extracted_text)\ No newline at end of file