A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat: simplified edit node modal
dextmorgn2ba9ebffc4c6
diff --git a/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx b/flowsint-app/src/renderer/src/components/graphs/context-menu.tsxindex af819410..37e63304 100644--- a/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx+++ b/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx@@ -190,7 +190,7 @@ export default function ContextMenu({)}</div><div className='flex items-center gap-1'>- <FavoriteButton isFavorite={false} />+ {/* <FavoriteButton isFavorite={false} /> */}<InfoButton description={transform.description} /></div></button>@@ -260,7 +260,7 @@ export default function ContextMenu({)}</div><div className='flex items-center gap-1'>- <FavoriteButton isFavorite={false} />+ {/* <FavoriteButton isFavorite={false} /> */}<InfoButton description={flow.description} /></div></button>diff --git a/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx b/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsxindex 583e9508..8d5bc2c3 100644--- a/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx+++ b/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx@@ -7,7 +7,6 @@ import NodeActions from "../node-actions"import { GraphNode } from "@/stores/graph-store"import { Button } from "../../ui/button"import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "../../ui/tooltip"-import { useIcon } from "@/hooks/use-icon"import { useParams } from "@tanstack/react-router"import NeighborsGraph from "./neighbors"import Relationships from "./relationships"@@ -63,7 +62,7 @@ const DetailsPanel = memo(({ node }: { node: GraphNode | null }) => {</div><div className="flex-1 min-h-0"><ResizablePanelGroup direction="vertical" className="h-full">- <ResizablePanel defaultSize={15} minSize={20}>+ <ResizablePanel defaultSize={30} minSize={20}><div className="h-full overflow-auto">{node.data?.description && (<div className="px-4 py-3 border-b border-border">@@ -83,7 +82,7 @@ const DetailsPanel = memo(({ node }: { node: GraphNode | null }) => {</div></ResizablePanel><ResizableHandle />- <ResizablePanel defaultSize={35} minSize={20}>+ <ResizablePanel defaultSize={30} minSize={20}><Relationships sketchId={sketchId as string} nodeId={node.id} /></ResizablePanel></ResizablePanelGroup>diff --git a/flowsint-app/src/renderer/src/components/graphs/details-panel/node-editor-modal.tsx b/flowsint-app/src/renderer/src/components/graphs/details-panel/node-editor-modal.tsxindex b5e2bb46..693316fe 100644--- a/flowsint-app/src/renderer/src/components/graphs/details-panel/node-editor-modal.tsx+++ b/flowsint-app/src/renderer/src/components/graphs/details-panel/node-editor-modal.tsx@@ -12,11 +12,7 @@ import { MapFromAddress } from "../../map/map"import { sketchService } from "@/api/sketch-service"import { useParams } from "@tanstack/react-router"import { toast } from "sonner"-import NeighborsGraph from "./neighbors"-import { DraggableCard } from "../draggable-card"import { Avatar, AvatarFallback, AvatarImage } from "@/components/ui/avatar"-import { useNodeEditorLayoutStore, type CardType } from "@/stores/node-editor-layout-store"-import { Reorder } from "framer-motion"import { useIcon } from "@/hooks/use-icon"export const NodeEditorModal: React.FC = () => {@@ -26,11 +22,6 @@ export const NodeEditorModal: React.FC = () => {const updateNode = useGraphStore(state => state.updateNode)const setCurrentNode = useGraphStore(state => state.setCurrentNode)const { id: sketchId } = useParams({ strict: false })- const cardLayout = useNodeEditorLayoutStore(s => s.cardLayout)- const setCardLayout = useNodeEditorLayoutStore(s => s.setCardLayout)- const toggleCardEdit = useNodeEditorLayoutStore(s => s.toggleCardEdit)- const setCardEditState = useNodeEditorLayoutStore(s => s.setCardEditState)- const resetLayout = useNodeEditorLayoutStore(s => s.resetLayout)const IconComponent = useIcon(currentNode?.data.type as string, currentNode?.data?.src as string | null);const [formData, setFormData] = useState<Partial<NodeData>>({@@ -50,12 +41,6 @@ export const NodeEditorModal: React.FC = () => {caption: currentNode.data.caption || "",type: currentNode.data.type || ""});- // Reset edit states when node changes- cardLayout.forEach(card => {- if (card.isEditing) {- setCardEditState(card.id, false)- }- })}}, [currentNode]);@@ -109,14 +94,6 @@ export const NodeEditorModal: React.FC = () => {}));};- const handleLayoutReorder = (newOrder: string[]) => {- const updatedLayout = newOrder.map((id, index) => {- const existingCard = cardLayout.find(card => card.id === id)- return existingCard ? { ...existingCard, order: index } : cardLayout[index]- })- setCardLayout(updatedLayout)- }-const isLocation = currentNode?.data.type === "location" || (currentNode?.data.latitude && currentNode?.data.longitude)if (!currentNode) return null;@@ -127,238 +104,9 @@ export const NodeEditorModal: React.FC = () => {([key]) => !coreFields.includes(key) && key !== 'ID');- // Sort cards by their order- const sortedCards = [...cardLayout].sort((a, b) => a.order - b.order)-- const renderCardContent = (cardId: CardType) => {- const isEditing = cardLayout.find(card => card.id === cardId)?.isEditing || false-- switch (cardId) {- case 'description':- if (!currentNode.data.description) return null- return (- <div- className="text-sm text-muted-foreground prose prose-sm max-w-none"- dangerouslySetInnerHTML={{ __html: currentNode.data.description }}- />- )-- case 'core-properties':- return (- <div className="space-y-4">- <div className="grid grid-cols-1 cq-sm:grid-cols-2 cq-md:grid-cols-3 cq-lg:grid-cols-4 cq-xl:grid-cols-5 gap-4">- <div className="space-y-2">- <Label htmlFor="label" className="text-sm opacity-60 font-medium flex items-center gap-2">- <FileText className="h-3 w-3" />- Label- </Label>- {isEditing ? (- <Input- id="label"- value={formData.label || ""}- onChange={(e) => handleInputChange("label", e.target.value)}- placeholder="Enter node label"- className="h-9"- />- ) : (- <div className="h-9 px-3 py-2 text-sm bg-muted/30 border rounded-md flex items-center">- <span className="text-foreground">{formData.label || "No label"}</span>- </div>- )}- </div>- <div className="space-y-2">- <Label htmlFor="type" className="text-sm opacity-60 font-medium flex items-center gap-2">- <Type className="h-3 w-3" />- Type- </Label>- {isEditing ? (- <Input- id="type"- value={formData.type || ""}- onChange={(e) => handleInputChange("type", e.target.value)}- placeholder="Enter node type"- className="h-9"- />- ) : (- <div className="h-9 px-3 py-2 text-sm bg-muted/30 border rounded-md flex items-center">- <span className="text-foreground">{formData.type || "No type"}</span>- </div>- )}- </div>- </div>- <div className="space-y-2">- <Label htmlFor="caption" className="text-sm opacity-60 font-medium flex items-center gap-2">- <FileText className="h-3 w-3" />- Caption- </Label>- {isEditing ? (- <Input- id="caption"- value={formData.caption || ""}- onChange={(e) => handleInputChange("caption", e.target.value)}- placeholder="Enter node caption"- className="h-9"- />- ) : (- <div className="h-9 px-3 py-2 text-sm bg-muted/30 border rounded-md flex items-center">- <span className="text-foreground">{formData.caption || "No caption"}</span>- </div>- )}- </div>- <div className="space-y-2">- <Label htmlFor="id" className="text-sm opacity-60 font-medium flex items-center gap-2">- <Hash className="h-3 w-3" />- ID- </Label>- <Input- id="id"- value={formData.id || ""}- onChange={(e) => handleInputChange("id", e.target.value)}- className="h-9 bg-muted/50"- disabled- />- </div>- </div>- )-- case 'additional-properties':- if (additionalFields.length === 0) return null- return (- <div className="space-y-4">- {additionalFields.map(([key, value]) => (- <div key={key} className="space-y-2">- <Label- htmlFor={key}- className="text-sm opacity-60 font-medium capitalize"- >- {key.replace(/_/g, ' ')}- </Label>- {isEditing ? (- typeof value === "boolean" ? (- <div className="flex items-center gap-2">- <Button- type="button"- variant={formData[key as keyof NodeData] ? "default" : "outline"}- size="sm"- onClick={() => handleInputChange(key as keyof NodeData, (!formData[key as keyof NodeData]).toString())}- className="gap-2"- >- {formData[key as keyof NodeData] ? (- <>- <Check className="h-3 w-3" />- True- </>- ) : (- "False"- )}- </Button>- </div>- ) : typeof value === "string" && value.length > 100 ? (- <Textarea- id={key}- value={formData[key as keyof NodeData] as string || ""}- onChange={(e) => handleInputChange(key as keyof NodeData, e.target.value)}- placeholder={`Enter ${key.replace(/_/g, ' ').toLowerCase()}`}- className="min-h-[80px] resize-none"- rows={3}- />- ) : (- <Input- id={key}- value={formData[key as keyof NodeData] as string || ""}- onChange={(e) => handleInputChange(key as keyof NodeData, e.target.value)}- placeholder={`Enter ${key.replace(/_/g, ' ').toLowerCase()}`}- className="h-9"- />- )- ) : (- <div className="h-9 px-3 py-2 text-sm bg-muted/30 border rounded-md flex items-center">- <span className="text-foreground">- {typeof value === "boolean"- ? (value ? "True" : "False")- : String(value || "No value")- }- </span>- </div>- )}- </div>- ))}- </div>- )-- case 'preview':- return (- <div className="p-4 rounded-lg bg-background border">- <div className="flex items-center gap-4">- {/* Large Profile Image/Icon */}- <div className="relative flex-shrink-0">- {currentNode.data.src ? (- <Avatar className="rounded-lg h-20 w-20 border-2 border-muted">- <AvatarImage className="rounded-lg object-cover" src={currentNode.data.src} />- <AvatarFallback className="text-lg font-semibold rounded-lg">- {formData.label?.[0]?.toUpperCase() || "?"}- </AvatarFallback>- </Avatar>- ) : (- <div className="h-30 w-30 rounded-lg border-2 border-muted bg-muted/30 flex items-center justify-center">- <IconComponent className="h-15 w-15 text-muted-foreground" />- </div>- )}- </div>-- {/* Main Information */}- <div className="flex-1 min-w-0">- <h3 className="text-lg font-semibold text-foreground mb-1">- {formData.label || "No label"}- </h3>- <div className="space-y-1">- <p className="text-sm font-medium text-primary">- {formData.type || "No type"}- </p>- {formData.caption && (- <p className="text-sm text-muted-foreground line-clamp-2">- {formData.caption}- </p>- )}- </div>- </div>- </div>- </div>- )-- case 'neighbors':- return <NeighborsGraph sketchId={sketchId as string} nodeId={currentNode.id as string} />-- case 'location':- return (- <div className="overflow-hidden rounded-lg bg-background border">- {isLocation ? (- <MapFromAddress- locations={[- {- lat: formData.latitude,- lon: formData.longitude,- address: formData.label as string,- label: formData.label as string- }- ]}- />- ) : (- <div className="p-4 text-center text-muted-foreground">- <p>No location data available for this node.</p>- </div>- )}- </div>- )-- default:- return null- }- }-return (<Sheet open={openNodeEditorModal} onOpenChange={setOpenNodeEditorModal}>- <SheetContent className="!w-full !max-w-4xl h-full !duration-100 p-0 flex flex-col h-full">+ <SheetContent className="!w-full !max-w-2xl h-full !duration-100 p-0 flex flex-col h-full"><div className="flex flex-col h-full">{/* Header */}<SheetHeader className="px-6 py-4 border-b bg-muted/30 flex-shrink-0">@@ -382,40 +130,156 @@ export const NodeEditorModal: React.FC = () => {{/* Content */}<div className="flex-1 px-6 py-6 grow overflow-y-auto">- <Reorder.Group- axis="y"- values={sortedCards.map(card => card.id)}- onReorder={handleLayoutReorder}- on- layoutScroll- className="space-y-6 pb-6"- >- {sortedCards.map((card) => {- const cardContent = renderCardContent(card.id)- if (!cardContent) return null+ <div className="space-y-6">+ {/* Preview Section */}+ <div className="p-4 rounded-lg bg-background border">+ <div className="flex items-center gap-4">+ <div className="relative flex-shrink-0">+ {currentNode.data.src ? (+ <Avatar className="rounded-lg h-16 w-16 border-2 border-muted">+ <AvatarImage className="rounded-lg object-cover" src={currentNode.data.src} />+ <AvatarFallback className="text-lg font-semibold rounded-lg">+ {formData.label?.[0]?.toUpperCase() || "?"}+ </AvatarFallback>+ </Avatar>+ ) : (+ <div className="h-16 w-16 rounded-lg border-2 border-muted bg-muted/30 flex items-center justify-center">+ <IconComponent className="h-8 w-8 text-muted-foreground" />+ </div>+ )}+ </div>+ <div className="flex-1 min-w-0">+ <h3 className="text-lg font-semibold text-foreground mb-1">+ {formData.label || "No label"}+ </h3>+ <p className="text-sm font-medium text-primary">+ {formData.type || "No type"}+ </p>+ </div>+ </div>+ </div>- return (- <DraggableCard- key={card.id}- id={card.id}- title={card.id.replace(/-/g, ' ').replace(/\b\w/g, l => l.toUpperCase())}- isEditing={card.isEditing}- onToggleEdit={() => toggleCardEdit(card.id)}- showEditButton={card.id !== 'neighbors' && card.id !== 'location'}- description={- card.id === 'core-properties' ? 'Basic node information' :- card.id === 'additional-properties' ? 'Custom properties for this node' :- card.id === 'preview' ? 'How this node appears in the graph' :- card.id === 'neighbors' ? 'Connected nodes and relationships' :- card.id === 'location' ? 'Geographic location data' :- undefined- }- >- {cardContent}- </DraggableCard>- )- })}- </Reorder.Group>+ {/* Core Properties */}+ <div className="space-y-4">+ <h3 className="text-sm font-medium text-muted-foreground">Core Properties</h3>+ <div className="grid grid-cols-1 gap-4">+ <div className="space-y-2">+ <Label htmlFor="label" className="text-sm font-medium flex items-center gap-2">+ <FileText className="h-3 w-3" />+ Label+ </Label>+ <Input+ id="label"+ value={formData.label || ""}+ onChange={(e) => handleInputChange("label", e.target.value)}+ placeholder="Enter node label"+ className="h-9"+ />+ </div>+ <div className="space-y-2">+ <Label htmlFor="type" className="text-sm font-medium flex items-center gap-2">+ <Type className="h-3 w-3" />+ Type+ </Label>+ <Input+ id="type"+ value={formData.type || ""}+ onChange={(e) => handleInputChange("type", e.target.value)}+ placeholder="Enter node type"+ className="h-9"+ />+ </div>+ <div className="space-y-2">+ <Label htmlFor="caption" className="text-sm font-medium flex items-center gap-2">+ <FileText className="h-3 w-3" />+ Caption+ </Label>+ <Input+ id="caption"+ value={formData.caption || ""}+ onChange={(e) => handleInputChange("caption", e.target.value)}+ placeholder="Enter node caption"+ className="h-9"+ />+ </div>+ </div>+ </div>++ {/* Additional Properties */}+ {additionalFields.length > 0 && (+ <div className="space-y-4">+ <h3 className="text-sm font-medium text-muted-foreground">Additional Properties</h3>+ <div className="space-y-4">+ {additionalFields.map(([key, value]) => (+ <div key={key} className="space-y-2">+ <Label+ htmlFor={key}+ className="text-sm font-medium capitalize"+ >+ {key.replace(/_/g, ' ')}+ </Label>+ {typeof value === "boolean" ? (+ <div className="flex items-center gap-2">+ <Button+ type="button"+ variant={formData[key as keyof NodeData] ? "default" : "outline"}+ size="sm"+ onClick={() => handleInputChange(key as keyof NodeData, (!formData[key as keyof NodeData]).toString())}+ className="gap-2"+ >+ {formData[key as keyof NodeData] ? (+ <>+ <Check className="h-3 w-3" />+ True+ </>+ ) : (+ "False"+ )}+ </Button>+ </div>+ ) : typeof value === "string" && value.length > 100 ? (+ <Textarea+ id={key}+ value={formData[key as keyof NodeData] as string || ""}+ onChange={(e) => handleInputChange(key as keyof NodeData, e.target.value)}+ placeholder={`Enter ${key.replace(/_/g, ' ').toLowerCase()}`}+ className="min-h-[80px] resize-none"+ rows={3}+ />+ ) : (+ <Input+ id={key}+ value={formData[key as keyof NodeData] as string || ""}+ onChange={(e) => handleInputChange(key as keyof NodeData, e.target.value)}+ placeholder={`Enter ${key.replace(/_/g, ' ').toLowerCase()}`}+ className="h-9"+ />+ )}+ </div>+ ))}+ </div>+ </div>+ )}++ {/* Location Map */}+ {isLocation && (+ <div className="space-y-4">+ <h3 className="text-sm font-medium text-muted-foreground">Location</h3>+ <div className="overflow-hidden rounded-lg bg-background border">+ <MapFromAddress+ locations={[+ {+ lat: formData.latitude,+ lon: formData.longitude,+ address: formData.label as string,+ label: formData.label as string+ }+ ]}+ />+ </div>+ </div>+ )}+ </div></div>{/* Footer */}@@ -425,13 +289,6 @@ export const NodeEditorModal: React.FC = () => {{additionalFields.length} additional properties</div><div className="flex gap-2">- <Button- variant="outline"- onClick={resetLayout}- className="gap-2"- >- Reset Layout- </Button><Buttonvariant="outline"onClick={handleCancel}diff --git a/flowsint-app/src/renderer/src/components/graphs/nodes-panel.tsx b/flowsint-app/src/renderer/src/components/graphs/nodes-panel.tsxindex b3964ffc..96e7ef7c 100644--- a/flowsint-app/src/renderer/src/components/graphs/nodes-panel.tsx+++ b/flowsint-app/src/renderer/src/components/graphs/nodes-panel.tsx@@ -90,8 +90,7 @@ const NodesPanel = memo(({ nodes, isLoading }: { nodes: GraphNode[]; isLoading?:const setSelectedNodes = useGraphStore((state) => state.setSelectedNodes)const selectedNodes = useGraphStore((state) => state.selectedNodes || [])const [searchQuery, setSearchQuery] = useState<string>("")- const filters = useGraphStore(s => s.filters);- const setFilters = useGraphStore(s => s.setFilters);+ const [filters, setFilters] = useState<string[] | null>(null)const { actionItems } = useActionItems()// Ref pour le conteneur parent du virtualizerdiff --git a/flowsint-core/src/flowsint_core/core/registry.py b/flowsint-core/src/flowsint_core/core/registry.pyindex 91b8dc78..31620a51 100644--- a/flowsint-core/src/flowsint_core/core/registry.py+++ b/flowsint-core/src/flowsint_core/core/registry.py@@ -11,14 +11,11 @@ from flowsint_transforms.domain.to_root_domain import DomainToRootDomainfrom flowsint_transforms.domain.to_asn import DomainToAsnScannerfrom flowsint_transforms.domain.to_history import DomainToHistoryScanner-# IP-related scannersfrom flowsint_transforms.email.to_domains import EmailToDomainsScannerfrom flowsint_transforms.individual.to_domains import IndividualToDomainsScannerfrom flowsint_transforms.ip.to_domain import ReverseResolveScannerfrom flowsint_transforms.ip.to_geolocation import GeolocationScanner--from flowsint_transforms.ip.to_asn import IpToAsnScanner# ASN-related scanners@@ -52,6 +49,9 @@ from flowsint_transforms.website.to_webtrackers import WebsiteToWebtrackersScannfrom flowsint_transforms.email.to_gravatar import EmailToGravatarScannerfrom flowsint_transforms.email.to_leaks import EmailToBreachesScanner+# Phone-related scanners+from flowsint_transforms.phone.to_leaks import PhoneToBreachesScanner+# Individual-related scannersfrom flowsint_transforms.individual.to_org import IndividualToOrgScanner@@ -179,6 +179,9 @@ TransformRegistry.register(EmailToGravatarScanner)TransformRegistry.register(EmailToBreachesScanner)TransformRegistry.register(EmailToDomainsScanner)+# Phone-related scanners+TransformRegistry.register(PhoneToBreachesScanner)+# Individual-related scannersTransformRegistry.register(IndividualToOrgScanner)TransformRegistry.register(IndividualToDomainsScanner)diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_history.py b/flowsint-transforms/src/flowsint_transforms/domain/to_history.pyindex 9cedbcef..0d88f651 100644--- a/flowsint-transforms/src/flowsint_transforms/domain/to_history.py+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_history.py@@ -5,6 +5,7 @@ from typing import Any, List, Union, Dict, Setfrom flowsint_core.core.scanner_base import Scannerfrom flowsint_types.domain import Domainfrom flowsint_types.individual import Individual+from flowsint_types.organization import Organizationfrom flowsint_core.utils import is_valid_domain, is_root_domainfrom flowsint_types.address import PhysicalAddressfrom flowsint_core.core.logger import Logger@@ -59,6 +60,7 @@ class DomainToHistoryScanner(Scanner):domains: OutputType = []self._extracted_data = [] # Store all extracted data for postprocessself._extracted_individuals = [] # Store extracted individuals for testing+ self._extracted_organizations = [] # Store extracted organizations for testingfor domain in data:infos_data = self.__get_infos_from_whoxy(domain.domain)@@ -111,12 +113,12 @@ class DomainToHistoryScanner(Scanner):return domainsdef __process_contacts_during_scan(self, extracted_info: Dict[str, Any]):- """Process contacts and extract individuals during scan method."""+ """Process contacts and extract individuals and organizations during scan method."""domain_name = extracted_info["domain"].domaincontacts = extracted_info["contacts"]for contact_type, contact in contacts.items():- if contact and self.__has_non_redacted_info(contact):+ if contact:Logger.info(self.sketch_id,{@@ -124,7 +126,7 @@ class DomainToHistoryScanner(Scanner):},)- # Extract individual+ # Extract individual (if name is not redacted)individual = self.__extract_individual_from_contact(contact, contact_type)@@ -145,18 +147,36 @@ class DomainToHistoryScanner(Scanner):"message": f"[WHOXY] Extracted individual: {individual.full_name} ({contact_type}) for {domain_name}"},)- else:++ # Extract organization (if company name is not redacted)+ organization = self.__extract_organization_from_contact(+ contact, contact_type+ )+ if organization:+ # Store the extracted organization for testing/debugging+ organization_info = {+ "organization": organization,+ "contact_type": contact_type,+ "domain_name": domain_name,+ "original_domain": extracted_info["original_domain"].domain,+ "contact_data": contact,+ }+ self._extracted_organizations.append(organization_info)+Logger.info(self.sketch_id,{- "message": f"[WHOXY] Failed to extract individual from {contact_type} contact for {domain_name}"+ "message": f"[WHOXY] Extracted organization: {organization.name} ({contact_type}) for {domain_name}"},)- elif contact:++ # Extract other non-redacted information (country, email, etc.)+ self.__extract_additional_info_from_contact(contact, contact_type, domain_name, extracted_info["original_domain"].domain)+ else:Logger.info(self.sketch_id,{- "message": f"[WHOXY] Skipping {contact_type} contact (all info redacted) for {domain_name}"+ "message": f"[WHOXY] No contact data for {contact_type} contact for {domain_name}"},)@@ -193,30 +213,7 @@ class DomainToHistoryScanner(Scanner):# A record is valid if it has a domain name - we'll filter contacts individually laterreturn True- def __has_non_redacted_info(self, contact: Dict[str, Any]) -> bool:- """Check if a contact has any non-redacted information."""- if not contact:- return False-- # Check key fields for non-redacted information- fields_to_check = [- "full_name",- "email_address",- "phone_number",- "mailing_address",- "city_name",- "zip_code",- "country_name",- ]- for field in fields_to_check:- value = contact.get(field, "")- if value and not self.__is_redacted(value):- return True-- return False-- return Falsedef __is_redacted(self, value: str) -> bool:"""Check if a value is redacted."""@@ -230,11 +227,11 @@ class DomainToHistoryScanner(Scanner):"""Extract individual information from contact data."""full_name = contact.get("full_name", "")- # Skip if name is redacted- if self.__is_redacted(full_name):+ # Skip if name is redacted - we can't create an individual without a name+ if self.__is_redacted(full_name) or not full_name:Logger.info(self.sketch_id,- {"message": f"[WHOXY] Skipping redacted contact: {full_name}"},+ {"message": f"[WHOXY] Skipping contact with redacted/empty name: {full_name}"},)return None@@ -322,6 +319,69 @@ class DomainToHistoryScanner(Scanner):address=address, city=city, zip=zip_code, country=country)+ def __extract_organization_from_contact(+ self, contact: Dict[str, Any], contact_type: str+ ) -> Organization:+ """Extract organization information from contact data."""+ company_name = contact.get("company_name", "")++ # Skip if company name is redacted or empty+ if not company_name or self.__is_redacted(company_name):+ return None++ # Create organization object+ organization = Organization(name=company_name)++ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] Extracted organization: {company_name} ({contact_type})"+ },+ )++ return organization++ def __extract_additional_info_from_contact(+ self, contact: Dict[str, Any], contact_type: str, domain_name: str, original_domain: str+ ):+ """Extract additional non-redacted information from contact data."""+ # Extract country information+ country_name = contact.get("country_name", "")+ country_code = contact.get("country_code", "")++ if country_name and not self.__is_redacted(country_name):+ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] Found country: {country_name} ({contact_type}) for {domain_name}"+ },+ )++ if country_code and not self.__is_redacted(country_code):+ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] Found country code: {country_code} ({contact_type}) for {domain_name}"+ },+ )++ # Extract email (even if individual name is redacted)+ email_raw = contact.get("email_address", "")+ if email_raw and not self.__is_redacted(email_raw):+ emails = []+ email_list = [e.strip() for e in email_raw.split(",")]+ for email in email_list:+ if email and self.__is_valid_email(email):+ emails.append(email)++ if emails:+ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] Found emails: {emails} ({contact_type}) for {domain_name}"+ },+ )+def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:"""Create Neo4j nodes and relationships from extracted data."""if not self.neo4j_conn:@@ -341,6 +401,7 @@ class DomainToHistoryScanner(Scanner):# Track processed entities to avoid duplicatesprocessed_domains: Set[str] = set()processed_individuals: Set[str] = set()+ processed_organizations: Set[str] = set()processed_emails: Set[str] = set()processed_phones: Set[str] = set()processed_addresses: Set[str] = set()@@ -505,10 +566,83 @@ class DomainToHistoryScanner(Scanner):f"Processed individual {individual.full_name} ({contact_type}) for domain {domain_name}")+ # Process extracted organizations+ for organization_info in self._extracted_organizations:+ organization = organization_info["organization"]+ contact_type = organization_info["contact_type"]+ domain_name = organization_info["domain_name"]+ original_domain_name = organization_info["original_domain"]++ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] Processing organization: {organization.name} ({contact_type}) for {domain_name}"+ },+ )++ # Create domain node if not already processed+ if domain_name not in processed_domains:+ processed_domains.add(domain_name)+ Logger.info(+ self.sketch_id,+ {"message": f"[WHOXY] Creating domain node: {domain_name}"},+ )+ self.create_node(+ "domain",+ "domain",+ domain_name,+ label=domain_name,+ caption=domain_name,+ type="domain",+ )++ # Create relationship between original domain and found domain+ self.create_relationship(+ "domain",+ "domain",+ original_domain_name,+ "domain",+ "domain",+ domain_name,+ "HAS_RELATED_DOMAIN",+ )++ # Create organization node if not already processed+ if organization.name not in processed_organizations:+ processed_organizations.add(organization.name)+ Logger.info(+ self.sketch_id,+ {+ "message": f"[WHOXY] Creating organization node: {organization.name}"+ },+ )+ self.create_node(+ "organization",+ "name",+ organization.name,+ caption=organization.name,+ type="organization",+ )++ # Create relationship between organization and domain+ self.create_relationship(+ "organization",+ "name",+ organization.name,+ "domain",+ "domain",+ domain_name,+ f"IS_{contact_type.upper()}_CONTACT",+ )++ self.log_graph_message(+ f"Processed organization {organization.name} ({contact_type}) for domain {domain_name}"+ )+Logger.info(self.sketch_id,{- "message": f"[WHOXY] Postprocess completed. Processed {len(processed_individuals)} individuals"+ "message": f"[WHOXY] Postprocess completed. Processed {len(processed_individuals)} individuals and {len(processed_organizations)} organizations"},)diff --git a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.pyindex 7cd6dc09..70149614 100644--- a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py+++ b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py@@ -16,7 +16,7 @@ HIBP_API_KEY = os.getenv("HIBP_API_KEY")class EmailToBreachesScanner(Scanner):- """[HIBPWNED] From email to breaches using Have I Been Pwned API."""+ """[HIBPWNED] Get the breaches the email might be invovled in."""InputType = List[Email]OutputType = List[tuple] # List of (email, breach) tuplesdiff --git a/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py b/flowsint-transforms/src/flowsint_transforms/phone/to_infos.pyindex da781460..cbcebeb5 100644--- a/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py+++ b/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py@@ -3,6 +3,7 @@ from typing import Dict, Any, List, Unionfrom flowsint_core.core.scanner_base import Scannerfrom flowsint_core.utils import is_valid_numberfrom flowsint_core.core.logger import Logger+from flowsint_types.phone import Phoneimport httpx@@ -26,11 +27,16 @@ class IgnorantScanner(Scanner):def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:cleaned: InputType = []- for item in data:- if isinstance(item, str):- cleaned.append(item)- elif isinstance(item, dict) and "number" in item:- cleaned.append(item["number"])+ for number in data:+ email_obj = None+ if isinstance(number, str):+ email_obj = Phone(number=number)+ elif isinstance(number, dict) and "number" in number:+ email_obj = Phone(number=number["number"])+ elif isinstance(number, Phone):+ email_obj = number+ if email_obj:+ cleaned.append(email_obj)return cleanedasync def scan(self, data: InputType) -> OutputType:diff --git a/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.pynew file mode 100644index 00000000..a7dc6a89--- /dev/null+++ b/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.py@@ -0,0 +1,239 @@+import os+from typing import Any, Dict, List, Optional, Union+import requests+from urllib.parse import urljoin+from flowsint_core.core.scanner_base import Scanner+from flowsint_core.core.logger import Logger+from flowsint_types.phone import Phone+from flowsint_types.breach import Breach+from dotenv import load_dotenv+from flowsint_core.core.graph_db import Neo4jConnection++# Load environment variables+load_dotenv()++HIBP_API_KEY = os.getenv("HIBP_API_KEY")+++class PhoneToBreachesScanner(Scanner):+ """[HIBPWNED] Get the breaches the phone number might be invovled in."""++ InputType = List[Phone]+ OutputType = List[tuple] # List of (phone, breach) tuples++ def __init__(+ self,+ sketch_id: Optional[str] = None,+ scan_id: Optional[str] = None,+ neo4j_conn: Optional[Neo4jConnection] = None,+ vault=None,+ params: Optional[Dict[str, Any]] = None,+ ):+ super().__init__(+ sketch_id=sketch_id,+ scan_id=scan_id,+ neo4j_conn=neo4j_conn,+ params_schema=self.get_params_schema(),+ vault=vault,+ params=params,+ )++ @classmethod+ def name(cls) -> str:+ return "phone_to_breaches"++ @classmethod+ def category(cls) -> str:+ return "Email"++ @classmethod+ def key(cls) -> str:+ return "phone"++ @classmethod+ def required_params(cls) -> bool:+ return True++ @classmethod+ def get_params_schema(cls) -> List[Dict[str, Any]]:+ """Declare required parameters for this scanner"""+ return [+ {+ "name": "HIBP_API_KEY",+ "type": "vaultSecret",+ "description": "The HIBP API key to use for breaches lookup.",+ "required": False,+ "default": HIBP_API_KEY,+ },+ {+ "name": "HIBP_API_URL",+ "type": "url",+ "description": "The HIBP API URL to use for breaches lookup.",+ "required": False,+ "default": "https://haveibeenpwned.com/api/v3/breachedaccount/",+ },+ ]++ def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:+ cleaned: InputType = []+ for number in data:+ phone_obj = None+ if isinstance(number, str):+ phone_obj = Phone(number=number)+ elif isinstance(number, dict) and "number" in number:+ phone_obj = Phone(number=number["number"])+ elif isinstance(number, Phone):+ phone_obj = number+ if phone_obj:+ cleaned.append(phone_obj)+ return cleaned++ async def scan(self, data: InputType) -> OutputType:+ results: OutputType = []+ print(self.get_params())+ api_key = self.get_params().get("HIBP_API_KEY", None)+ api_url = self.get_params().get("HIBP_API_URL", None)+ if not api_key:+ Logger.error(+ self.sketch_id,+ {"message": "A valid HIBP_API_KEY is required to scan for breaches."},+ )+ if not api_url:+ Logger.error(+ self.sketch_id, {"message": "Could not find HIBP_API_URL in params."}+ )+ headers = {"hibp-api-key": api_key, "User-Agent": "FlowsInt-Scanner"}+ Logger.info(self.sketch_id, {"message": f"HIBP API URL: {api_url}"})+ for phone in data:+ try:+ # Query Have I Been Pwned API+ full_url = urljoin(api_url, f"{phone.number}?truncateResponse=false")+ Logger.error(self.sketch_id, {"message": f"full url: {full_url}"})+ response = requests.get(full_url, headers=headers, timeout=10)+ Logger.info(+ self.sketch_id, {"message": f"HIBP API response: {response.json()}"}+ )+ if response.status_code == 200:+ breaches_data = response.json()+ Logger.info(+ self.sketch_id,+ {+ "message": f"Found {len(breaches_data)} breaches for {phone.number}"+ },+ )+ for breach_data in breaches_data:+ breach = Breach(+ name=breach_data.get("Name", ""),+ title=breach_data.get("Title", ""),+ domain=breach_data.get("Domain", ""),+ breachdate=breach_data.get("BreachDate", ""),+ addeddate=breach_data.get("AddedDate", ""),+ modifieddate=breach_data.get("ModifiedDate", ""),+ pwncount=breach_data.get("PwnCount", 0),+ description=breach_data.get("Description", ""),+ dataclasses=breach_data.get("DataClasses", []),+ isverified=breach_data.get("IsVerified", False),+ isfabricated=breach_data.get("IsFabricated", False),+ issensitive=breach_data.get("IsSensitive", False),+ isretired=breach_data.get("IsRetired", False),+ isspamlist=breach_data.get("IsSpamList", False),+ logopath=breach_data.get("LogoPath", ""),+ )+ # Store phone and breach as a tuple+ results.append((phone.number, breach))+ Logger.info(+ self.sketch_id,+ {+ "message": f"Added breach: {breach.name} for phone: {phone.number}"+ },+ )++ elif response.status_code == 404:+ # No breaches found for this phone+ Logger.info(+ self.sketch_id,+ {"message": f"No breaches found for phone {phone.number}"},+ )+ continue++ else:+ Logger.error(+ self.sketch_id,+ {+ "message": f"HIBP API error for {phone.number}: {response.status_code}"+ },+ )+ continue++ except Exception as e:+ Logger.error(+ self.sketch_id,+ {+ "message": f"Error checking breaches for phone {phone.number}: {e}"+ },+ )+ continue++ Logger.info(+ self.sketch_id,+ {"message": f"Scan completed. Total results: {len(results)}"},+ )+ return results++ def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:+ Logger.info(+ self.sketch_id,+ {+ "message": f"Postprocess started. Results count: {len(results)}, Original input count: {len(original_input)}"+ },+ )++ # Create phone nodes first+ for phone_obj in original_input:+ if not self.neo4j_conn:+ continue+ # Create phone node+ self.create_node("phone", "phone", phone_obj.number, **phone_obj.__dict__)+ Logger.info(+ self.sketch_id, {"message": f"Created phone node: {phone_obj.number}"}+ )++ # Process all breaches+ for phone, breach_obj in results:+ if not self.neo4j_conn:+ continue++ # Create breach node+ breach_key = f"{breach_obj.name}_{self.sketch_id}"+ self.create_node(+ "breach",+ "breach_id",+ breach_key,+ **breach_obj.dict(),+ label=breach_obj.name,+ type="breach",+ )+ Logger.info(+ self.sketch_id, {"message": f"Created breach node: {breach_key}"}+ )++ # Create relationship between the specific phone and this breach+ self.create_relationship(+ "phone",+ "number",+ phone,+ "breach",+ "breach_id",+ breach_key,+ "FOUND_IN_BREACH",+ )+ self.log_graph_message(+ f"Breach found for phone {phone} -> {breach_obj.name} ({breach_obj.title})"+ )++ return results+++# Make types available at module level for easy access+InputType = PhoneToBreachesScanner.InputType+OutputType = PhoneToBreachesScanner.OutputType