Skip to content

A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.

feat: simplified edit node modal

dextmorgn2ba9ebffc4c6
diff --git a/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx b/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx
index af819410..37e63304 100644
--- a/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx
@@ -190,7 +190,7 @@ export default function ContextMenu({
)}
</div>
<div className='flex items-center gap-1'>
- <FavoriteButton isFavorite={false} />
+ {/* <FavoriteButton isFavorite={false} /> */}
<InfoButton description={transform.description} />
</div>
</button>
@@ -260,7 +260,7 @@ export default function ContextMenu({
)}
</div>
<div className='flex items-center gap-1'>
- <FavoriteButton isFavorite={false} />
+ {/* <FavoriteButton isFavorite={false} /> */}
<InfoButton description={flow.description} />
</div>
</button>
diff --git a/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx b/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx
index 583e9508..8d5bc2c3 100644
--- a/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx
@@ -7,7 +7,6 @@ import NodeActions from "../node-actions"
import { GraphNode } from "@/stores/graph-store"
import { Button } from "../../ui/button"
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "../../ui/tooltip"
-import { useIcon } from "@/hooks/use-icon"
import { useParams } from "@tanstack/react-router"
import NeighborsGraph from "./neighbors"
import Relationships from "./relationships"
@@ -63,7 +62,7 @@ const DetailsPanel = memo(({ node }: { node: GraphNode | null }) => {
</div>
<div className="flex-1 min-h-0">
<ResizablePanelGroup direction="vertical" className="h-full">
- <ResizablePanel defaultSize={15} minSize={20}>
+ <ResizablePanel defaultSize={30} minSize={20}>
<div className="h-full overflow-auto">
{node.data?.description && (
<div className="px-4 py-3 border-b border-border">
@@ -83,7 +82,7 @@ const DetailsPanel = memo(({ node }: { node: GraphNode | null }) => {
</div>
</ResizablePanel>
<ResizableHandle />
- <ResizablePanel defaultSize={35} minSize={20}>
+ <ResizablePanel defaultSize={30} minSize={20}>
<Relationships sketchId={sketchId as string} nodeId={node.id} />
</ResizablePanel>
</ResizablePanelGroup>
diff --git a/flowsint-app/src/renderer/src/components/graphs/details-panel/node-editor-modal.tsx b/flowsint-app/src/renderer/src/components/graphs/details-panel/node-editor-modal.tsx
index b5e2bb46..693316fe 100644
--- a/flowsint-app/src/renderer/src/components/graphs/details-panel/node-editor-modal.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/details-panel/node-editor-modal.tsx
@@ -12,11 +12,7 @@ import { MapFromAddress } from "../../map/map"
import { sketchService } from "@/api/sketch-service"
import { useParams } from "@tanstack/react-router"
import { toast } from "sonner"
-import NeighborsGraph from "./neighbors"
-import { DraggableCard } from "../draggable-card"
import { Avatar, AvatarFallback, AvatarImage } from "@/components/ui/avatar"
-import { useNodeEditorLayoutStore, type CardType } from "@/stores/node-editor-layout-store"
-import { Reorder } from "framer-motion"
import { useIcon } from "@/hooks/use-icon"
export const NodeEditorModal: React.FC = () => {
@@ -26,11 +22,6 @@ export const NodeEditorModal: React.FC = () => {
const updateNode = useGraphStore(state => state.updateNode)
const setCurrentNode = useGraphStore(state => state.setCurrentNode)
const { id: sketchId } = useParams({ strict: false })
- const cardLayout = useNodeEditorLayoutStore(s => s.cardLayout)
- const setCardLayout = useNodeEditorLayoutStore(s => s.setCardLayout)
- const toggleCardEdit = useNodeEditorLayoutStore(s => s.toggleCardEdit)
- const setCardEditState = useNodeEditorLayoutStore(s => s.setCardEditState)
- const resetLayout = useNodeEditorLayoutStore(s => s.resetLayout)
const IconComponent = useIcon(currentNode?.data.type as string, currentNode?.data?.src as string | null);
const [formData, setFormData] = useState<Partial<NodeData>>({
@@ -50,12 +41,6 @@ export const NodeEditorModal: React.FC = () => {
caption: currentNode.data.caption || "",
type: currentNode.data.type || ""
});
- // Reset edit states when node changes
- cardLayout.forEach(card => {
- if (card.isEditing) {
- setCardEditState(card.id, false)
- }
- })
}
}, [currentNode]);
@@ -109,14 +94,6 @@ export const NodeEditorModal: React.FC = () => {
}));
};
- const handleLayoutReorder = (newOrder: string[]) => {
- const updatedLayout = newOrder.map((id, index) => {
- const existingCard = cardLayout.find(card => card.id === id)
- return existingCard ? { ...existingCard, order: index } : cardLayout[index]
- })
- setCardLayout(updatedLayout)
- }
-
const isLocation = currentNode?.data.type === "location" || (currentNode?.data.latitude && currentNode?.data.longitude)
if (!currentNode) return null;
@@ -127,238 +104,9 @@ export const NodeEditorModal: React.FC = () => {
([key]) => !coreFields.includes(key) && key !== 'ID'
);
- // Sort cards by their order
- const sortedCards = [...cardLayout].sort((a, b) => a.order - b.order)
-
- const renderCardContent = (cardId: CardType) => {
- const isEditing = cardLayout.find(card => card.id === cardId)?.isEditing || false
-
- switch (cardId) {
- case 'description':
- if (!currentNode.data.description) return null
- return (
- <div
- className="text-sm text-muted-foreground prose prose-sm max-w-none"
- dangerouslySetInnerHTML={{ __html: currentNode.data.description }}
- />
- )
-
- case 'core-properties':
- return (
- <div className="space-y-4">
- <div className="grid grid-cols-1 cq-sm:grid-cols-2 cq-md:grid-cols-3 cq-lg:grid-cols-4 cq-xl:grid-cols-5 gap-4">
- <div className="space-y-2">
- <Label htmlFor="label" className="text-sm opacity-60 font-medium flex items-center gap-2">
- <FileText className="h-3 w-3" />
- Label
- </Label>
- {isEditing ? (
- <Input
- id="label"
- value={formData.label || ""}
- onChange={(e) => handleInputChange("label", e.target.value)}
- placeholder="Enter node label"
- className="h-9"
- />
- ) : (
- <div className="h-9 px-3 py-2 text-sm bg-muted/30 border rounded-md flex items-center">
- <span className="text-foreground">{formData.label || "No label"}</span>
- </div>
- )}
- </div>
- <div className="space-y-2">
- <Label htmlFor="type" className="text-sm opacity-60 font-medium flex items-center gap-2">
- <Type className="h-3 w-3" />
- Type
- </Label>
- {isEditing ? (
- <Input
- id="type"
- value={formData.type || ""}
- onChange={(e) => handleInputChange("type", e.target.value)}
- placeholder="Enter node type"
- className="h-9"
- />
- ) : (
- <div className="h-9 px-3 py-2 text-sm bg-muted/30 border rounded-md flex items-center">
- <span className="text-foreground">{formData.type || "No type"}</span>
- </div>
- )}
- </div>
- </div>
- <div className="space-y-2">
- <Label htmlFor="caption" className="text-sm opacity-60 font-medium flex items-center gap-2">
- <FileText className="h-3 w-3" />
- Caption
- </Label>
- {isEditing ? (
- <Input
- id="caption"
- value={formData.caption || ""}
- onChange={(e) => handleInputChange("caption", e.target.value)}
- placeholder="Enter node caption"
- className="h-9"
- />
- ) : (
- <div className="h-9 px-3 py-2 text-sm bg-muted/30 border rounded-md flex items-center">
- <span className="text-foreground">{formData.caption || "No caption"}</span>
- </div>
- )}
- </div>
- <div className="space-y-2">
- <Label htmlFor="id" className="text-sm opacity-60 font-medium flex items-center gap-2">
- <Hash className="h-3 w-3" />
- ID
- </Label>
- <Input
- id="id"
- value={formData.id || ""}
- onChange={(e) => handleInputChange("id", e.target.value)}
- className="h-9 bg-muted/50"
- disabled
- />
- </div>
- </div>
- )
-
- case 'additional-properties':
- if (additionalFields.length === 0) return null
- return (
- <div className="space-y-4">
- {additionalFields.map(([key, value]) => (
- <div key={key} className="space-y-2">
- <Label
- htmlFor={key}
- className="text-sm opacity-60 font-medium capitalize"
- >
- {key.replace(/_/g, ' ')}
- </Label>
- {isEditing ? (
- typeof value === "boolean" ? (
- <div className="flex items-center gap-2">
- <Button
- type="button"
- variant={formData[key as keyof NodeData] ? "default" : "outline"}
- size="sm"
- onClick={() => handleInputChange(key as keyof NodeData, (!formData[key as keyof NodeData]).toString())}
- className="gap-2"
- >
- {formData[key as keyof NodeData] ? (
- <>
- <Check className="h-3 w-3" />
- True
- </>
- ) : (
- "False"
- )}
- </Button>
- </div>
- ) : typeof value === "string" && value.length > 100 ? (
- <Textarea
- id={key}
- value={formData[key as keyof NodeData] as string || ""}
- onChange={(e) => handleInputChange(key as keyof NodeData, e.target.value)}
- placeholder={`Enter ${key.replace(/_/g, ' ').toLowerCase()}`}
- className="min-h-[80px] resize-none"
- rows={3}
- />
- ) : (
- <Input
- id={key}
- value={formData[key as keyof NodeData] as string || ""}
- onChange={(e) => handleInputChange(key as keyof NodeData, e.target.value)}
- placeholder={`Enter ${key.replace(/_/g, ' ').toLowerCase()}`}
- className="h-9"
- />
- )
- ) : (
- <div className="h-9 px-3 py-2 text-sm bg-muted/30 border rounded-md flex items-center">
- <span className="text-foreground">
- {typeof value === "boolean"
- ? (value ? "True" : "False")
- : String(value || "No value")
- }
- </span>
- </div>
- )}
- </div>
- ))}
- </div>
- )
-
- case 'preview':
- return (
- <div className="p-4 rounded-lg bg-background border">
- <div className="flex items-center gap-4">
- {/* Large Profile Image/Icon */}
- <div className="relative flex-shrink-0">
- {currentNode.data.src ? (
- <Avatar className="rounded-lg h-20 w-20 border-2 border-muted">
- <AvatarImage className="rounded-lg object-cover" src={currentNode.data.src} />
- <AvatarFallback className="text-lg font-semibold rounded-lg">
- {formData.label?.[0]?.toUpperCase() || "?"}
- </AvatarFallback>
- </Avatar>
- ) : (
- <div className="h-30 w-30 rounded-lg border-2 border-muted bg-muted/30 flex items-center justify-center">
- <IconComponent className="h-15 w-15 text-muted-foreground" />
- </div>
- )}
- </div>
-
- {/* Main Information */}
- <div className="flex-1 min-w-0">
- <h3 className="text-lg font-semibold text-foreground mb-1">
- {formData.label || "No label"}
- </h3>
- <div className="space-y-1">
- <p className="text-sm font-medium text-primary">
- {formData.type || "No type"}
- </p>
- {formData.caption && (
- <p className="text-sm text-muted-foreground line-clamp-2">
- {formData.caption}
- </p>
- )}
- </div>
- </div>
- </div>
- </div>
- )
-
- case 'neighbors':
- return <NeighborsGraph sketchId={sketchId as string} nodeId={currentNode.id as string} />
-
- case 'location':
- return (
- <div className="overflow-hidden rounded-lg bg-background border">
- {isLocation ? (
- <MapFromAddress
- locations={[
- {
- lat: formData.latitude,
- lon: formData.longitude,
- address: formData.label as string,
- label: formData.label as string
- }
- ]}
- />
- ) : (
- <div className="p-4 text-center text-muted-foreground">
- <p>No location data available for this node.</p>
- </div>
- )}
- </div>
- )
-
- default:
- return null
- }
- }
-
return (
<Sheet open={openNodeEditorModal} onOpenChange={setOpenNodeEditorModal}>
- <SheetContent className="!w-full !max-w-4xl h-full !duration-100 p-0 flex flex-col h-full">
+ <SheetContent className="!w-full !max-w-2xl h-full !duration-100 p-0 flex flex-col h-full">
<div className="flex flex-col h-full">
{/* Header */}
<SheetHeader className="px-6 py-4 border-b bg-muted/30 flex-shrink-0">
@@ -382,40 +130,156 @@ export const NodeEditorModal: React.FC = () => {
{/* Content */}
<div className="flex-1 px-6 py-6 grow overflow-y-auto">
- <Reorder.Group
- axis="y"
- values={sortedCards.map(card => card.id)}
- onReorder={handleLayoutReorder}
- on
- layoutScroll
- className="space-y-6 pb-6"
- >
- {sortedCards.map((card) => {
- const cardContent = renderCardContent(card.id)
- if (!cardContent) return null
+ <div className="space-y-6">
+ {/* Preview Section */}
+ <div className="p-4 rounded-lg bg-background border">
+ <div className="flex items-center gap-4">
+ <div className="relative flex-shrink-0">
+ {currentNode.data.src ? (
+ <Avatar className="rounded-lg h-16 w-16 border-2 border-muted">
+ <AvatarImage className="rounded-lg object-cover" src={currentNode.data.src} />
+ <AvatarFallback className="text-lg font-semibold rounded-lg">
+ {formData.label?.[0]?.toUpperCase() || "?"}
+ </AvatarFallback>
+ </Avatar>
+ ) : (
+ <div className="h-16 w-16 rounded-lg border-2 border-muted bg-muted/30 flex items-center justify-center">
+ <IconComponent className="h-8 w-8 text-muted-foreground" />
+ </div>
+ )}
+ </div>
+ <div className="flex-1 min-w-0">
+ <h3 className="text-lg font-semibold text-foreground mb-1">
+ {formData.label || "No label"}
+ </h3>
+ <p className="text-sm font-medium text-primary">
+ {formData.type || "No type"}
+ </p>
+ </div>
+ </div>
+ </div>
- return (
- <DraggableCard
- key={card.id}
- id={card.id}
- title={card.id.replace(/-/g, ' ').replace(/\b\w/g, l => l.toUpperCase())}
- isEditing={card.isEditing}
- onToggleEdit={() => toggleCardEdit(card.id)}
- showEditButton={card.id !== 'neighbors' && card.id !== 'location'}
- description={
- card.id === 'core-properties' ? 'Basic node information' :
- card.id === 'additional-properties' ? 'Custom properties for this node' :
- card.id === 'preview' ? 'How this node appears in the graph' :
- card.id === 'neighbors' ? 'Connected nodes and relationships' :
- card.id === 'location' ? 'Geographic location data' :
- undefined
- }
- >
- {cardContent}
- </DraggableCard>
- )
- })}
- </Reorder.Group>
+ {/* Core Properties */}
+ <div className="space-y-4">
+ <h3 className="text-sm font-medium text-muted-foreground">Core Properties</h3>
+ <div className="grid grid-cols-1 gap-4">
+ <div className="space-y-2">
+ <Label htmlFor="label" className="text-sm font-medium flex items-center gap-2">
+ <FileText className="h-3 w-3" />
+ Label
+ </Label>
+ <Input
+ id="label"
+ value={formData.label || ""}
+ onChange={(e) => handleInputChange("label", e.target.value)}
+ placeholder="Enter node label"
+ className="h-9"
+ />
+ </div>
+ <div className="space-y-2">
+ <Label htmlFor="type" className="text-sm font-medium flex items-center gap-2">
+ <Type className="h-3 w-3" />
+ Type
+ </Label>
+ <Input
+ id="type"
+ value={formData.type || ""}
+ onChange={(e) => handleInputChange("type", e.target.value)}
+ placeholder="Enter node type"
+ className="h-9"
+ />
+ </div>
+ <div className="space-y-2">
+ <Label htmlFor="caption" className="text-sm font-medium flex items-center gap-2">
+ <FileText className="h-3 w-3" />
+ Caption
+ </Label>
+ <Input
+ id="caption"
+ value={formData.caption || ""}
+ onChange={(e) => handleInputChange("caption", e.target.value)}
+ placeholder="Enter node caption"
+ className="h-9"
+ />
+ </div>
+ </div>
+ </div>
+
+ {/* Additional Properties */}
+ {additionalFields.length > 0 && (
+ <div className="space-y-4">
+ <h3 className="text-sm font-medium text-muted-foreground">Additional Properties</h3>
+ <div className="space-y-4">
+ {additionalFields.map(([key, value]) => (
+ <div key={key} className="space-y-2">
+ <Label
+ htmlFor={key}
+ className="text-sm font-medium capitalize"
+ >
+ {key.replace(/_/g, ' ')}
+ </Label>
+ {typeof value === "boolean" ? (
+ <div className="flex items-center gap-2">
+ <Button
+ type="button"
+ variant={formData[key as keyof NodeData] ? "default" : "outline"}
+ size="sm"
+ onClick={() => handleInputChange(key as keyof NodeData, (!formData[key as keyof NodeData]).toString())}
+ className="gap-2"
+ >
+ {formData[key as keyof NodeData] ? (
+ <>
+ <Check className="h-3 w-3" />
+ True
+ </>
+ ) : (
+ "False"
+ )}
+ </Button>
+ </div>
+ ) : typeof value === "string" && value.length > 100 ? (
+ <Textarea
+ id={key}
+ value={formData[key as keyof NodeData] as string || ""}
+ onChange={(e) => handleInputChange(key as keyof NodeData, e.target.value)}
+ placeholder={`Enter ${key.replace(/_/g, ' ').toLowerCase()}`}
+ className="min-h-[80px] resize-none"
+ rows={3}
+ />
+ ) : (
+ <Input
+ id={key}
+ value={formData[key as keyof NodeData] as string || ""}
+ onChange={(e) => handleInputChange(key as keyof NodeData, e.target.value)}
+ placeholder={`Enter ${key.replace(/_/g, ' ').toLowerCase()}`}
+ className="h-9"
+ />
+ )}
+ </div>
+ ))}
+ </div>
+ </div>
+ )}
+
+ {/* Location Map */}
+ {isLocation && (
+ <div className="space-y-4">
+ <h3 className="text-sm font-medium text-muted-foreground">Location</h3>
+ <div className="overflow-hidden rounded-lg bg-background border">
+ <MapFromAddress
+ locations={[
+ {
+ lat: formData.latitude,
+ lon: formData.longitude,
+ address: formData.label as string,
+ label: formData.label as string
+ }
+ ]}
+ />
+ </div>
+ </div>
+ )}
+ </div>
</div>
{/* Footer */}
@@ -425,13 +289,6 @@ export const NodeEditorModal: React.FC = () => {
{additionalFields.length} additional properties
</div>
<div className="flex gap-2">
- <Button
- variant="outline"
- onClick={resetLayout}
- className="gap-2"
- >
- Reset Layout
- </Button>
<Button
variant="outline"
onClick={handleCancel}
diff --git a/flowsint-app/src/renderer/src/components/graphs/nodes-panel.tsx b/flowsint-app/src/renderer/src/components/graphs/nodes-panel.tsx
index b3964ffc..96e7ef7c 100644
--- a/flowsint-app/src/renderer/src/components/graphs/nodes-panel.tsx
+++ b/flowsint-app/src/renderer/src/components/graphs/nodes-panel.tsx
@@ -90,8 +90,7 @@ const NodesPanel = memo(({ nodes, isLoading }: { nodes: GraphNode[]; isLoading?:
const setSelectedNodes = useGraphStore((state) => state.setSelectedNodes)
const selectedNodes = useGraphStore((state) => state.selectedNodes || [])
const [searchQuery, setSearchQuery] = useState<string>("")
- const filters = useGraphStore(s => s.filters);
- const setFilters = useGraphStore(s => s.setFilters);
+ const [filters, setFilters] = useState<string[] | null>(null)
const { actionItems } = useActionItems()
// Ref pour le conteneur parent du virtualizer
diff --git a/flowsint-core/src/flowsint_core/core/registry.py b/flowsint-core/src/flowsint_core/core/registry.py
index 91b8dc78..31620a51 100644
--- a/flowsint-core/src/flowsint_core/core/registry.py
+++ b/flowsint-core/src/flowsint_core/core/registry.py
@@ -11,14 +11,11 @@ from flowsint_transforms.domain.to_root_domain import DomainToRootDomain
from flowsint_transforms.domain.to_asn import DomainToAsnScanner
from flowsint_transforms.domain.to_history import DomainToHistoryScanner
-
# IP-related scanners
from flowsint_transforms.email.to_domains import EmailToDomainsScanner
from flowsint_transforms.individual.to_domains import IndividualToDomainsScanner
from flowsint_transforms.ip.to_domain import ReverseResolveScanner
from flowsint_transforms.ip.to_geolocation import GeolocationScanner
-
-
from flowsint_transforms.ip.to_asn import IpToAsnScanner
# ASN-related scanners
@@ -52,6 +49,9 @@ from flowsint_transforms.website.to_webtrackers import WebsiteToWebtrackersScann
from flowsint_transforms.email.to_gravatar import EmailToGravatarScanner
from flowsint_transforms.email.to_leaks import EmailToBreachesScanner
+# Phone-related scanners
+from flowsint_transforms.phone.to_leaks import PhoneToBreachesScanner
+
# Individual-related scanners
from flowsint_transforms.individual.to_org import IndividualToOrgScanner
@@ -179,6 +179,9 @@ TransformRegistry.register(EmailToGravatarScanner)
TransformRegistry.register(EmailToBreachesScanner)
TransformRegistry.register(EmailToDomainsScanner)
+# Phone-related scanners
+TransformRegistry.register(PhoneToBreachesScanner)
+
# Individual-related scanners
TransformRegistry.register(IndividualToOrgScanner)
TransformRegistry.register(IndividualToDomainsScanner)
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_history.py b/flowsint-transforms/src/flowsint_transforms/domain/to_history.py
index 9cedbcef..0d88f651 100644
--- a/flowsint-transforms/src/flowsint_transforms/domain/to_history.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_history.py
@@ -5,6 +5,7 @@ from typing import Any, List, Union, Dict, Set
from flowsint_core.core.scanner_base import Scanner
from flowsint_types.domain import Domain
from flowsint_types.individual import Individual
+from flowsint_types.organization import Organization
from flowsint_core.utils import is_valid_domain, is_root_domain
from flowsint_types.address import PhysicalAddress
from flowsint_core.core.logger import Logger
@@ -59,6 +60,7 @@ class DomainToHistoryScanner(Scanner):
domains: OutputType = []
self._extracted_data = [] # Store all extracted data for postprocess
self._extracted_individuals = [] # Store extracted individuals for testing
+ self._extracted_organizations = [] # Store extracted organizations for testing
for domain in data:
infos_data = self.__get_infos_from_whoxy(domain.domain)
@@ -111,12 +113,12 @@ class DomainToHistoryScanner(Scanner):
return domains
def __process_contacts_during_scan(self, extracted_info: Dict[str, Any]):
- """Process contacts and extract individuals during scan method."""
+ """Process contacts and extract individuals and organizations during scan method."""
domain_name = extracted_info["domain"].domain
contacts = extracted_info["contacts"]
for contact_type, contact in contacts.items():
- if contact and self.__has_non_redacted_info(contact):
+ if contact:
Logger.info(
self.sketch_id,
{
@@ -124,7 +126,7 @@ class DomainToHistoryScanner(Scanner):
},
)
- # Extract individual
+ # Extract individual (if name is not redacted)
individual = self.__extract_individual_from_contact(
contact, contact_type
)
@@ -145,18 +147,36 @@ class DomainToHistoryScanner(Scanner):
"message": f"[WHOXY] Extracted individual: {individual.full_name} ({contact_type}) for {domain_name}"
},
)
- else:
+
+ # Extract organization (if company name is not redacted)
+ organization = self.__extract_organization_from_contact(
+ contact, contact_type
+ )
+ if organization:
+ # Store the extracted organization for testing/debugging
+ organization_info = {
+ "organization": organization,
+ "contact_type": contact_type,
+ "domain_name": domain_name,
+ "original_domain": extracted_info["original_domain"].domain,
+ "contact_data": contact,
+ }
+ self._extracted_organizations.append(organization_info)
+
Logger.info(
self.sketch_id,
{
- "message": f"[WHOXY] Failed to extract individual from {contact_type} contact for {domain_name}"
+ "message": f"[WHOXY] Extracted organization: {organization.name} ({contact_type}) for {domain_name}"
},
)
- elif contact:
+
+ # Extract other non-redacted information (country, email, etc.)
+ self.__extract_additional_info_from_contact(contact, contact_type, domain_name, extracted_info["original_domain"].domain)
+ else:
Logger.info(
self.sketch_id,
{
- "message": f"[WHOXY] Skipping {contact_type} contact (all info redacted) for {domain_name}"
+ "message": f"[WHOXY] No contact data for {contact_type} contact for {domain_name}"
},
)
@@ -193,30 +213,7 @@ class DomainToHistoryScanner(Scanner):
# A record is valid if it has a domain name - we'll filter contacts individually later
return True
- def __has_non_redacted_info(self, contact: Dict[str, Any]) -> bool:
- """Check if a contact has any non-redacted information."""
- if not contact:
- return False
-
- # Check key fields for non-redacted information
- fields_to_check = [
- "full_name",
- "email_address",
- "phone_number",
- "mailing_address",
- "city_name",
- "zip_code",
- "country_name",
- ]
- for field in fields_to_check:
- value = contact.get(field, "")
- if value and not self.__is_redacted(value):
- return True
-
- return False
-
- return False
def __is_redacted(self, value: str) -> bool:
"""Check if a value is redacted."""
@@ -230,11 +227,11 @@ class DomainToHistoryScanner(Scanner):
"""Extract individual information from contact data."""
full_name = contact.get("full_name", "")
- # Skip if name is redacted
- if self.__is_redacted(full_name):
+ # Skip if name is redacted - we can't create an individual without a name
+ if self.__is_redacted(full_name) or not full_name:
Logger.info(
self.sketch_id,
- {"message": f"[WHOXY] Skipping redacted contact: {full_name}"},
+ {"message": f"[WHOXY] Skipping contact with redacted/empty name: {full_name}"},
)
return None
@@ -322,6 +319,69 @@ class DomainToHistoryScanner(Scanner):
address=address, city=city, zip=zip_code, country=country
)
+ def __extract_organization_from_contact(
+ self, contact: Dict[str, Any], contact_type: str
+ ) -> Organization:
+ """Extract organization information from contact data."""
+ company_name = contact.get("company_name", "")
+
+ # Skip if company name is redacted or empty
+ if not company_name or self.__is_redacted(company_name):
+ return None
+
+ # Create organization object
+ organization = Organization(name=company_name)
+
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"[WHOXY] Extracted organization: {company_name} ({contact_type})"
+ },
+ )
+
+ return organization
+
+ def __extract_additional_info_from_contact(
+ self, contact: Dict[str, Any], contact_type: str, domain_name: str, original_domain: str
+ ):
+ """Extract additional non-redacted information from contact data."""
+ # Extract country information
+ country_name = contact.get("country_name", "")
+ country_code = contact.get("country_code", "")
+
+ if country_name and not self.__is_redacted(country_name):
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"[WHOXY] Found country: {country_name} ({contact_type}) for {domain_name}"
+ },
+ )
+
+ if country_code and not self.__is_redacted(country_code):
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"[WHOXY] Found country code: {country_code} ({contact_type}) for {domain_name}"
+ },
+ )
+
+ # Extract email (even if individual name is redacted)
+ email_raw = contact.get("email_address", "")
+ if email_raw and not self.__is_redacted(email_raw):
+ emails = []
+ email_list = [e.strip() for e in email_raw.split(",")]
+ for email in email_list:
+ if email and self.__is_valid_email(email):
+ emails.append(email)
+
+ if emails:
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"[WHOXY] Found emails: {emails} ({contact_type}) for {domain_name}"
+ },
+ )
+
def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
"""Create Neo4j nodes and relationships from extracted data."""
if not self.neo4j_conn:
@@ -341,6 +401,7 @@ class DomainToHistoryScanner(Scanner):
# Track processed entities to avoid duplicates
processed_domains: Set[str] = set()
processed_individuals: Set[str] = set()
+ processed_organizations: Set[str] = set()
processed_emails: Set[str] = set()
processed_phones: Set[str] = set()
processed_addresses: Set[str] = set()
@@ -505,10 +566,83 @@ class DomainToHistoryScanner(Scanner):
f"Processed individual {individual.full_name} ({contact_type}) for domain {domain_name}"
)
+ # Process extracted organizations
+ for organization_info in self._extracted_organizations:
+ organization = organization_info["organization"]
+ contact_type = organization_info["contact_type"]
+ domain_name = organization_info["domain_name"]
+ original_domain_name = organization_info["original_domain"]
+
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"[WHOXY] Processing organization: {organization.name} ({contact_type}) for {domain_name}"
+ },
+ )
+
+ # Create domain node if not already processed
+ if domain_name not in processed_domains:
+ processed_domains.add(domain_name)
+ Logger.info(
+ self.sketch_id,
+ {"message": f"[WHOXY] Creating domain node: {domain_name}"},
+ )
+ self.create_node(
+ "domain",
+ "domain",
+ domain_name,
+ label=domain_name,
+ caption=domain_name,
+ type="domain",
+ )
+
+ # Create relationship between original domain and found domain
+ self.create_relationship(
+ "domain",
+ "domain",
+ original_domain_name,
+ "domain",
+ "domain",
+ domain_name,
+ "HAS_RELATED_DOMAIN",
+ )
+
+ # Create organization node if not already processed
+ if organization.name not in processed_organizations:
+ processed_organizations.add(organization.name)
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"[WHOXY] Creating organization node: {organization.name}"
+ },
+ )
+ self.create_node(
+ "organization",
+ "name",
+ organization.name,
+ caption=organization.name,
+ type="organization",
+ )
+
+ # Create relationship between organization and domain
+ self.create_relationship(
+ "organization",
+ "name",
+ organization.name,
+ "domain",
+ "domain",
+ domain_name,
+ f"IS_{contact_type.upper()}_CONTACT",
+ )
+
+ self.log_graph_message(
+ f"Processed organization {organization.name} ({contact_type}) for domain {domain_name}"
+ )
+
Logger.info(
self.sketch_id,
{
- "message": f"[WHOXY] Postprocess completed. Processed {len(processed_individuals)} individuals"
+ "message": f"[WHOXY] Postprocess completed. Processed {len(processed_individuals)} individuals and {len(processed_organizations)} organizations"
},
)
diff --git a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py
index 7cd6dc09..70149614 100644
--- a/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py
+++ b/flowsint-transforms/src/flowsint_transforms/email/to_leaks.py
@@ -16,7 +16,7 @@ HIBP_API_KEY = os.getenv("HIBP_API_KEY")
class EmailToBreachesScanner(Scanner):
- """[HIBPWNED] From email to breaches using Have I Been Pwned API."""
+ """[HIBPWNED] Get the breaches the email might be invovled in."""
InputType = List[Email]
OutputType = List[tuple] # List of (email, breach) tuples
diff --git a/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py b/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py
index da781460..cbcebeb5 100644
--- a/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py
+++ b/flowsint-transforms/src/flowsint_transforms/phone/to_infos.py
@@ -3,6 +3,7 @@ from typing import Dict, Any, List, Union
from flowsint_core.core.scanner_base import Scanner
from flowsint_core.utils import is_valid_number
from flowsint_core.core.logger import Logger
+from flowsint_types.phone import Phone
import httpx
@@ -26,11 +27,16 @@ class IgnorantScanner(Scanner):
def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:
cleaned: InputType = []
- for item in data:
- if isinstance(item, str):
- cleaned.append(item)
- elif isinstance(item, dict) and "number" in item:
- cleaned.append(item["number"])
+ for number in data:
+ email_obj = None
+ if isinstance(number, str):
+ email_obj = Phone(number=number)
+ elif isinstance(number, dict) and "number" in number:
+ email_obj = Phone(number=number["number"])
+ elif isinstance(number, Phone):
+ email_obj = number
+ if email_obj:
+ cleaned.append(email_obj)
return cleaned
async def scan(self, data: InputType) -> OutputType:
diff --git a/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.py b/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.py
new file mode 100644
index 00000000..a7dc6a89
--- /dev/null
+++ b/flowsint-transforms/src/flowsint_transforms/phone/to_leaks.py
@@ -0,0 +1,239 @@
+import os
+from typing import Any, Dict, List, Optional, Union
+import requests
+from urllib.parse import urljoin
+from flowsint_core.core.scanner_base import Scanner
+from flowsint_core.core.logger import Logger
+from flowsint_types.phone import Phone
+from flowsint_types.breach import Breach
+from dotenv import load_dotenv
+from flowsint_core.core.graph_db import Neo4jConnection
+
+# Load environment variables
+load_dotenv()
+
+HIBP_API_KEY = os.getenv("HIBP_API_KEY")
+
+
+class PhoneToBreachesScanner(Scanner):
+ """[HIBPWNED] Get the breaches the phone number might be invovled in."""
+
+ InputType = List[Phone]
+ OutputType = List[tuple] # List of (phone, breach) tuples
+
+ def __init__(
+ self,
+ sketch_id: Optional[str] = None,
+ scan_id: Optional[str] = None,
+ neo4j_conn: Optional[Neo4jConnection] = None,
+ vault=None,
+ params: Optional[Dict[str, Any]] = None,
+ ):
+ super().__init__(
+ sketch_id=sketch_id,
+ scan_id=scan_id,
+ neo4j_conn=neo4j_conn,
+ params_schema=self.get_params_schema(),
+ vault=vault,
+ params=params,
+ )
+
+ @classmethod
+ def name(cls) -> str:
+ return "phone_to_breaches"
+
+ @classmethod
+ def category(cls) -> str:
+ return "Email"
+
+ @classmethod
+ def key(cls) -> str:
+ return "phone"
+
+ @classmethod
+ def required_params(cls) -> bool:
+ return True
+
+ @classmethod
+ def get_params_schema(cls) -> List[Dict[str, Any]]:
+ """Declare required parameters for this scanner"""
+ return [
+ {
+ "name": "HIBP_API_KEY",
+ "type": "vaultSecret",
+ "description": "The HIBP API key to use for breaches lookup.",
+ "required": False,
+ "default": HIBP_API_KEY,
+ },
+ {
+ "name": "HIBP_API_URL",
+ "type": "url",
+ "description": "The HIBP API URL to use for breaches lookup.",
+ "required": False,
+ "default": "https://haveibeenpwned.com/api/v3/breachedaccount/",
+ },
+ ]
+
+ def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType:
+ cleaned: InputType = []
+ for number in data:
+ phone_obj = None
+ if isinstance(number, str):
+ phone_obj = Phone(number=number)
+ elif isinstance(number, dict) and "number" in number:
+ phone_obj = Phone(number=number["number"])
+ elif isinstance(number, Phone):
+ phone_obj = number
+ if phone_obj:
+ cleaned.append(phone_obj)
+ return cleaned
+
+ async def scan(self, data: InputType) -> OutputType:
+ results: OutputType = []
+ print(self.get_params())
+ api_key = self.get_params().get("HIBP_API_KEY", None)
+ api_url = self.get_params().get("HIBP_API_URL", None)
+ if not api_key:
+ Logger.error(
+ self.sketch_id,
+ {"message": "A valid HIBP_API_KEY is required to scan for breaches."},
+ )
+ if not api_url:
+ Logger.error(
+ self.sketch_id, {"message": "Could not find HIBP_API_URL in params."}
+ )
+ headers = {"hibp-api-key": api_key, "User-Agent": "FlowsInt-Scanner"}
+ Logger.info(self.sketch_id, {"message": f"HIBP API URL: {api_url}"})
+ for phone in data:
+ try:
+ # Query Have I Been Pwned API
+ full_url = urljoin(api_url, f"{phone.number}?truncateResponse=false")
+ Logger.error(self.sketch_id, {"message": f"full url: {full_url}"})
+ response = requests.get(full_url, headers=headers, timeout=10)
+ Logger.info(
+ self.sketch_id, {"message": f"HIBP API response: {response.json()}"}
+ )
+ if response.status_code == 200:
+ breaches_data = response.json()
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"Found {len(breaches_data)} breaches for {phone.number}"
+ },
+ )
+ for breach_data in breaches_data:
+ breach = Breach(
+ name=breach_data.get("Name", ""),
+ title=breach_data.get("Title", ""),
+ domain=breach_data.get("Domain", ""),
+ breachdate=breach_data.get("BreachDate", ""),
+ addeddate=breach_data.get("AddedDate", ""),
+ modifieddate=breach_data.get("ModifiedDate", ""),
+ pwncount=breach_data.get("PwnCount", 0),
+ description=breach_data.get("Description", ""),
+ dataclasses=breach_data.get("DataClasses", []),
+ isverified=breach_data.get("IsVerified", False),
+ isfabricated=breach_data.get("IsFabricated", False),
+ issensitive=breach_data.get("IsSensitive", False),
+ isretired=breach_data.get("IsRetired", False),
+ isspamlist=breach_data.get("IsSpamList", False),
+ logopath=breach_data.get("LogoPath", ""),
+ )
+ # Store phone and breach as a tuple
+ results.append((phone.number, breach))
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"Added breach: {breach.name} for phone: {phone.number}"
+ },
+ )
+
+ elif response.status_code == 404:
+ # No breaches found for this phone
+ Logger.info(
+ self.sketch_id,
+ {"message": f"No breaches found for phone {phone.number}"},
+ )
+ continue
+
+ else:
+ Logger.error(
+ self.sketch_id,
+ {
+ "message": f"HIBP API error for {phone.number}: {response.status_code}"
+ },
+ )
+ continue
+
+ except Exception as e:
+ Logger.error(
+ self.sketch_id,
+ {
+ "message": f"Error checking breaches for phone {phone.number}: {e}"
+ },
+ )
+ continue
+
+ Logger.info(
+ self.sketch_id,
+ {"message": f"Scan completed. Total results: {len(results)}"},
+ )
+ return results
+
+ def postprocess(self, results: OutputType, original_input: InputType) -> OutputType:
+ Logger.info(
+ self.sketch_id,
+ {
+ "message": f"Postprocess started. Results count: {len(results)}, Original input count: {len(original_input)}"
+ },
+ )
+
+ # Create phone nodes first
+ for phone_obj in original_input:
+ if not self.neo4j_conn:
+ continue
+ # Create phone node
+ self.create_node("phone", "phone", phone_obj.number, **phone_obj.__dict__)
+ Logger.info(
+ self.sketch_id, {"message": f"Created phone node: {phone_obj.number}"}
+ )
+
+ # Process all breaches
+ for phone, breach_obj in results:
+ if not self.neo4j_conn:
+ continue
+
+ # Create breach node
+ breach_key = f"{breach_obj.name}_{self.sketch_id}"
+ self.create_node(
+ "breach",
+ "breach_id",
+ breach_key,
+ **breach_obj.dict(),
+ label=breach_obj.name,
+ type="breach",
+ )
+ Logger.info(
+ self.sketch_id, {"message": f"Created breach node: {breach_key}"}
+ )
+
+ # Create relationship between the specific phone and this breach
+ self.create_relationship(
+ "phone",
+ "number",
+ phone,
+ "breach",
+ "breach_id",
+ breach_key,
+ "FOUND_IN_BREACH",
+ )
+ self.log_graph_message(
+ f"Breach found for phone {phone} -> {breach_obj.name} ({breach_obj.title})"
+ )
+
+ return results
+
+
+# Make types available at module level for easy access
+InputType = PhoneToBreachesScanner.InputType
+OutputType = PhoneToBreachesScanner.OutputType