Skip to content

A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.

feat: website scanner

dextmorgnd8d28a5eb910
diff --git a/flowsint-api/app/core/celery.py b/flowsint-api/app/core/celery.py
index 8dc08d80..8630b082 100644
--- a/flowsint-api/app/core/celery.py
+++ b/flowsint-api/app/core/celery.py
@@ -2,7 +2,7 @@ from celery import Celery
celery_app = Celery(
"flowsint",
- broker="redis://redis:6379/0",
- backend="redis://redis:6379/0",
+ broker="redis://127.0.0.1:6379/0",
+ backend="redis://127.0.0.1:6379/0",
include=["app.tasks.scanner"]
)
\ No newline at end of file
diff --git a/flowsint-api/app/scanners/domains/domain_infos_scanner.py b/flowsint-api/app/scanners/domains/domain_infos_scanner.py
new file mode 100644
index 00000000..d3716c43
--- /dev/null
+++ b/flowsint-api/app/scanners/domains/domain_infos_scanner.py
@@ -0,0 +1,96 @@
+import json
+import subprocess
+import uuid
+from typing import Dict, Any, List
+from pathlib import Path
+
+import requests
+
+from app.utils import extract_domain
+from app.scanners.base import Scanner
+
+class DomainInfosScanner(Scanner):
+ """Scan for subdomains via crt.sh and certificate transparency logs."""
+
+ @property
+ def name(self) -> str:
+ return "domain_infos_scanner"
+
+ def _get_subdomains_from_crtsh(self, domain: str) -> List[str]:
+ url = f"https://crt.sh/?q=%25.{domain}&output=json"
+ try:
+ resp = requests.get(url, timeout=10)
+ if resp.status_code != 200:
+ return []
+
+ data = resp.json()
+ subdomains = set()
+
+ for entry in data:
+ name = entry.get("name_value")
+ if name:
+ for sub in name.splitlines():
+ sub = sub.strip().lower().lstrip('*.')
+ if sub.endswith(domain):
+ subdomains.add(sub)
+
+ return sorted(subdomains)
+ except Exception as e:
+ return [f"[crt.sh] Error: {str(e)}"]
+
+ def scan(self, domain: str) -> Dict[str, Any]:
+ report_id = str(uuid.uuid4())
+ clean_domain = extract_domain(domain)
+ result = {"domain": clean_domain, "report_id": report_id}
+
+ try:
+ # WHOIS info
+ whois_output = subprocess.run(
+ ["whois", clean_domain],
+ capture_output=True,
+ text=True,
+ timeout=30
+ )
+ result["whois_raw"] = whois_output.stdout.strip()
+
+ # DNS lookup (IP)
+ dig_output = subprocess.run(
+ ["dig", "+short", clean_domain],
+ capture_output=True,
+ text=True,
+ timeout=10
+ )
+ ips = [line.strip() for line in dig_output.stdout.strip().split("\n") if line.strip()]
+ result["ips"] = ips
+
+ # IP Geolocation info via ipinfo.io
+ ipinfo_data = []
+ for ip in ips:
+ ipinfo_output = subprocess.run(
+ ["curl", f"https://ipinfo.io/{ip}/json"],
+ capture_output=True,
+ text=True,
+ timeout=10
+ )
+ try:
+ ipinfo_data.append(json.loads(ipinfo_output.stdout))
+ except Exception:
+ ipinfo_data.append({"ip": ip, "error": "Failed to parse ipinfo response"})
+
+ result["ipinfo"] = ipinfo_data
+
+ # crt.sh subdomains
+ result["subdomains"] = self._get_subdomains_from_crtsh(clean_domain)
+
+ return result
+
+ except subprocess.TimeoutExpired:
+ return {"error": "Domain scan timed out."}
+ except Exception as e:
+ return {"error": f"Unexpected error in Domain scan: {str(e)}"}
+
+ def postprocess(self, results: Dict[str, Any]) -> Dict[str, Any]:
+ results["scanner"] = "domain_infos_scanner"
+ results["count_ips"] = len(results.get("ips", []))
+ results["count_subdomains"] = len(results.get("subdomains", []))
+ return results
diff --git a/flowsint-api/app/scanners/holehe_scanner.py b/flowsint-api/app/scanners/emails/holehe_scanner.py
similarity index 100%
rename from flowsint-api/app/scanners/holehe_scanner.py
rename to flowsint-api/app/scanners/emails/holehe_scanner.py
diff --git a/flowsint-api/app/scanners/registry.py b/flowsint-api/app/scanners/registry.py
index 99cbb5ca..97a4be9f 100644
--- a/flowsint-api/app/scanners/registry.py
+++ b/flowsint-api/app/scanners/registry.py
@@ -1,9 +1,10 @@
from typing import Dict, Type
from app.scanners.base import Scanner
-from app.scanners.sherlock_scanner import SherlockScanner
-from app.scanners.holehe_scanner import HoleheScanner
-from app.scanners.maigret_scanner import MaigretScanner
+from app.scanners.usernames.sherlock_scanner import SherlockScanner
+from app.scanners.usernames.maigret_scanner import MaigretScanner
+from app.scanners.emails.holehe_scanner import HoleheScanner
+from app.scanners.domains.domain_infos_scanner import DomainInfosScanner
class ScannerRegistry:
@@ -29,4 +30,5 @@ class ScannerRegistry:
ScannerRegistry.register(SherlockScanner)
ScannerRegistry.register(HoleheScanner)
-ScannerRegistry.register(MaigretScanner)
\ No newline at end of file
+ScannerRegistry.register(MaigretScanner)
+ScannerRegistry.register(DomainInfosScanner)
\ No newline at end of file
diff --git a/flowsint-api/app/scanners/maigret_scanner.py b/flowsint-api/app/scanners/usernames/maigret_scanner.py
similarity index 100%
rename from flowsint-api/app/scanners/maigret_scanner.py
rename to flowsint-api/app/scanners/usernames/maigret_scanner.py
diff --git a/flowsint-api/app/scanners/sherlock_scanner.py b/flowsint-api/app/scanners/usernames/sherlock_scanner.py
similarity index 100%
rename from flowsint-api/app/scanners/sherlock_scanner.py
rename to flowsint-api/app/scanners/usernames/sherlock_scanner.py
diff --git a/flowsint-api/app/utils.py b/flowsint-api/app/utils.py
new file mode 100644
index 00000000..8890a7e8
--- /dev/null
+++ b/flowsint-api/app/utils.py
@@ -0,0 +1,4 @@
+from urllib.parse import urlparse
+def extract_domain(url_or_domain: str) -> str:
+ parsed = urlparse(url_or_domain if "://" in url_or_domain else "http://" + url_or_domain)
+ return parsed.hostname or url_or_domain
diff --git a/flowsint-web/src/components/sketches/scan-drawer.tsx b/flowsint-web/src/components/sketches/scan-drawer.tsx
index dd19f170..3e15f070 100644
--- a/flowsint-web/src/components/sketches/scan-drawer.tsx
+++ b/flowsint-web/src/components/sketches/scan-drawer.tsx
@@ -577,16 +577,16 @@ export function ScanDrawer() {
})
const RawResults = memo(() => (
- <>
- <div className="p-4">
- <pre className="overflow-auto max-h-[500px] relative rounded-md p-3 border">
+ <div className="p-4">
+ <div className="relative">
+ <pre className="overflow-auto max-h-[500px] rounded-md p-3 border">
<code>{JSON.stringify(currentScan?.results, null, 2)}</code>
<div className="absolute top-2 right-2">
<CopyButton content={JSON.stringify(currentScan?.results)} />
</div>
</pre>
</div>
- </>
+ </div>
))
return (
diff --git a/flowsint-web/src/components/sketches/sketch/profile-panel.tsx b/flowsint-web/src/components/sketches/sketch/profile-panel.tsx
index f307e94e..654222e9 100644
--- a/flowsint-web/src/components/sketches/sketch/profile-panel.tsx
+++ b/flowsint-web/src/components/sketches/sketch/profile-panel.tsx
@@ -57,6 +57,16 @@ export default function ProfilePanel({ data }: { data: any }) {
<span>Search</span>
</span>
<span className="absolute inset-0 bg-black opacity-0 hover:opacity-10 transition-opacity duration-300" />
+ </Button> : data?.type === "website" ? <Button
+ onClick={() => launchScan("website", data?.url, sketch_id as string)}
+ disabled={!data?.url}
+ className="relative min-w-[80px] h-8 overflow-hidden truncate bg-gradient-to-r from-primary to-primary/80 hover:from-primary/90 hover:to-primary transition-all duration-300 px-6 py-2 text-white border-none font-medium rounded-full"
+ >
+ <span className="flex items-center gap-2">
+ <Sparkles className={'h-4 w-4 transition-transform duration-300'} />
+ <span>Search</span>
+ </span>
+ <span className="absolute inset-0 bg-black opacity-0 hover:opacity-10 transition-opacity duration-300" />
</Button> : null}
<DropdownMenu>
<DropdownMenuTrigger asChild><Button variant={"ghost"} size={"icon"}><MoreHorizontalIcon /></Button></DropdownMenuTrigger>
diff --git a/flowsint-web/src/hooks/use-launch-scan.ts b/flowsint-web/src/hooks/use-launch-scan.ts
index 4ac4f84c..d9f50966 100644
--- a/flowsint-web/src/hooks/use-launch-scan.ts
+++ b/flowsint-web/src/hooks/use-launch-scan.ts
@@ -6,15 +6,16 @@ import { scans } from "@/lib/utils"
export function useLaunchSan() {
const { confirm } = useConfirm()
const launchScan = async (scan_name: string, value: string, sketch_id: string) => {
- if (!value || !scans.find((s) => s.name === scan_name)) {
+ const scan = scans.find((s) => s.name === scan_name)
+ if (!value || !scan) {
return toast.error("The item you want to search for was not found.")
}
const confirmed = await confirm({
- title: `${scan_name} scan`,
- message: "This scan will look for some socials that the email might be associated with. The list is not exhaustive and might return false positives.",
+ title: `${scan.scan_name} scan`,
+ message: scan.description,
})
if (!confirmed) return
- toast.promise(performSearch(value, scan_name, sketch_id), {
+ toast.promise(performSearch(value, scan.scan_name, sketch_id), {
loading: "Loading...",
success: () => `Scan on "${value}" has been launched.`,
error: (error: any) => 'An error occurred.'
diff --git a/flowsint-web/src/lib/actions/search.ts b/flowsint-web/src/lib/actions/search.ts
index 57922270..494f98d0 100644
--- a/flowsint-web/src/lib/actions/search.ts
+++ b/flowsint-web/src/lib/actions/search.ts
@@ -6,7 +6,6 @@ export async function performSearch(value: string, scan_name: string, sketch_id:
await supabase.auth.refreshSession()
const { data: { session } } = await supabase.auth.getSession();
const jwt = session?.access_token;
- console.log(jwt)
const scan = scans.find((s) => s.name === scan_name)
if (!scan)
return { error: `Task name "${scan_name}" not found.` }
diff --git a/flowsint-web/src/lib/utils.ts b/flowsint-web/src/lib/utils.ts
index 08f3ccc9..abed7d69 100644
--- a/flowsint-web/src/lib/utils.ts
+++ b/flowsint-web/src/lib/utils.ts
@@ -514,6 +514,20 @@ export function encodedRedirect(
}
export const scans = [
- { name: "email", scan_name: "holehe_scanner" },
- { name: "username", scan_name: "maigret_scanner" }
+ {
+ name: "email",
+ scan_name: "holehe_scanner",
+ description: "Check if an email address is associated with existing online accounts using Holehe."
+ },
+ {
+ name: "username",
+ scan_name: "maigret_scanner",
+ description: "Search for social media and online platform accounts associated with a given username using Maigret."
+ },
+ {
+ name: "website",
+ scan_name: "domain_infos_scanner",
+ description: "Gather public information about a domain, including IP addresses, WHOIS data, geolocation, and subdomains."
+ }
]
+