reconFTW is a tool designed to perform automated recon on a target domain by running the best set of tools to perform scanning and finding out vulnerabilities
Shell bug-bountybugbountybugbounty-tooldnshackingnucleiosintpenetration-testingpentestpentest-toolpentestingreconreconnaissancescannersecuritysecurity-toolssubdomainsubdomain-enumerationvulnerabilities
#!/bin/bash
# shellcheck disable=SC2154 # Variables defined in reconftw.cfg
# reconFTW - OSINT module
# Contains: google_dorks, github_dorks, github_repos, github_leaks, metadata,
# apileaks, emails, domain_info, third_party_misconfigs, spoof,
# mail_hygiene, cloud_enum_scan, ip_info
# This file is sourced by reconftw.sh - do not execute directly
[[ -z "${SCRIPTPATH:-}" ]] && {
echo "Error: This module must be sourced by reconftw.sh" >&2
exit 1
}
#####################################################################cc##########################################
################################################### OSINT #####################################################
###############################################################################################################
function google_dorks() {
ensure_dirs osint
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $GOOGLE_DORKS == true ]] && [[ $OSINT == true ]]; then
start_func "${FUNCNAME[0]}" "Running: Google Dorks in process"
if ! run_command "${tools}/dorks_hunter/venv/bin/python3" "${tools}/dorks_hunter/dorks_hunter.py" -d "$domain" -o "osint/dorks.txt" 2>>"$LOGFILE" >/dev/null; then
_print_error "dorks_hunter command failed"
fi
end_func "Results are saved in $domain/osint/dorks.txt" "${FUNCNAME[0]}"
else
if [[ $GOOGLE_DORKS == false ]] || [[ $OSINT == false ]]; then
skip_notification "disabled"
else
skip_notification "processed"
fi
fi
}
function github_dorks() {
ensure_dirs osint
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $GITHUB_DORKS == true ]] && [[ $OSINT == true ]]; then
start_func "${FUNCNAME[0]}" "Running: Github Dorks in process"
if [[ -s $GITHUB_TOKENS ]]; then
if [[ $DEEP == true ]]; then
if ! run_command gitdorks_go -gd "${tools}/gitdorks_go/Dorks/medium_dorks.txt" -nws 20 -target "$domain" -tf "$GITHUB_TOKENS" -ew 3 | anew -q osint/gitdorks.txt; then
_print_error "gitdorks_go command failed"
return 1
fi
else
if ! run_command gitdorks_go -gd "${tools}/gitdorks_go/Dorks/smalldorks.txt" -nws 20 -target "$domain" -tf "$GITHUB_TOKENS" -ew 3 | anew -q osint/gitdorks.txt; then
_print_error "gitdorks_go command failed"
return 1
fi
fi
else
_print_error "Required file ${GITHUB_TOKENS} does not exist or is empty"
return 1
fi
end_func "Results are saved in $domain/osint/gitdorks.txt" "${FUNCNAME[0]}"
else
if [[ $GITHUB_DORKS == false ]] || [[ $OSINT == false ]]; then
skip_notification "disabled"
else
skip_notification "processed"
fi
fi
}
function github_repos() {
ensure_dirs osint
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $GITHUB_REPOS == true ]] && [[ $OSINT == true ]]; then
start_func "${FUNCNAME[0]}" "Github Repos analysis in process"
if [[ ! -s $GITHUB_TOKENS ]]; then
_print_error "Required file ${GITHUB_TOKENS} does not exist or is empty"
return 1
fi
GH_TOKEN=$(head -n 1 "$GITHUB_TOKENS")
register_secret "$GH_TOKEN"
echo "$domain" | unfurl format %r >.tmp/company_name.txt
# Use temp file for token to avoid exposing it in process list
local _token_file
_token_file=$(mktemp)
printf '%s' "$GH_TOKEN" > "$_token_file"
if ! run_command enumerepo -token-file "$_token_file" -usernames .tmp/company_name.txt -o .tmp/company_repos.txt 2>>"$LOGFILE" >/dev/null; then
_print_error "enumerepo command failed"
fi
rm -f "$_token_file"
if [[ -s ".tmp/company_repos.txt" ]]; then
if ! jq -r '.[].repos[]|.url' <.tmp/company_repos.txt >.tmp/company_repos_url.txt 2>>"$LOGFILE"; then
_print_error "jq command failed"
fi
fi
ensure_dirs .tmp/github_repos .tmp/github
if [[ -s ".tmp/company_repos_url.txt" ]]; then
# argv-safe parallel replacement for interlace -c. Each clone runs in a
# background subshell; failures are recorded via a marker file so the
# caller can log a summary after wait.
local _clone_fail_dir
_clone_fail_dir=$(mktemp -d)
while IFS= read -r _repo_target; do
[[ -z "$_repo_target" ]] && continue
(
_repo_safe=$(printf '%s' "$_repo_target" | sha256sum | cut -d' ' -f1)
if ! git clone "$_repo_target" ".tmp/github_repos/${_repo_safe}" 2>>"$LOGFILE" >/dev/null; then
: >"$_clone_fail_dir/$_repo_safe"
fi
) &
_throttle_jobs "${INTERLACE_THREADS:-10}"
done <.tmp/company_repos_url.txt
wait
local _clone_failures
_clone_failures=$(find "$_clone_fail_dir" -type f 2>/dev/null | wc -l | tr -d ' ')
rm -rf "$_clone_fail_dir"
if [[ "$_clone_failures" -gt 0 ]]; then
_print_error "git clone failed for $_clone_failures repo(s); continuing with successful clones"
fi
else
log_note "No GitHub repos found for ${domain}; continuing" "${FUNCNAME[0]}" "${LINENO}"
end_func "Results are saved in $domain/osint/github_company_secrets.json" "${FUNCNAME[0]}"
return 0
fi
if [[ -d ".tmp/github_repos/" ]]; then
ls .tmp/github_repos >.tmp/github_repos_folders.txt
else
log_note "GitHub clone directory missing for ${domain}; no repo artifacts to process" "${FUNCNAME[0]}" "${LINENO}"
end_func "Results are saved in $domain/osint/github_company_secrets.json" "${FUNCNAME[0]}"
return 0
fi
if [[ ! -s ".tmp/github_repos_folders.txt" ]]; then
log_note "No cloned GitHub repos to scan for ${domain}" "${FUNCNAME[0]}" "${LINENO}"
end_func "Results are saved in $domain/osint/github_company_secrets.json" "${FUNCNAME[0]}"
return 0
fi
local secrets_engine="${SECRETS_ENGINE:-titus}"
local titus_bin=""
if command -v titus >/dev/null 2>&1; then
titus_bin="$(command -v titus)"
elif [[ -x "${tools}/titus/dist/titus" ]]; then
titus_bin="${tools}/titus/dist/titus"
elif [[ -x "${HOME}/go/bin/titus" ]]; then
titus_bin="${HOME}/go/bin/titus"
fi
case "$secrets_engine" in
noseyparker)
if ! command -v noseyparker >/dev/null 2>&1; then
_print_msg WARN "noseyparker not found; skipping secrets scan"
end_func "Results are saved in $domain/osint/github_company_secrets.json" "${FUNCNAME[0]}" WARN
return 0
fi
local -a np_scan_opts=()
[[ "${SECRETS_SCAN_GIT_HISTORY:-false}" == "true" ]] && np_scan_opts+=(--git-history)
# argv-safe parallel replacement for interlace -c.
local _np_fail_dir
_np_fail_dir=$(mktemp -d)
while IFS= read -r _np_target; do
[[ -z "$_np_target" ]] && continue
(
_np_safe=$(printf '%s' "$_np_target" | sha256sum | cut -d' ' -f1)
if ! noseyparker scan "${np_scan_opts[@]}" \
--datastore ".tmp/github/np_ds_${_np_safe}" \
".tmp/github_repos/${_np_target}" >/dev/null 2>>"$LOGFILE"; then
: >"$_np_fail_dir/$_np_safe"
elif ! noseyparker report --datastore ".tmp/github/np_ds_${_np_safe}" --format json \
>".tmp/github/nosey_${_np_safe}.json" 2>>"$LOGFILE"; then
: >"$_np_fail_dir/$_np_safe"
fi
) &
_throttle_jobs "${INTERLACE_THREADS:-10}"
done <.tmp/github_repos_folders.txt
wait
local _np_failures
_np_failures=$(find "$_np_fail_dir" -type f 2>/dev/null | wc -l | tr -d ' ')
rm -rf "$_np_fail_dir"
if [[ "$_np_failures" -gt 0 ]]; then
_print_error "noseyparker failed for $_np_failures repo(s); continuing"
fi
;;
titus|*)
[[ "$secrets_engine" != "titus" ]] && _print_msg WARN "Unknown SECRETS_ENGINE='${secrets_engine}', using titus"
if [[ -z "$titus_bin" ]]; then
_print_msg WARN "titus not found; skipping secrets scan"
end_func "Results are saved in $domain/osint/github_company_secrets.json" "${FUNCNAME[0]}" WARN
return 0
fi
local -a titus_opts=()
[[ "${SECRETS_SCAN_GIT_HISTORY:-false}" == "true" ]] && titus_opts+=(--git)
[[ "${SECRETS_VALIDATE:-false}" == "true" ]] && titus_opts+=(--validate)
# argv-safe parallel replacement for interlace -c.
local _titus_fail_dir
_titus_fail_dir=$(mktemp -d)
while IFS= read -r _titus_target; do
[[ -z "$_titus_target" ]] && continue
(
_titus_safe=$(printf '%s' "$_titus_target" | sha256sum | cut -d' ' -f1)
"$titus_bin" scan --format json "${titus_opts[@]}" \
".tmp/github_repos/${_titus_target}" \
>".tmp/github/titus_${_titus_safe}.json" 2>>"$LOGFILE" \
|| : >"$_titus_fail_dir/$_titus_safe"
) &
_throttle_jobs "${INTERLACE_THREADS:-10}"
done <.tmp/github_repos_folders.txt
wait
local _titus_failures
_titus_failures=$(find "$_titus_fail_dir" -type f 2>/dev/null | wc -l | tr -d ' ')
rm -rf "$_titus_fail_dir"
if [[ "$_titus_failures" -gt 0 ]]; then
_print_error "titus scan failed for $_titus_failures repo(s); continuing"
fi
;;
esac
# Keep trufflehog enrichment (all engines).
if [[ -s ".tmp/company_repos_url.txt" ]]; then
# argv-safe parallel replacement for interlace -c.
local _th_fail_dir
_th_fail_dir=$(mktemp -d)
while IFS= read -r _th_target; do
[[ -z "$_th_target" ]] && continue
(
_th_safe=$(printf '%s' "$_th_target" | sha256sum | cut -d' ' -f1)
if ! trufflehog git "$_th_target" -j 2>&1 | jq -c >".tmp/github/${_th_safe}" 2>>"$LOGFILE"; then
: >"$_th_fail_dir/$_th_safe"
fi
) &
_throttle_jobs "${INTERLACE_THREADS:-10}"
done <.tmp/company_repos_url.txt
wait
local _th_failures
_th_failures=$(find "$_th_fail_dir" -type f 2>/dev/null | wc -l | tr -d ' ')
rm -rf "$_th_fail_dir"
if [[ "$_th_failures" -gt 0 ]]; then
_print_error "trufflehog failed for $_th_failures repo(s); continuing"
fi
fi
if [[ -d ".tmp/github/" ]]; then
if ! cat .tmp/github/* 2>/dev/null | jq -c | jq -r >"osint/github_company_secrets.json" 2>>"$LOGFILE"; then
_print_error "Error combining results"
return 1
fi
else
log_note "No GitHub scan outputs available to merge for ${domain}" "${FUNCNAME[0]}" "${LINENO}"
end_func "Results are saved in $domain/osint/github_company_secrets.json" "${FUNCNAME[0]}"
return 0
fi
end_func "Results are saved in $domain/osint/github_company_secrets.json" "${FUNCNAME[0]}"
else
if [[ $GITHUB_REPOS == false ]] || [[ $OSINT == false ]]; then
skip_notification "disabled"
else
skip_notification "processed"
fi
fi
}
function github_leaks() {
ensure_dirs osint
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
&& [[ $GITHUB_LEAKS == true ]] && [[ $OSINT == true ]] \
&& ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
start_func "${FUNCNAME[0]}" "Running: GitHub-wide secret search with ghleaks"
if [[ -s $GITHUB_TOKENS ]]; then
GH_TOKEN=$(head -n 1 "$GITHUB_TOKENS")
register_secret "$GH_TOKEN"
local -a ghleaks_cmd=(
"${tools}/ghleaks/ghleaks"
-q "$domain"
--token "$GH_TOKEN"
--threads "${GHLEAKS_THREADS:-5}"
--report "${dir}/osint/github_leaks.json"
--no-banner
)
[[ $DEEP == true ]] && ghleaks_cmd+=(--exhaustive)
if ! run_command "${ghleaks_cmd[@]}" 2>>"$LOGFILE" >/dev/null; then
_print_error "ghleaks command failed"
fi
else
_print_error "Required file ${GITHUB_TOKENS} does not exist or is empty"
return 1
fi
end_func "Results are saved in $domain/osint/github_leaks.json" "${FUNCNAME[0]}"
else
if [[ $GITHUB_LEAKS == false ]] || [[ $OSINT == false ]]; then
skip_notification "disabled"
elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
else
skip_notification "processed"
fi
fi
}
function github_actions_audit() {
ensure_dirs osint .tmp
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
&& [[ "${GITHUB_ACTIONS_AUDIT:-false}" == "true" ]] && [[ $OSINT == true ]] \
&& ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
start_func "${FUNCNAME[0]}" "GitHub Actions security audit (gato)"
if [[ ! -s $GITHUB_TOKENS ]]; then
_print_error "Required file ${GITHUB_TOKENS} does not exist or is empty"
end_func "GitHub token missing, github_actions_audit skipped" "${FUNCNAME[0]}" SKIP
return 0
fi
local company_name
company_name=$(unfurl format %r <<<"$domain")
printf "%s\n" "$company_name" >".tmp/gato_orgs.txt"
local gato_cmd=""
if command -v gato >/dev/null 2>&1; then
gato_cmd="$(command -v gato)"
elif [[ -x "${tools}/gato/venv/bin/gato" ]]; then
gato_cmd="${tools}/gato/venv/bin/gato"
fi
if [[ -z "$gato_cmd" ]]; then
_print_msg WARN "${FUNCNAME[0]}: gato not found in PATH or ${tools}/gato/venv/bin/gato"
end_func "gato not available, github_actions_audit skipped" "${FUNCNAME[0]}" SKIP
return 0
fi
local -a gato_args=(e --enum_wf_artifacts --skip_sh_runner_enum -O ".tmp/gato_orgs.txt" -oJ "osint/github_actions_audit.json")
[[ "${GATO_INCLUDE_ALL_ARTIFACT_SECRETS:-false}" == "true" ]] && gato_args+=(--include_all_artifact_secrets)
if [[ -f "$GITHUB_TOKENS" ]]; then
export GH_TOKEN
GH_TOKEN="$(head -n 1 "$GITHUB_TOKENS")"
fi
if ! run_command "$gato_cmd" "${gato_args[@]}" 2>>"$LOGFILE" >/dev/null; then
_print_msg WARN "${FUNCNAME[0]}: gato execution failed"
fi
if [[ -s "osint/github_actions_audit.json" ]]; then
jq -r '.. | strings | select(test("(?i)(artifact|workflow|runner|secret)"))' "osint/github_actions_audit.json" 2>/dev/null \
| sort -u \
| anew -q "osint/github_actions_audit.txt"
fi
end_func "Results are saved in osint/github_actions_audit.[json|txt]" "${FUNCNAME[0]}"
else
if [[ "${GITHUB_ACTIONS_AUDIT:-false}" == "false" ]] || [[ $OSINT == false ]]; then
skip_notification "disabled"
elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
else
skip_notification "processed"
fi
fi
}
function metadata() {
ensure_dirs osint .tmp
# Check if the function should run
if { [[ ! -f "${called_fn_dir}/.${FUNCNAME[0]}" ]] || [[ ${DIFF} == true ]]; } && [[ ${METADATA} == true ]] && [[ ${OSINT} == true ]] && ! [[ ${domain} =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
start_func "${FUNCNAME[0]}" "Scanning metadata in public files"
if command -v exifray &>/dev/null; then
if ! run_command exifray -d "${domain}" --show-urls \
--timeout "${EXIFRAY_TIMEOUT:-30}" \
--max-retries "${EXIFRAY_RETRIES:-2}" \
-o "${dir}/osint/metadata.json" 2>>"${LOGFILE}"; then
log_note "metadata: exifray failed" "${FUNCNAME[0]}" "${LINENO}"
fi
# Extract human-readable summary from JSON output
if [[ -s "${dir}/osint/metadata.json" ]]; then
jq -r '.findings[] | "\(.category): \(.details.Value) (\(.file))"' \
"${dir}/osint/metadata.json" 2>/dev/null \
| sort -u \
| anew -q "osint/metadata_results.txt" || true
fi
# Complement with urlfinder-discovered URLs
if [[ ${METADATA_URLFINDER:-true} == true ]] && command -v urlfinder &>/dev/null; then
run_command urlfinder -d "${domain}" -all \
-o .tmp/metadata_urlfinder_raw.txt 2>>"${LOGFILE}" >/dev/null || true
if [[ -s ".tmp/metadata_urlfinder_raw.txt" ]]; then
# Filter for metadata-relevant extensions only
grep -aiE '\.(pdf|docx?|xlsx?|pptx?|odt|ods|odp|jpe?g|png|gif|tiff?|svg|webp|mp3)([?#].*)?$' \
.tmp/metadata_urlfinder_raw.txt \
| sort -u > .tmp/metadata_urlfinder_urls.txt || true
if [[ -s ".tmp/metadata_urlfinder_urls.txt" ]]; then
run_command exifray --urls .tmp/metadata_urlfinder_urls.txt \
--timeout "${EXIFRAY_TIMEOUT:-30}" \
--max-retries "${EXIFRAY_RETRIES:-2}" \
-o "${dir}/osint/metadata_urlfinder.json" 2>>"${LOGFILE}" || true
# Merge findings into main results
if [[ -s "${dir}/osint/metadata_urlfinder.json" ]]; then
jq -r '.findings[] | "\(.category): \(.details.Value) (\(.file))"' \
"${dir}/osint/metadata_urlfinder.json" 2>/dev/null \
| sort -u \
| anew -q "osint/metadata_results.txt" || true
fi
fi
fi
fi
else
_print_error "exifray is not installed. Skipping metadata extraction"
printf "exifray not installed - metadata extraction skipped\n" >>"${LOGFILE}"
fi
end_func "Results are saved in ${domain}/osint/metadata[.json|_results.txt]" "${FUNCNAME[0]}"
else
if [[ ${METADATA} == false ]] || [[ ${OSINT} == false ]]; then
skip_notification "disabled"
elif [[ ${domain} =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
else
skip_notification "processed"
fi
fi
}
function apileaks() {
ensure_dirs osint
# Check if the function should run
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
&& [[ $API_LEAKS == true ]] && [[ $OSINT == true ]] \
&& ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
start_func "${FUNCNAME[0]}" "Scanning for leaks in public API directories"
# Run porch-pirate (full dump); fallback to light mode if dump fails
if ! run_command porch-pirate -s "$domain" -l 25 --dump 2>>"$LOGFILE" >"${dir}/osint/postman_leaks.txt"; then
log_note "porch-pirate --dump failed; retrying without --dump" "${FUNCNAME[0]}" "${LINENO}"
if ! run_command porch-pirate -s "$domain" -l 25 2>>"$LOGFILE" >"${dir}/osint/postman_leaks.txt"; then
log_note "porch-pirate failed even without --dump" "${FUNCNAME[0]}" "${LINENO}"
fi
fi
if [[ ! -s "${dir}/osint/postman_leaks.txt" ]]; then
log_note "apileaks: porch-pirate returned empty output; continuing with swagger pipeline" "${FUNCNAME[0]}" "${LINENO}"
fi
# Run SwaggerSpy in a subshell to avoid CWD pollution
(
cd "${tools}/SwaggerSpy" || exit 1
local swag_cmd=("${tools}/SwaggerSpy/venv/bin/python3" "swaggerspy.py" "$domain")
local swag_timeout="${SWAGGERSPY_TIMEOUT:-5m}"
[[ -n ${TIMEOUT_CMD:-} ]] && swag_cmd=("$TIMEOUT_CMD" "$swag_timeout" "${swag_cmd[@]}")
local swagger_rc=0
{
run_command "${swag_cmd[@]}" 2>>"$LOGFILE" | grep -i "[*]\|URL" >"${dir}/osint/swagger_leaks.txt"
swagger_rc=${PIPESTATUS[0]:-0}
} || true
if ((swagger_rc != 0)); then
_print_msg WARN "SwaggerSpy failed (exit ${swagger_rc}), continuing without swagger results."
log_note "SwaggerSpy failed (exit ${swagger_rc}), continuing without swagger results." "${FUNCNAME[0]}" "${LINENO}"
fi
)
# Optional postleaksNg integration (Postman public library leaks)
if [[ ${API_LEAKS_POSTLEAKS:-true} == true ]]; then
local postleaks_bin
postleaks_bin="$(command -v postleaksNg 2>/dev/null || true)"
local postleaks_out="${dir}/osint/postman_leaks_postleaksng"
mkdir -p "$postleaks_out"
if [[ -n "$postleaks_bin" && -x "$postleaks_bin" ]]; then
local -a postleaks_cmd=(
"$postleaks_bin"
-k "$domain"
--output "$postleaks_out"
-t "${POSTLEAKS_THREADS:-10}"
)
[[ -n "${POSTLEAKS_INCLUDE:-}" ]] && postleaks_cmd+=(--include "${POSTLEAKS_INCLUDE}")
[[ -n "${POSTLEAKS_EXCLUDE:-}" ]] && postleaks_cmd+=(--exclude "${POSTLEAKS_EXCLUDE}")
if ! run_command "${postleaks_cmd[@]}" 2>>"$LOGFILE" >/dev/null; then
log_note "apileaks: postleaksNg failed" "${FUNCNAME[0]}" "${LINENO}"
fi
# Aggregate discovered URLs into the existing postman leaks artifact.
if compgen -G "${postleaks_out}/*.json" >/dev/null 2>&1; then
jq -r '.url // empty' "${postleaks_out}"/*.json 2>/dev/null \
| grep -aE '^https?://' \
| anew -q "${dir}/osint/postman_leaks.txt"
fi
else
log_note "apileaks: postleaksNg not found in PATH" "${FUNCNAME[0]}" "${LINENO}"
fi
fi
if [[ "${DRY_RUN:-false}" == "true" ]]; then
end_func "Dry-run: apileaks commands recorded" "${FUNCNAME[0]}" "SKIP"
return
fi
merge_scoped_urls_into_url_extract "${dir}/osint/postman_leaks.txt" "apileaks_postman"
merge_scoped_urls_into_url_extract "${dir}/osint/swagger_leaks.txt" "apileaks_swagger"
# Analyze leaks with trufflehog
if [[ -s "${dir}/osint/postman_leaks.txt" ]]; then
run_command trufflehog filesystem "${dir}/osint/postman_leaks.txt" -j 2>/dev/null | jq -c | anew -q "${dir}/osint/postman_leaks_trufflehog.json"
fi
if [[ -d "${dir}/osint/postman_leaks_postleaksng" ]]; then
run_command trufflehog filesystem "${dir}/osint/postman_leaks_postleaksng" -j 2>/dev/null | jq -c | anew -q "${dir}/osint/postman_leaks_trufflehog.json"
fi
if [[ -s "${dir}/osint/swagger_leaks.txt" ]]; then
run_command trufflehog filesystem "${dir}/osint/swagger_leaks.txt" -j 2>/dev/null | jq -c | anew -q "${dir}/osint/swagger_leaks_trufflehog.json"
fi
end_func "Results are saved in $domain/osint/[postman_leaks_trufflehog.json, swagger_leaks_trufflehog.json]" "${FUNCNAME[0]}"
else
if [[ $API_LEAKS == false ]] || [[ $OSINT == false ]]; then
skip_notification "disabled"
elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
else
skip_notification "processed"
fi
fi
}
function emails() {
ensure_dirs .tmp osint
# Check if the function should run
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
&& [[ $EMAILS == true ]] && [[ $OSINT == true ]] \
&& ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
start_func "${FUNCNAME[0]}" "Searching for emails/users/passwords leaks"
run_command env PYTHONWARNINGS=ignore "${tools}/EmailHarvester/venv/bin/python3" "${tools}/EmailHarvester/EmailHarvester.py" -d "$domain" -e all -l 20 2>>"$LOGFILE" | anew -q .tmp/EmailHarvester.txt || true
# Process emailfinder results
if [[ -s ".tmp/EmailHarvester.txt" ]]; then
grep "@" .tmp/EmailHarvester.txt | anew -q osint/emails.txt || true
fi
# Run LeakSearch in a subshell to avoid CWD pollution
(
cd "${tools}/LeakSearch" || exit 1
run_command "${tools}/LeakSearch/venv/bin/python3" LeakSearch.py -k "$domain" -o "${dir}/.tmp/passwords.txt" 1>>"$LOGFILE"
)
# Process passwords.txt
if [[ -s "${dir}/.tmp/passwords.txt" ]]; then
anew -q osint/passwords.txt <"${dir}/.tmp/passwords.txt"
fi
end_func "Results are saved in $domain/osint/emails.txt and passwords.txt" "${FUNCNAME[0]}"
else
if [[ $EMAILS == false ]] || [[ $OSINT == false ]]; then
skip_notification "disabled"
elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
else
skip_notification "processed"
fi
fi
}
function domain_info() {
ensure_dirs osint
# Check if the function should run
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
&& [[ $DOMAIN_INFO == true ]] && [[ $OSINT == true ]] \
&& ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
start_func "${FUNCNAME[0]}" "Searching domain info (whois, registrant name/email domains)"
# Run whois command and check for errors
run_command whois "$domain" >"osint/domain_info_general.txt"
: >osint/azure_tenant_domains.txt
local msftrecon_output msftrecon_rc
msftrecon_output=$(mktemp)
msftrecon_rc=0
if ! run_command "${tools}/msftrecon/venv/bin/python3" "${tools}/msftrecon/msftrecon/msftrecon.py" -d "${domain}" \
>"$msftrecon_output" 2>>"$LOGFILE"; then
msftrecon_rc=$?
fi
if [[ $msftrecon_rc -eq 0 ]]; then
cat "$msftrecon_output" >osint/azure_tenant_domains.txt
else
log_note "domain_info: msftrecon failed (rc=${msftrecon_rc}), continuing" "${FUNCNAME[0]}" "${LINENO}"
_print_msg WARN "domain_info: msftrecon failed, continuing"
: >osint/azure_tenant_domains.txt
fi
rm -f "$msftrecon_output"
company_name=$(unfurl format %r <<<"$domain")
run_command "${tools}/Scopify/venv/bin/python3" "${tools}/Scopify/scopify.py" -c ${company_name} >osint/scopify.txt
end_func "Results are saved in ${domain}/osint/domain_info_[general/azure_tenant_domains].txt" "${FUNCNAME[0]}"
else
if [[ $DOMAIN_INFO == false ]] || [[ $OSINT == false ]]; then
skip_notification "disabled"
elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
else
skip_notification "processed"
fi
fi
}
function third_party_misconfigs() {
ensure_dirs osint
# Check if the function should run
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
&& [[ $THIRD_PARTIES == true ]] && [[ $OSINT == true ]] \
&& ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
start_func "${FUNCNAME[0]}" "Searching for third parties misconfigurations"
# Extract company name from domain
company_name=$(unfurl format %r <<<"$domain")
# Run misconfig-mapper in a subshell to avoid CWD pollution
(
cd "${tools}/misconfig-mapper" || exit 1
run_command misconfig-mapper -update-templates 1>>"$LOGFILE"
run_command misconfig-mapper -target "$domain" -as-domain true -permutations false -skip-ssl \
-service "*" -verbose 0 | anew -q "${dir}/osint/3rdparts_misconfigurations.txt"
run_command misconfig-mapper -target "$company_name" -skip-ssl -verbose 0 -service "*" \
| anew -q "${dir}/osint/3rdparts_misconfigurations.txt"
)
end_func "Results are saved in $domain/osint/3rdparts_misconfigurations.txt" "${FUNCNAME[0]}"
else
if [[ $THIRD_PARTIES == false ]] || [[ $OSINT == false ]]; then
skip_notification "disabled"
elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
else
skip_notification "processed"
fi
fi
}
function lunar_exposure() {
ensure_dirs osint
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
&& [[ $LUNAR_EXPOSURE == true ]] && [[ $OSINT == true ]] \
&& ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
start_func "${FUNCNAME[0]}" "Domain exposure in infostealer logs and breaches (Lunar)"
local _out="${dir}/osint/lunar_exposure.json"
local _tmp _code
if ! _tmp=$(mktemp -t reconftw_lunar.XXXXXX); then
end_func "Could not create a temporary file for the Lunar query" "${FUNCNAME[0]}"
return
fi
# Staged, then published by rename: a truncated or error body must not
# replace a good result from a previous run.
_code=$(run_command curl -s --max-time "${LUNAR_TIMEOUT:-30}" \
-H 'Accept: application/json' \
-w '%{http_code}' -o "$_tmp" \
"https://api.lunarcyber.com/domain-exposure?domain=${domain}" 2>>"$LOGFILE") || _code="000"
if [[ "$_code" != "200" ]]; then
rm -f "$_tmp" 2>/dev/null
end_func "Lunar API returned HTTP ${_code:-000}; no exposure data recorded" "${FUNCNAME[0]}"
return
fi
# A 200 carrying a non-JSON body is an outage or interstitial page, not
# data. Record nothing rather than a file downstream has to guess about.
if ! jq -e . "$_tmp" >/dev/null 2>&1; then
rm -f "$_tmp" 2>/dev/null
end_func "Lunar API returned 200 with a non-JSON body; nothing recorded" "${FUNCNAME[0]}"
return
fi
mv "$_tmp" "$_out"
end_func "Results are saved in $domain/osint/lunar_exposure.json" "${FUNCNAME[0]}"
else
if [[ $LUNAR_EXPOSURE == false ]] || [[ $OSINT == false ]]; then
skip_notification "disabled"
elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
else
skip_notification "processed"
fi
fi
}
function spoof() {
ensure_dirs osint
# Check if the function should run
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
&& [[ $SPOOF == true ]] && [[ $OSINT == true ]] \
&& ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
start_func "${FUNCNAME[0]}" "Searching for spoofable domains"
# Run Spoofy in a subshell to avoid CWD pollution
(
cd "${tools}/Spoofy" || exit 1
run_command "${tools}/Spoofy/venv/bin/python3" spoofy.py -d "$domain" >"${dir}/osint/spoof.txt"
)
end_func "Results are saved in $domain/osint/spoof.txt" "${FUNCNAME[0]}"
else
if [[ $SPOOF == false ]] || [[ $OSINT == false ]]; then
skip_notification "disabled"
elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
else
skip_notification "processed"
fi
fi
}
function mail_hygiene() {
ensure_dirs osint
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
&& [[ $MAIL_HYGIENE == true ]] && [[ $OSINT == true ]] \
&& ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
start_func "${FUNCNAME[0]}" "Mail hygiene (SPF/DMARC)"
{
printf "Domain: %s\n" "$domain"
printf "\nTXT records:\n"
run_command dig +short TXT "$domain" | sed 's/^/ /'
printf "\nDMARC record:\n"
run_command dig +short TXT "_dmarc.$domain" | sed 's/^/ /'
} >"osint/mail_hygiene.txt" 2>>"$LOGFILE"
end_func "Results are saved in $domain/osint/mail_hygiene.txt" "${FUNCNAME[0]}"
else
if [[ $MAIL_HYGIENE == false ]] || [[ $OSINT == false ]]; then
skip_notification "disabled"
elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
else
skip_notification "processed"
fi
fi
}
function cloud_enum_scan() {
ensure_dirs osint
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
&& [[ $CLOUD_ENUM == true ]] && [[ $OSINT == true ]] \
&& ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
local -a cloud_enum_cmd
start_func "${FUNCNAME[0]}" "Cloud storage enumeration"
if ! build_cloud_enum_command cloud_enum_cmd "${FUNCNAME[0]}" "$domain"; then
return 0
fi
run_command "${cloud_enum_cmd[@]}" 2>>"$LOGFILE" | anew -q osint/cloud_enum.txt
end_func "Results are saved in $domain/osint/cloud_enum.txt" "${FUNCNAME[0]}"
else
if [[ $CLOUD_ENUM == false ]] || [[ $OSINT == false ]]; then
skip_notification "disabled"
elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
else
skip_notification "processed"
fi
fi
}
function ip_info() {
ensure_dirs osint
# Check if the function should run
if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
&& [[ $IP_INFO == true ]] && [[ $OSINT == true ]] \
&& [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
start_func "${FUNCNAME[0]}" "Searching IP info"
if [[ -n $WHOISXML_API ]]; then
# Reverse IP lookup
run_command curl -s "https://reverse-ip.whoisxmlapi.com/api/v1?apiKey=${WHOISXML_API}&ip=${domain}" \
| jq -r '.result[].name' 2>>"$LOGFILE" \
| sed -e "s/$/ ${domain}/" \
| anew -q "osint/ip_${domain}_relations.txt"
# WHOIS lookup
run_command curl -s "https://www.whoisxmlapi.com/whoisserver/WhoisService?apiKey=${WHOISXML_API}&domainName=${domain}&outputFormat=json&da=2®istryRawText=1®istrarRawText=1&ignoreRawTexts=1" \
| jq 2>>"$LOGFILE" \
| anew -q "osint/ip_${domain}_whois.txt"
# IP Geolocation
run_command curl -s "https://ip-geolocation.whoisxmlapi.com/api/v1?apiKey=${WHOISXML_API}&ipAddress=${domain}" \
| jq -r '.ip,.location' 2>>"$LOGFILE" \
| anew -q "osint/ip_${domain}_location.txt"
end_func "Results are saved in ${domain}/osint/ip_[domain_relations|whois|location].txt" "${FUNCNAME[0]}"
else
_print_msg WARN "WHOISXML_API variable is not defined. Skipping function."
end_func "WHOISXML_API not configured, ip_info skipped." "${FUNCNAME[0]}"
fi
else
if [[ $IP_INFO == false ]] || [[ $OSINT == false ]]; then
skip_notification "disabled"
elif ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
return
else
skip_notification "processed"
fi
fi
}