Skip to content

reconFTW is a tool designed to perform automated recon on a target domain by running the best set of tools to perform scanning and finding out vulnerabilities

35.9 KB
Raw Blame
#!/bin/bash
# shellcheck disable=SC2154  # Variables defined in reconftw.cfg
# reconFTW - OSINT module
# Contains: google_dorks, github_dorks, github_repos, github_leaks, metadata,
#           apileaks, emails, domain_info, third_party_misconfigs, spoof,
#           mail_hygiene, cloud_enum_scan, ip_info
# This file is sourced by reconftw.sh - do not execute directly
[[ -z "${SCRIPTPATH:-}" ]] && {
    echo "Error: This module must be sourced by reconftw.sh" >&2
    exit 1
}

#####################################################################cc##########################################
################################################### OSINT #####################################################
###############################################################################################################

function google_dorks() {
    ensure_dirs osint

    if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $GOOGLE_DORKS == true ]] && [[ $OSINT == true ]]; then
        start_func "${FUNCNAME[0]}" "Running: Google Dorks in process"

        if ! run_command "${tools}/dorks_hunter/venv/bin/python3" "${tools}/dorks_hunter/dorks_hunter.py" -d "$domain" -o "osint/dorks.txt" 2>>"$LOGFILE" >/dev/null; then
            _print_error "dorks_hunter command failed"
        fi
        end_func "Results are saved in $domain/osint/dorks.txt" "${FUNCNAME[0]}"
    else
        if [[ $GOOGLE_DORKS == false ]] || [[ $OSINT == false ]]; then
            skip_notification "disabled"
        else
            skip_notification "processed"
        fi
    fi
}

function github_dorks() {
    ensure_dirs osint

    if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $GITHUB_DORKS == true ]] && [[ $OSINT == true ]]; then
        start_func "${FUNCNAME[0]}" "Running: Github Dorks in process"

        if [[ -s $GITHUB_TOKENS ]]; then
            if [[ $DEEP == true ]]; then
                if ! run_command gitdorks_go -gd "${tools}/gitdorks_go/Dorks/medium_dorks.txt" -nws 20 -target "$domain" -tf "$GITHUB_TOKENS" -ew 3 | anew -q osint/gitdorks.txt; then
                    _print_error "gitdorks_go command failed"
                    return 1
                fi
            else
                if ! run_command gitdorks_go -gd "${tools}/gitdorks_go/Dorks/smalldorks.txt" -nws 20 -target "$domain" -tf "$GITHUB_TOKENS" -ew 3 | anew -q osint/gitdorks.txt; then
                    _print_error "gitdorks_go command failed"
                    return 1
                fi
            fi
        else
            _print_error "Required file ${GITHUB_TOKENS} does not exist or is empty"
            return 1
        fi
        end_func "Results are saved in $domain/osint/gitdorks.txt" "${FUNCNAME[0]}"
    else
        if [[ $GITHUB_DORKS == false ]] || [[ $OSINT == false ]]; then
            skip_notification "disabled"
        else
            skip_notification "processed"
        fi
    fi
}

function github_repos() {
    ensure_dirs osint

    if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } && [[ $GITHUB_REPOS == true ]] && [[ $OSINT == true ]]; then
        start_func "${FUNCNAME[0]}" "Github Repos analysis in process"

        if [[ ! -s $GITHUB_TOKENS ]]; then
            _print_error "Required file ${GITHUB_TOKENS} does not exist or is empty"
            return 1
        fi

        GH_TOKEN=$(head -n 1 "$GITHUB_TOKENS")
        register_secret "$GH_TOKEN"
        echo "$domain" | unfurl format %r >.tmp/company_name.txt

        # Use temp file for token to avoid exposing it in process list
        local _token_file
        _token_file=$(mktemp)
        printf '%s' "$GH_TOKEN" > "$_token_file"
        if ! run_command enumerepo -token-file "$_token_file" -usernames .tmp/company_name.txt -o .tmp/company_repos.txt 2>>"$LOGFILE" >/dev/null; then
            _print_error "enumerepo command failed"
        fi
        rm -f "$_token_file"

        if [[ -s ".tmp/company_repos.txt" ]]; then
            if ! jq -r '.[].repos[]|.url' <.tmp/company_repos.txt >.tmp/company_repos_url.txt 2>>"$LOGFILE"; then
                _print_error "jq command failed"
            fi
        fi

        ensure_dirs .tmp/github_repos .tmp/github

        if [[ -s ".tmp/company_repos_url.txt" ]]; then
            # argv-safe parallel replacement for interlace -c. Each clone runs in a
            # background subshell; failures are recorded via a marker file so the
            # caller can log a summary after wait.
            local _clone_fail_dir
            _clone_fail_dir=$(mktemp -d)
            while IFS= read -r _repo_target; do
                [[ -z "$_repo_target" ]] && continue
                (
                    _repo_safe=$(printf '%s' "$_repo_target" | sha256sum | cut -d' ' -f1)
                    if ! git clone "$_repo_target" ".tmp/github_repos/${_repo_safe}" 2>>"$LOGFILE" >/dev/null; then
                        : >"$_clone_fail_dir/$_repo_safe"
                    fi
                ) &
                _throttle_jobs "${INTERLACE_THREADS:-10}"
            done <.tmp/company_repos_url.txt
            wait
            local _clone_failures
            _clone_failures=$(find "$_clone_fail_dir" -type f 2>/dev/null | wc -l | tr -d ' ')
            rm -rf "$_clone_fail_dir"
            if [[ "$_clone_failures" -gt 0 ]]; then
                _print_error "git clone failed for $_clone_failures repo(s); continuing with successful clones"
            fi
        else
            log_note "No GitHub repos found for ${domain}; continuing" "${FUNCNAME[0]}" "${LINENO}"
            end_func "Results are saved in $domain/osint/github_company_secrets.json" "${FUNCNAME[0]}"
            return 0
        fi

        if [[ -d ".tmp/github_repos/" ]]; then
            ls .tmp/github_repos >.tmp/github_repos_folders.txt
        else
            log_note "GitHub clone directory missing for ${domain}; no repo artifacts to process" "${FUNCNAME[0]}" "${LINENO}"
            end_func "Results are saved in $domain/osint/github_company_secrets.json" "${FUNCNAME[0]}"
            return 0
        fi
        if [[ ! -s ".tmp/github_repos_folders.txt" ]]; then
            log_note "No cloned GitHub repos to scan for ${domain}" "${FUNCNAME[0]}" "${LINENO}"
            end_func "Results are saved in $domain/osint/github_company_secrets.json" "${FUNCNAME[0]}"
            return 0
        fi

        local secrets_engine="${SECRETS_ENGINE:-titus}"
        local titus_bin=""
        if command -v titus >/dev/null 2>&1; then
            titus_bin="$(command -v titus)"
        elif [[ -x "${tools}/titus/dist/titus" ]]; then
            titus_bin="${tools}/titus/dist/titus"
        elif [[ -x "${HOME}/go/bin/titus" ]]; then
            titus_bin="${HOME}/go/bin/titus"
        fi

        case "$secrets_engine" in
            noseyparker)
                if ! command -v noseyparker >/dev/null 2>&1; then
                    _print_msg WARN "noseyparker not found; skipping secrets scan"
                    end_func "Results are saved in $domain/osint/github_company_secrets.json" "${FUNCNAME[0]}" WARN
                    return 0
                fi
                local -a np_scan_opts=()
                [[ "${SECRETS_SCAN_GIT_HISTORY:-false}" == "true" ]] && np_scan_opts+=(--git-history)
                # argv-safe parallel replacement for interlace -c.
                local _np_fail_dir
                _np_fail_dir=$(mktemp -d)
                while IFS= read -r _np_target; do
                    [[ -z "$_np_target" ]] && continue
                    (
                        _np_safe=$(printf '%s' "$_np_target" | sha256sum | cut -d' ' -f1)
                        if ! noseyparker scan "${np_scan_opts[@]}" \
                            --datastore ".tmp/github/np_ds_${_np_safe}" \
                            ".tmp/github_repos/${_np_target}" >/dev/null 2>>"$LOGFILE"; then
                            : >"$_np_fail_dir/$_np_safe"
                        elif ! noseyparker report --datastore ".tmp/github/np_ds_${_np_safe}" --format json \
                                >".tmp/github/nosey_${_np_safe}.json" 2>>"$LOGFILE"; then
                            : >"$_np_fail_dir/$_np_safe"
                        fi
                    ) &
                    _throttle_jobs "${INTERLACE_THREADS:-10}"
                done <.tmp/github_repos_folders.txt
                wait
                local _np_failures
                _np_failures=$(find "$_np_fail_dir" -type f 2>/dev/null | wc -l | tr -d ' ')
                rm -rf "$_np_fail_dir"
                if [[ "$_np_failures" -gt 0 ]]; then
                    _print_error "noseyparker failed for $_np_failures repo(s); continuing"
                fi
                ;;
            titus|*)
                [[ "$secrets_engine" != "titus" ]] && _print_msg WARN "Unknown SECRETS_ENGINE='${secrets_engine}', using titus"
                if [[ -z "$titus_bin" ]]; then
                    _print_msg WARN "titus not found; skipping secrets scan"
                    end_func "Results are saved in $domain/osint/github_company_secrets.json" "${FUNCNAME[0]}" WARN
                    return 0
                fi
                local -a titus_opts=()
                [[ "${SECRETS_SCAN_GIT_HISTORY:-false}" == "true" ]] && titus_opts+=(--git)
                [[ "${SECRETS_VALIDATE:-false}" == "true" ]] && titus_opts+=(--validate)
                # argv-safe parallel replacement for interlace -c.
                local _titus_fail_dir
                _titus_fail_dir=$(mktemp -d)
                while IFS= read -r _titus_target; do
                    [[ -z "$_titus_target" ]] && continue
                    (
                        _titus_safe=$(printf '%s' "$_titus_target" | sha256sum | cut -d' ' -f1)
                        "$titus_bin" scan --format json "${titus_opts[@]}" \
                            ".tmp/github_repos/${_titus_target}" \
                            >".tmp/github/titus_${_titus_safe}.json" 2>>"$LOGFILE" \
                            || : >"$_titus_fail_dir/$_titus_safe"
                    ) &
                    _throttle_jobs "${INTERLACE_THREADS:-10}"
                done <.tmp/github_repos_folders.txt
                wait
                local _titus_failures
                _titus_failures=$(find "$_titus_fail_dir" -type f 2>/dev/null | wc -l | tr -d ' ')
                rm -rf "$_titus_fail_dir"
                if [[ "$_titus_failures" -gt 0 ]]; then
                    _print_error "titus scan failed for $_titus_failures repo(s); continuing"
                fi
                ;;
        esac

        # Keep trufflehog enrichment (all engines).
        if [[ -s ".tmp/company_repos_url.txt" ]]; then
            # argv-safe parallel replacement for interlace -c.
            local _th_fail_dir
            _th_fail_dir=$(mktemp -d)
            while IFS= read -r _th_target; do
                [[ -z "$_th_target" ]] && continue
                (
                    _th_safe=$(printf '%s' "$_th_target" | sha256sum | cut -d' ' -f1)
                    if ! trufflehog git "$_th_target" -j 2>&1 | jq -c >".tmp/github/${_th_safe}" 2>>"$LOGFILE"; then
                        : >"$_th_fail_dir/$_th_safe"
                    fi
                ) &
                _throttle_jobs "${INTERLACE_THREADS:-10}"
            done <.tmp/company_repos_url.txt
            wait
            local _th_failures
            _th_failures=$(find "$_th_fail_dir" -type f 2>/dev/null | wc -l | tr -d ' ')
            rm -rf "$_th_fail_dir"
            if [[ "$_th_failures" -gt 0 ]]; then
                _print_error "trufflehog failed for $_th_failures repo(s); continuing"
            fi
        fi

        if [[ -d ".tmp/github/" ]]; then
            if ! cat .tmp/github/* 2>/dev/null | jq -c | jq -r >"osint/github_company_secrets.json" 2>>"$LOGFILE"; then
                _print_error "Error combining results"
                return 1
            fi
        else
            log_note "No GitHub scan outputs available to merge for ${domain}" "${FUNCNAME[0]}" "${LINENO}"
            end_func "Results are saved in $domain/osint/github_company_secrets.json" "${FUNCNAME[0]}"
            return 0
        fi

        end_func "Results are saved in $domain/osint/github_company_secrets.json" "${FUNCNAME[0]}"
    else
        if [[ $GITHUB_REPOS == false ]] || [[ $OSINT == false ]]; then
            skip_notification "disabled"
        else
            skip_notification "processed"
        fi
    fi
}

function github_leaks() {
    ensure_dirs osint

    if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
        && [[ $GITHUB_LEAKS == true ]] && [[ $OSINT == true ]] \
        && ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then

        start_func "${FUNCNAME[0]}" "Running: GitHub-wide secret search with ghleaks"

        if [[ -s $GITHUB_TOKENS ]]; then
            GH_TOKEN=$(head -n 1 "$GITHUB_TOKENS")
        register_secret "$GH_TOKEN"
            local -a ghleaks_cmd=(
                "${tools}/ghleaks/ghleaks"
                -q "$domain"
                --token "$GH_TOKEN"
                --threads "${GHLEAKS_THREADS:-5}"
                --report "${dir}/osint/github_leaks.json"
                --no-banner
            )
            [[ $DEEP == true ]] && ghleaks_cmd+=(--exhaustive)

            if ! run_command "${ghleaks_cmd[@]}" 2>>"$LOGFILE" >/dev/null; then
                _print_error "ghleaks command failed"
            fi
        else
            _print_error "Required file ${GITHUB_TOKENS} does not exist or is empty"
            return 1
        fi
        end_func "Results are saved in $domain/osint/github_leaks.json" "${FUNCNAME[0]}"
    else
        if [[ $GITHUB_LEAKS == false ]] || [[ $OSINT == false ]]; then
            skip_notification "disabled"
        elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
            return
        else
            skip_notification "processed"
        fi
    fi
}

function github_actions_audit() {
    ensure_dirs osint .tmp

    if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
        && [[ "${GITHUB_ACTIONS_AUDIT:-false}" == "true" ]] && [[ $OSINT == true ]] \
        && ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then

        start_func "${FUNCNAME[0]}" "GitHub Actions security audit (gato)"

        if [[ ! -s $GITHUB_TOKENS ]]; then
            _print_error "Required file ${GITHUB_TOKENS} does not exist or is empty"
            end_func "GitHub token missing, github_actions_audit skipped" "${FUNCNAME[0]}" SKIP
            return 0
        fi

        local company_name
        company_name=$(unfurl format %r <<<"$domain")
        printf "%s\n" "$company_name" >".tmp/gato_orgs.txt"

        local gato_cmd=""
        if command -v gato >/dev/null 2>&1; then
            gato_cmd="$(command -v gato)"
        elif [[ -x "${tools}/gato/venv/bin/gato" ]]; then
            gato_cmd="${tools}/gato/venv/bin/gato"
        fi
        if [[ -z "$gato_cmd" ]]; then
            _print_msg WARN "${FUNCNAME[0]}: gato not found in PATH or ${tools}/gato/venv/bin/gato"
            end_func "gato not available, github_actions_audit skipped" "${FUNCNAME[0]}" SKIP
            return 0
        fi

        local -a gato_args=(e --enum_wf_artifacts --skip_sh_runner_enum -O ".tmp/gato_orgs.txt" -oJ "osint/github_actions_audit.json")
        [[ "${GATO_INCLUDE_ALL_ARTIFACT_SECRETS:-false}" == "true" ]] && gato_args+=(--include_all_artifact_secrets)

        if [[ -f "$GITHUB_TOKENS" ]]; then
            export GH_TOKEN
            GH_TOKEN="$(head -n 1 "$GITHUB_TOKENS")"
        fi

        if ! run_command "$gato_cmd" "${gato_args[@]}" 2>>"$LOGFILE" >/dev/null; then
            _print_msg WARN "${FUNCNAME[0]}: gato execution failed"
        fi

        if [[ -s "osint/github_actions_audit.json" ]]; then
            jq -r '.. | strings | select(test("(?i)(artifact|workflow|runner|secret)"))' "osint/github_actions_audit.json" 2>/dev/null \
                | sort -u \
                | anew -q "osint/github_actions_audit.txt"
        fi

        end_func "Results are saved in osint/github_actions_audit.[json|txt]" "${FUNCNAME[0]}"
    else
        if [[ "${GITHUB_ACTIONS_AUDIT:-false}" == "false" ]] || [[ $OSINT == false ]]; then
            skip_notification "disabled"
        elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
            return
        else
            skip_notification "processed"
        fi
    fi
}

function metadata() {
    ensure_dirs osint .tmp

    # Check if the function should run
    if { [[ ! -f "${called_fn_dir}/.${FUNCNAME[0]}" ]] || [[ ${DIFF} == true ]]; } && [[ ${METADATA} == true ]] && [[ ${OSINT} == true ]] && ! [[ ${domain} =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
        start_func "${FUNCNAME[0]}" "Scanning metadata in public files"

        if command -v exifray &>/dev/null; then
            if ! run_command exifray -d "${domain}" --show-urls \
                --timeout "${EXIFRAY_TIMEOUT:-30}" \
                --max-retries "${EXIFRAY_RETRIES:-2}" \
                -o "${dir}/osint/metadata.json" 2>>"${LOGFILE}"; then
                log_note "metadata: exifray failed" "${FUNCNAME[0]}" "${LINENO}"
            fi

            # Extract human-readable summary from JSON output
            if [[ -s "${dir}/osint/metadata.json" ]]; then
                jq -r '.findings[] | "\(.category): \(.details.Value) (\(.file))"' \
                    "${dir}/osint/metadata.json" 2>/dev/null \
                    | sort -u \
                    | anew -q "osint/metadata_results.txt" || true
            fi

            # Complement with urlfinder-discovered URLs
            if [[ ${METADATA_URLFINDER:-true} == true ]] && command -v urlfinder &>/dev/null; then
                run_command urlfinder -d "${domain}" -all \
                    -o .tmp/metadata_urlfinder_raw.txt 2>>"${LOGFILE}" >/dev/null || true

                if [[ -s ".tmp/metadata_urlfinder_raw.txt" ]]; then
                    # Filter for metadata-relevant extensions only
                    grep -aiE '\.(pdf|docx?|xlsx?|pptx?|odt|ods|odp|jpe?g|png|gif|tiff?|svg|webp|mp3)([?#].*)?$' \
                        .tmp/metadata_urlfinder_raw.txt \
                        | sort -u > .tmp/metadata_urlfinder_urls.txt || true

                    if [[ -s ".tmp/metadata_urlfinder_urls.txt" ]]; then
                        run_command exifray --urls .tmp/metadata_urlfinder_urls.txt \
                            --timeout "${EXIFRAY_TIMEOUT:-30}" \
                            --max-retries "${EXIFRAY_RETRIES:-2}" \
                            -o "${dir}/osint/metadata_urlfinder.json" 2>>"${LOGFILE}" || true

                        # Merge findings into main results
                        if [[ -s "${dir}/osint/metadata_urlfinder.json" ]]; then
                            jq -r '.findings[] | "\(.category): \(.details.Value) (\(.file))"' \
                                "${dir}/osint/metadata_urlfinder.json" 2>/dev/null \
                                | sort -u \
                                | anew -q "osint/metadata_results.txt" || true
                        fi
                    fi
                fi
            fi
        else
            _print_error "exifray is not installed. Skipping metadata extraction"
            printf "exifray not installed - metadata extraction skipped\n" >>"${LOGFILE}"
        fi

        end_func "Results are saved in ${domain}/osint/metadata[.json|_results.txt]" "${FUNCNAME[0]}"
    else
        if [[ ${METADATA} == false ]] || [[ ${OSINT} == false ]]; then
            skip_notification "disabled"
        elif [[ ${domain} =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
            return
        else
            skip_notification "processed"
        fi
    fi
}

function apileaks() {
    ensure_dirs osint

    # Check if the function should run
    if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
        && [[ $API_LEAKS == true ]] && [[ $OSINT == true ]] \
        && ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then

        start_func "${FUNCNAME[0]}" "Scanning for leaks in public API directories"

        # Run porch-pirate (full dump); fallback to light mode if dump fails
        if ! run_command porch-pirate -s "$domain" -l 25 --dump 2>>"$LOGFILE" >"${dir}/osint/postman_leaks.txt"; then
            log_note "porch-pirate --dump failed; retrying without --dump" "${FUNCNAME[0]}" "${LINENO}"
            if ! run_command porch-pirate -s "$domain" -l 25 2>>"$LOGFILE" >"${dir}/osint/postman_leaks.txt"; then
                log_note "porch-pirate failed even without --dump" "${FUNCNAME[0]}" "${LINENO}"
            fi
        fi
        if [[ ! -s "${dir}/osint/postman_leaks.txt" ]]; then
            log_note "apileaks: porch-pirate returned empty output; continuing with swagger pipeline" "${FUNCNAME[0]}" "${LINENO}"
        fi

        # Run SwaggerSpy in a subshell to avoid CWD pollution
        (
            cd "${tools}/SwaggerSpy" || exit 1
            local swag_cmd=("${tools}/SwaggerSpy/venv/bin/python3" "swaggerspy.py" "$domain")
            local swag_timeout="${SWAGGERSPY_TIMEOUT:-5m}"
            [[ -n ${TIMEOUT_CMD:-} ]] && swag_cmd=("$TIMEOUT_CMD" "$swag_timeout" "${swag_cmd[@]}")
            local swagger_rc=0
            {
                run_command "${swag_cmd[@]}" 2>>"$LOGFILE" | grep -i "[*]\|URL" >"${dir}/osint/swagger_leaks.txt"
                swagger_rc=${PIPESTATUS[0]:-0}
            } || true
            if ((swagger_rc != 0)); then
                _print_msg WARN "SwaggerSpy failed (exit ${swagger_rc}), continuing without swagger results."
                log_note "SwaggerSpy failed (exit ${swagger_rc}), continuing without swagger results." "${FUNCNAME[0]}" "${LINENO}"
            fi
        )

        # Optional postleaksNg integration (Postman public library leaks)
        if [[ ${API_LEAKS_POSTLEAKS:-true} == true ]]; then
            local postleaks_bin
            postleaks_bin="$(command -v postleaksNg 2>/dev/null || true)"
            local postleaks_out="${dir}/osint/postman_leaks_postleaksng"
            mkdir -p "$postleaks_out"
            if [[ -n "$postleaks_bin" && -x "$postleaks_bin" ]]; then
                local -a postleaks_cmd=(
                    "$postleaks_bin"
                    -k "$domain"
                    --output "$postleaks_out"
                    -t "${POSTLEAKS_THREADS:-10}"
                )
                [[ -n "${POSTLEAKS_INCLUDE:-}" ]] && postleaks_cmd+=(--include "${POSTLEAKS_INCLUDE}")
                [[ -n "${POSTLEAKS_EXCLUDE:-}" ]] && postleaks_cmd+=(--exclude "${POSTLEAKS_EXCLUDE}")

                if ! run_command "${postleaks_cmd[@]}" 2>>"$LOGFILE" >/dev/null; then
                    log_note "apileaks: postleaksNg failed" "${FUNCNAME[0]}" "${LINENO}"
                fi

                # Aggregate discovered URLs into the existing postman leaks artifact.
                if compgen -G "${postleaks_out}/*.json" >/dev/null 2>&1; then
                    jq -r '.url // empty' "${postleaks_out}"/*.json 2>/dev/null \
                        | grep -aE '^https?://' \
                        | anew -q "${dir}/osint/postman_leaks.txt"
                fi
            else
                log_note "apileaks: postleaksNg not found in PATH" "${FUNCNAME[0]}" "${LINENO}"
            fi
        fi

        if [[ "${DRY_RUN:-false}" == "true" ]]; then
            end_func "Dry-run: apileaks commands recorded" "${FUNCNAME[0]}" "SKIP"
            return
        fi

        merge_scoped_urls_into_url_extract "${dir}/osint/postman_leaks.txt" "apileaks_postman"
        merge_scoped_urls_into_url_extract "${dir}/osint/swagger_leaks.txt" "apileaks_swagger"

        # Analyze leaks with trufflehog
        if [[ -s "${dir}/osint/postman_leaks.txt" ]]; then
            run_command trufflehog filesystem "${dir}/osint/postman_leaks.txt" -j 2>/dev/null | jq -c | anew -q "${dir}/osint/postman_leaks_trufflehog.json"
        fi
        if [[ -d "${dir}/osint/postman_leaks_postleaksng" ]]; then
            run_command trufflehog filesystem "${dir}/osint/postman_leaks_postleaksng" -j 2>/dev/null | jq -c | anew -q "${dir}/osint/postman_leaks_trufflehog.json"
        fi

        if [[ -s "${dir}/osint/swagger_leaks.txt" ]]; then
            run_command trufflehog filesystem "${dir}/osint/swagger_leaks.txt" -j 2>/dev/null | jq -c | anew -q "${dir}/osint/swagger_leaks_trufflehog.json"
        fi
        end_func "Results are saved in $domain/osint/[postman_leaks_trufflehog.json, swagger_leaks_trufflehog.json]" "${FUNCNAME[0]}"
    else
        if [[ $API_LEAKS == false ]] || [[ $OSINT == false ]]; then
            skip_notification "disabled"
        elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
            return
        else
            skip_notification "processed"
        fi
    fi
}

function emails() {
    ensure_dirs .tmp osint

    # Check if the function should run
    if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
        && [[ $EMAILS == true ]] && [[ $OSINT == true ]] \
        && ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then

        start_func "${FUNCNAME[0]}" "Searching for emails/users/passwords leaks"

        run_command env PYTHONWARNINGS=ignore "${tools}/EmailHarvester/venv/bin/python3" "${tools}/EmailHarvester/EmailHarvester.py" -d "$domain" -e all -l 20 2>>"$LOGFILE" | anew -q .tmp/EmailHarvester.txt || true

        # Process emailfinder results
        if [[ -s ".tmp/EmailHarvester.txt" ]]; then
            grep "@" .tmp/EmailHarvester.txt | anew -q osint/emails.txt || true
        fi

        # Run LeakSearch in a subshell to avoid CWD pollution
        (
            cd "${tools}/LeakSearch" || exit 1
            run_command "${tools}/LeakSearch/venv/bin/python3" LeakSearch.py -k "$domain" -o "${dir}/.tmp/passwords.txt" 1>>"$LOGFILE"
        )

        # Process passwords.txt
        if [[ -s "${dir}/.tmp/passwords.txt" ]]; then
            anew -q osint/passwords.txt <"${dir}/.tmp/passwords.txt"
        fi

        end_func "Results are saved in $domain/osint/emails.txt and passwords.txt" "${FUNCNAME[0]}"
    else
        if [[ $EMAILS == false ]] || [[ $OSINT == false ]]; then
            skip_notification "disabled"
        elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
            return
        else
            skip_notification "processed"
        fi
    fi
}

function domain_info() {

    ensure_dirs osint

    # Check if the function should run
    if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
        && [[ $DOMAIN_INFO == true ]] && [[ $OSINT == true ]] \
        && ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then

        start_func "${FUNCNAME[0]}" "Searching domain info (whois, registrant name/email domains)"

        # Run whois command and check for errors
        run_command whois "$domain" >"osint/domain_info_general.txt"
        : >osint/azure_tenant_domains.txt
        local msftrecon_output msftrecon_rc
        msftrecon_output=$(mktemp)
        msftrecon_rc=0
        if ! run_command "${tools}/msftrecon/venv/bin/python3" "${tools}/msftrecon/msftrecon/msftrecon.py" -d "${domain}" \
            >"$msftrecon_output" 2>>"$LOGFILE"; then
            msftrecon_rc=$?
        fi

        if [[ $msftrecon_rc -eq 0 ]]; then
            cat "$msftrecon_output" >osint/azure_tenant_domains.txt
        else
            log_note "domain_info: msftrecon failed (rc=${msftrecon_rc}), continuing" "${FUNCNAME[0]}" "${LINENO}"
            _print_msg WARN "domain_info: msftrecon failed, continuing"
            : >osint/azure_tenant_domains.txt
        fi
        rm -f "$msftrecon_output"

        company_name=$(unfurl format %r <<<"$domain")
        run_command "${tools}/Scopify/venv/bin/python3" "${tools}/Scopify/scopify.py" -c ${company_name} >osint/scopify.txt

        end_func "Results are saved in ${domain}/osint/domain_info_[general/azure_tenant_domains].txt" "${FUNCNAME[0]}"

    else
        if [[ $DOMAIN_INFO == false ]] || [[ $OSINT == false ]]; then
            skip_notification "disabled"
        elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
            return
        else
            skip_notification "processed"
        fi
    fi
}

function third_party_misconfigs() {
    ensure_dirs osint

    # Check if the function should run
    if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
        && [[ $THIRD_PARTIES == true ]] && [[ $OSINT == true ]] \
        && ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then

        start_func "${FUNCNAME[0]}" "Searching for third parties misconfigurations"

        # Extract company name from domain
        company_name=$(unfurl format %r <<<"$domain")

        # Run misconfig-mapper in a subshell to avoid CWD pollution
        (
            cd "${tools}/misconfig-mapper" || exit 1
            run_command misconfig-mapper -update-templates 1>>"$LOGFILE"
            run_command misconfig-mapper -target "$domain" -as-domain true -permutations false -skip-ssl \
                -service "*" -verbose 0 | anew -q "${dir}/osint/3rdparts_misconfigurations.txt"
            run_command misconfig-mapper -target "$company_name" -skip-ssl -verbose 0 -service "*" \
                | anew -q "${dir}/osint/3rdparts_misconfigurations.txt"
        )

        end_func "Results are saved in $domain/osint/3rdparts_misconfigurations.txt" "${FUNCNAME[0]}"

    else
        if [[ $THIRD_PARTIES == false ]] || [[ $OSINT == false ]]; then
            skip_notification "disabled"
        elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
            return
        else
            skip_notification "processed"
        fi
    fi
}

function lunar_exposure() {
    ensure_dirs osint

    if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
        && [[ $LUNAR_EXPOSURE == true ]] && [[ $OSINT == true ]] \
        && ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then

        start_func "${FUNCNAME[0]}" "Domain exposure in infostealer logs and breaches (Lunar)"

        local _out="${dir}/osint/lunar_exposure.json"
        local _tmp _code
        if ! _tmp=$(mktemp -t reconftw_lunar.XXXXXX); then
            end_func "Could not create a temporary file for the Lunar query" "${FUNCNAME[0]}"
            return
        fi

        # Staged, then published by rename: a truncated or error body must not
        # replace a good result from a previous run.
        _code=$(run_command curl -s --max-time "${LUNAR_TIMEOUT:-30}" \
            -H 'Accept: application/json' \
            -w '%{http_code}' -o "$_tmp" \
            "https://api.lunarcyber.com/domain-exposure?domain=${domain}" 2>>"$LOGFILE") || _code="000"

        if [[ "$_code" != "200" ]]; then
            rm -f "$_tmp" 2>/dev/null
            end_func "Lunar API returned HTTP ${_code:-000}; no exposure data recorded" "${FUNCNAME[0]}"
            return
        fi

        # A 200 carrying a non-JSON body is an outage or interstitial page, not
        # data. Record nothing rather than a file downstream has to guess about.
        if ! jq -e . "$_tmp" >/dev/null 2>&1; then
            rm -f "$_tmp" 2>/dev/null
            end_func "Lunar API returned 200 with a non-JSON body; nothing recorded" "${FUNCNAME[0]}"
            return
        fi

        mv "$_tmp" "$_out"
        end_func "Results are saved in $domain/osint/lunar_exposure.json" "${FUNCNAME[0]}"

    else
        if [[ $LUNAR_EXPOSURE == false ]] || [[ $OSINT == false ]]; then
            skip_notification "disabled"
        elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
            return
        else
            skip_notification "processed"
        fi
    fi
}

function spoof() {
    ensure_dirs osint

    # Check if the function should run
    if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
        && [[ $SPOOF == true ]] && [[ $OSINT == true ]] \
        && ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then

        start_func "${FUNCNAME[0]}" "Searching for spoofable domains"

        # Run Spoofy in a subshell to avoid CWD pollution
        (
            cd "${tools}/Spoofy" || exit 1
            run_command "${tools}/Spoofy/venv/bin/python3" spoofy.py -d "$domain" >"${dir}/osint/spoof.txt"
        )

        end_func "Results are saved in $domain/osint/spoof.txt" "${FUNCNAME[0]}"

    else
        if [[ $SPOOF == false ]] || [[ $OSINT == false ]]; then
            skip_notification "disabled"
        elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
            return
        else
            skip_notification "processed"
        fi
    fi
}

function mail_hygiene() {
    ensure_dirs osint

    if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
        && [[ $MAIL_HYGIENE == true ]] && [[ $OSINT == true ]] \
        && ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then

        start_func "${FUNCNAME[0]}" "Mail hygiene (SPF/DMARC)"
        {
            printf "Domain: %s\n" "$domain"
            printf "\nTXT records:\n"
            run_command dig +short TXT "$domain" | sed 's/^/  /'
            printf "\nDMARC record:\n"
            run_command dig +short TXT "_dmarc.$domain" | sed 's/^/  /'
        } >"osint/mail_hygiene.txt" 2>>"$LOGFILE"
        end_func "Results are saved in $domain/osint/mail_hygiene.txt" "${FUNCNAME[0]}"
    else
        if [[ $MAIL_HYGIENE == false ]] || [[ $OSINT == false ]]; then
            skip_notification "disabled"
        elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
            return
        else
            skip_notification "processed"
        fi
    fi
}

function cloud_enum_scan() {
    ensure_dirs osint

    if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
        && [[ $CLOUD_ENUM == true ]] && [[ $OSINT == true ]] \
        && ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then

        local -a cloud_enum_cmd

        start_func "${FUNCNAME[0]}" "Cloud storage enumeration"
        if ! build_cloud_enum_command cloud_enum_cmd "${FUNCNAME[0]}" "$domain"; then
            return 0
        fi

        run_command "${cloud_enum_cmd[@]}" 2>>"$LOGFILE" | anew -q osint/cloud_enum.txt
        end_func "Results are saved in $domain/osint/cloud_enum.txt" "${FUNCNAME[0]}"
    else
        if [[ $CLOUD_ENUM == false ]] || [[ $OSINT == false ]]; then
            skip_notification "disabled"
        elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
            return
        else
            skip_notification "processed"
        fi
    fi
}

function ip_info() {

    ensure_dirs osint

    # Check if the function should run
    if { [[ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ]] || [[ $DIFF == true ]]; } \
        && [[ $IP_INFO == true ]] && [[ $OSINT == true ]] \
        && [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then

        start_func "${FUNCNAME[0]}" "Searching IP info"

        if [[ -n $WHOISXML_API ]]; then

            # Reverse IP lookup
            run_command curl -s "https://reverse-ip.whoisxmlapi.com/api/v1?apiKey=${WHOISXML_API}&ip=${domain}" \
                | jq -r '.result[].name' 2>>"$LOGFILE" \
                | sed -e "s/$/ ${domain}/" \
                | anew -q "osint/ip_${domain}_relations.txt"

            # WHOIS lookup
            run_command curl -s "https://www.whoisxmlapi.com/whoisserver/WhoisService?apiKey=${WHOISXML_API}&domainName=${domain}&outputFormat=json&da=2&registryRawText=1&registrarRawText=1&ignoreRawTexts=1" \
                | jq 2>>"$LOGFILE" \
                | anew -q "osint/ip_${domain}_whois.txt"

            # IP Geolocation
            run_command curl -s "https://ip-geolocation.whoisxmlapi.com/api/v1?apiKey=${WHOISXML_API}&ipAddress=${domain}" \
                | jq -r '.ip,.location' 2>>"$LOGFILE" \
                | anew -q "osint/ip_${domain}_location.txt"

            end_func "Results are saved in ${domain}/osint/ip_[domain_relations|whois|location].txt" "${FUNCNAME[0]}"

        else
            _print_msg WARN "WHOISXML_API variable is not defined. Skipping function."
            end_func "WHOISXML_API not configured, ip_info skipped." "${FUNCNAME[0]}"
        fi

    else
        if [[ $IP_INFO == false ]] || [[ $OSINT == false ]]; then
            skip_notification "disabled"
        elif ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
            return
        else
            skip_notification "processed"
        fi
    fi

}