Skip to content

reconFTW is a tool designed to perform automated recon on a target domain by running the best set of tools to perform scanning and finding out vulnerabilities

Fixes

six2deze19a2657a06d
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
deleted file mode 100644
index 07b49c2..0000000
--- a/.github/workflows/codeql.yml
+++ /dev/null
@@ -1,65 +0,0 @@
-name: "CodeQL"
-
-on:
- push:
- branches: [ 'main', 'dev' ]
- pull_request:
- # The branches below must be a subset of the branches above
- branches: [ 'main' ]
- schedule:
- - cron: '46 8 * * 3'
-
-jobs:
- analyze:
- name: Analyze
- runs-on: ubuntu-latest
- permissions:
- actions: read
- contents: read
- security-events: write
-
- strategy:
- fail-fast: false
- matrix:
- language: [ 'javascript', 'python' ]
- # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
- # Use only 'java' to analyze code written in Java, Kotlin or both
- # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both
- # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
-
- steps:
- - name: Checkout repository
- uses: actions/checkout@v4
-
- # Initializes the CodeQL tools for scanning.
- - name: Initialize CodeQL
- uses: github/codeql-action/init@v3
- with:
- languages: ${{ matrix.language }}
- # If you wish to specify custom queries, you can do so here or in a config file.
- # By default, queries listed here will override any specified in a config file.
- # Prefix the list here with "+" to use these queries and those in the config file.
-
- # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
- # queries: security-extended,security-and-quality
-
-
- # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
- # If this step fails, then you should remove it and run the build manually (see below)
- - name: Autobuild
- uses: github/codeql-action/autobuild@v3
-
- # ℹ️ Command-line programs to run using the OS shell.
- # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-
- # If the Autobuild fails above, remove it and uncomment the following three lines.
- # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
-
- # - run: |
- # echo "Run, Build Application using script"
- # ./location_of_script_within_repo/buildscript.sh
-
- - name: Perform CodeQL Analysis
- uses: github/codeql-action/analyze@v3
- with:
- category: "/language:${{matrix.language}}"
diff --git a/.github/workflows/docker_nightly.yml b/.github/workflows/docker_nightly.yml
index 3a068a9..e987728 100644
--- a/.github/workflows/docker_nightly.yml
+++ b/.github/workflows/docker_nightly.yml
@@ -34,6 +34,22 @@ jobs:
run: |
sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /opt/hostedtoolcache
sudo docker system prune -af || true
+ - name: Extract Docker metadata
+ id: meta
+ uses: docker/metadata-action@v5
+ with:
+ images: six2dez/reconftw
+ # Tagging strategy:
+ # release -> vX.Y.Z, X.Y, X, latest, main
+ # schedule (cron)-> nightly, main
+ # push to main -> main
+ tags: |
+ type=raw,value=main,enable={{is_default_branch}}
+ type=raw,value=nightly,enable=${{ github.event_name == 'schedule' }}
+ type=raw,value=latest,enable=${{ github.event_name == 'release' }}
+ type=semver,pattern={{version}}
+ type=semver,pattern={{major}}.{{minor}}
+ type=semver,pattern={{major}}
- name: Build and push
id: docker_build
uses: docker/build-push-action@v5
@@ -41,16 +57,20 @@ jobs:
context: ./Docker
platforms: linux/amd64,linux/arm64
push: true
- tags: six2dez/reconftw:main
+ tags: ${{ steps.meta.outputs.tags }}
+ labels: ${{ steps.meta.outputs.labels }}
- name: Verify published manifest
run: |
- docker buildx imagetools inspect six2dez/reconftw:main > /tmp/reconftw-manifest.txt
+ # Pick the first pushed tag to inspect.
+ first_tag=$(printf '%s\n' "${{ steps.meta.outputs.tags }}" | head -n1)
+ docker buildx imagetools inspect "$first_tag" > /tmp/reconftw-manifest.txt
grep -q 'linux/amd64' /tmp/reconftw-manifest.txt
grep -q 'linux/arm64' /tmp/reconftw-manifest.txt
- name: Verify image health
run: |
- docker pull six2dez/reconftw:main
- docker run --rm six2dez/reconftw:main --help
- docker run --rm six2dez/reconftw:main --health-check
+ first_tag=$(printf '%s\n' "${{ steps.meta.outputs.tags }}" | head -n1)
+ docker pull "$first_tag"
+ docker run --rm "$first_tag" --help
+ docker run --rm "$first_tag" --health-check
diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml
new file mode 100644
index 0000000..c703444
--- /dev/null
+++ b/.github/workflows/semgrep.yml
@@ -0,0 +1,49 @@
+name: Semgrep
+
+# Replaces the previous CodeQL workflow, which was configured for JavaScript
+# and Python — neither language exists in this repo. Semgrep covers bash and
+# other languages reconFTW actually ships.
+
+on:
+ push:
+ branches: [main, dev]
+ pull_request:
+ branches: [main, dev]
+ schedule:
+ - cron: '46 8 * * 3'
+ workflow_dispatch:
+
+permissions:
+ contents: read
+ security-events: write
+
+jobs:
+ semgrep:
+ name: Semgrep SAST
+ runs-on: ubuntu-latest
+ container:
+ image: semgrep/semgrep:latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Run Semgrep
+ # No --error so the job does not fail CI on findings; results are
+ # uploaded to the GitHub security tab for review. Flip to --error
+ # once the backlog is burned down.
+ run: |
+ semgrep scan \
+ --config p/bash \
+ --config p/security-audit \
+ --config p/secrets \
+ --config p/dockerfile \
+ --config p/github-actions \
+ --sarif \
+ --output=semgrep.sarif \
+ --metrics=off
+
+ - name: Upload SARIF
+ if: always()
+ uses: github/codeql-action/upload-sarif@v3
+ with:
+ sarif_file: semgrep.sarif
+ category: semgrep
diff --git a/.gitignore b/.gitignore
index b858017..051b667 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,9 +14,40 @@ test/
.DS_Store
install.log
reconftw.cfg-personal
+
+# Secrets and credential-shaped files (never commit)
secrets.cfg
+secrets.cfg.*
+!secrets.cfg.example
+.env
+.env.*
+!.env.example
.env.cache/
+*.tokens
+**/*_tokens*
+**/.github_tokens
+*.pem
+*.key
+.shodan.cfg
+
documentation/
# Vendored wordlists that are stored compressed in-repo (expanded on-demand)
data/wordlists/subdomains.txt
+
+# Terraform
+**/.terraform/
+*.tfstate
+*.tfstate.*
+*.tfvars
+*.tfvars.json
+crash.log
+crash.*.log
+override.tf
+*_override.tf
+override.tf.json
+*_override.tf.json
+.terraformrc
+terraform.rc
+
+.claude
diff --git a/.shellcheckrc b/.shellcheckrc
new file mode 100644
index 0000000..616e985
--- /dev/null
+++ b/.shellcheckrc
@@ -0,0 +1,5 @@
+external-sources=true
+severity=warning
+source-path=SCRIPTDIR
+source-path=SCRIPTDIR/lib
+source-path=SCRIPTDIR/modules
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8df4dfb..57fb79b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Removed
- Removed metagoofil from `install.sh` git repos, `config/tool_versions.txt`, and `modules/core.sh` tool checks.
+### Fixed
+- `sanitize_domain()` (`modules/utils.sh`) now URL-aware. Previously `tr -cd 'a-zA-Z0-9.-'` stripped `://` from inputs like `https://example.com`, producing a garbage hostname (`httpsexample.com`) that poisoned the DNS/probe pipeline. The function now normalizes the input first (strip scheme, userinfo, path/query/fragment, port) and validates IPv4 octets inline (rejects `https://1.2.3.999`). The whitelist `tr` step is preserved as last-line injection defense. New bats cases cover scheme, path, port, userinfo, query/fragment, `@` in path, and IPv4 URLs (valid + invalid).
+- Web mode (`-w`) accepts both `-d <domain|url>` and `-l <file>`. Previously `-d ... -w` errored out with "Web mode needs a website list file as target" because the `-w` dispatch branch only supported `-l`. The rewritten branch in `reconftw.sh` unifies both paths and errors only when neither is provided.
+- `-l <file> -w` pipeline now runs end-to-end. Before this fix, the `-w` branch copied the raw list to `webs/webs.txt` but `webprobe_full` reads from `subdomains/subdomains.txt` — which stayed empty — so probing skipped with `SKIP_NOINPUT` and cascaded skips through `urlchecks`, `sub_js_extract`, `url_gf`, `url_ext`. The branch now populates `subdomains/subdomains.txt` with sanitized hostnames (via `_sanitize_list_entry`), and truncates `webs/webs.txt` after `start()` so `webprobe_full` is the sole source of live URLs — downstream modules no longer mix raw user input (URLs with arbitrary paths/ports/dead hosts) with probed results.
+- `webprobe_full` and its helpers (`_extract_probe_urls_from_json`, `_write_plain_webinfo`) no longer discard all probe results in list mode. The `jq` scope filter `select((.url // "") | contains($dom))` in `modules/web.sh:213,225` matched the `$domain` value literally, which is `"Multi"` in `-l` runs — so every real URL was filtered out. Now bypassed when `$dom == "Multi"`. Same fix applied to `filter_in_scope_urls` (`lib/validation.sh`) and the `awk index($0, dom)` gate in `_write_plain_webinfo`. `webs/webs.txt`, `webs/webs_uncommon_ports.txt`, `webs/web_full_info.txt`, and `webs/web_full_info_plain.txt` are now properly populated in list mode.
+- `-w` mode no longer leaks stale state across runs, nor clobbers enumeration data from prior non-web runs. The shared output dir (`Recon/Multi` for `-l`, `Recon/<domain>` for `-d`) is reused between invocations, and many webs_menu modules persist via `anew` / `>>` across a wide set of paths (webs/webs_all.txt, webs/url_extract*, webs/virtualhosts.txt, .tmp/* caches, root-level `assets.jsonl`, js/, fuzzing/, gf/, screenshots/, nuclei_output/**, cms/, vulns/**, subdomains/subtakeover.txt, subdomains/* from sub_js_extract/tls_ip_pivots, hosts/grpc_reflection.txt) or skip on checkpoint (`.called_fn/.<func>`). A new helper `prepare_web_mode_scope()` in `modules/modes.sh` differentiates by input mode to match each scope's semantics:
+ * **`-l` Multi mode**: the shared dir is *only* ever populated by `-l -w` invocations, so nothing in it predates a web run. The helper makes a read-only snapshot of the input list via `cp "$flist" "$_list_snapshot"` (tmpfile outside `$dir`) up front, then wipes every file under `$dir` except cross-run tracking (`.log/*`, `.incremental/*`, `.called_fn/*`, `debug.log`) plus `$flist` itself (belt-and-suspenders for when the list path falls inside `$dir`, e.g. `-l Recon/Multi/scope.txt`). All subsequent operations — including CRLF stripping via bash parameter expansion `${_t%$'\r'}` — run against the snapshot. The user's input file is provably untouched post-snapshot: content md5, mtime, and inode all survive a run. This catches current and future output dirs (`nuclei_output/`, `cms/`, `virtualhosts/`, any new module's outputs) without requiring an explicit allowlist to stay in sync. `subdomains/subdomains.txt` is written with `sort -u >` from the sanitized snapshot — each list defines its own scope.
+ * **`-d <domain>` mode**: the per-domain dir may hold prior `-a`/`-s`/`-p` recon state that must survive (subdomain enumeration, `hosts/ips.txt` and `hosts/origin_ips.txt` needed by `virtualhosts()`/`tls_ip_pivots()`, `nuclei_output/**` history, `osint/*`, etc.). The helper wipes only pure web-pipeline artifacts (`webs/*`, `.tmp/*`, `js/*`, `fuzzing/*`, `gf/*`, `screenshots/*`, `assets.jsonl`) and preserves the rest. The target is appended to `subdomains/subdomains.txt` via `anew -q` so enumerated subs stay in scope.
+The one-shot `-w` dispatch also sets `DIFF=true`, matching the global `DIFF=true` that `monitor_mode` already sets. Both the one-shot `-w` dispatch and every `monitor_mode -w -l` cycle now call the same helper and check its return status — if the new scope can't be validated (mktemp/cp failed, or every list entry was rejected by sanitization so the staged hostname list ended up empty), the helper aborts **before** wiping `$dir` so prior state is preserved, the one-shot path exits with a clear error, and the monitor path skips `webs_menu` for that cycle only and falls through to the normal post-case flow (`monitor_snapshot`, `MONITOR_MAX_CYCLES` check, `sleep $interval_sec`). An earlier `continue` branch was removed because it bypassed those checks and busy-looped on persistent prep failures. Consecutive `-w` runs produce fully scoped output with zero cross-run contamination while preserving INCREMENTAL_MODE baselines and prior enumeration coverage.
+
## [v4.1] - 2026-03-06
### Added
diff --git a/Docker/Dockerfile b/Docker/Dockerfile
index ed95442..414455e 100644
--- a/Docker/Dockerfile
+++ b/Docker/Dockerfile
@@ -114,7 +114,11 @@ rm -rf /root/.cache
rm -rf /root/go/pkg /root/go/src
eot
-COPY github_tokens.txt /root/Tools/.github_tokens
+# Tokens for github-subdomains / gitdorks_go / github-endpoints / trufflehog are
+# expected at runtime via the GITHUB_TOKENS env var (one token per line, or a
+# path to a file that contains them). See Docker/entrypoint.sh — building them
+# into the image would ship a tracked "shape of a secret" that invites users
+# to fill it with real tokens and accidentally commit them.
COPY notify.conf /root/.config/notify/provider-config.yaml
###>> Configure Axiom Provider <<###
diff --git a/Docker/entrypoint.sh b/Docker/entrypoint.sh
index e27b66c..6d46d43 100644
--- a/Docker/entrypoint.sh
+++ b/Docker/entrypoint.sh
@@ -6,4 +6,20 @@ if [[ "${INSTALL_AXIOM}" == "true" ]] && [[ ! -f /root/.ssh/axiom_rsa ]]; then
cat /root/.ssh/axiom_rsa.pub > /root/.axiom/configs/authorized_keys
fi
+# Accept GitHub tokens at runtime instead of baking them into the image.
+# Supported inputs:
+# GITHUB_TOKENS="ghp_aaaa\nghp_bbbb" (literal newline-separated tokens)
+# GITHUB_TOKENS=/path/to/file (path to a file inside the container)
+# Writes them to the location reconftw.cfg expects (${tools}/.github_tokens).
+if [[ -n "${GITHUB_TOKENS:-}" ]]; then
+ mkdir -p /root/Tools
+ target=/root/Tools/.github_tokens
+ if [[ -f "$GITHUB_TOKENS" ]]; then
+ cp -f -- "$GITHUB_TOKENS" "$target"
+ else
+ printf '%b\n' "$GITHUB_TOKENS" > "$target"
+ fi
+ chmod 600 "$target"
+fi
+
exec ./reconftw.sh "$@"
diff --git a/Docker/github_tokens.txt b/Docker/github_tokens.txt
deleted file mode 100644
index 5cdb70b..0000000
--- a/Docker/github_tokens.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 545cea3..d35a4fc 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,7 @@
GH_CLI := $(shell command -v gh 2> /dev/null)
-PRIVATE_REPO := $(shell echo $${PRIV_REPO-reconftw-data})
+# PRIV_REPO is read from the environment by each recipe below (not interpolated
+# via $(shell ...) so that shell metacharacters stay as data rather than syntax).
+# Default: reconftw-data.
.PHONY: sync upload bootstrap rm lint lint-fix fmt test test-all test-security test-unit test-integration-smoke test-integration-full test-release-gate setup-dev help
@@ -23,31 +25,53 @@ help:
@echo " make upload - Upload data to private repo"
# bootstrap a private repo to store data
+#
+# All repo/branch values are validated and quoted inside each recipe so shell
+# metacharacters in PRIV_REPO env or in the remote's HEAD ref name cannot be
+# interpreted by /bin/sh (cmd injection mitigation).
bootstrap:
- @if [ -z $(GH_CLI) ]; then echo "github cli is missing. please install"; exit 2; fi
- gh repo create $(PRIVATE_REPO) --private
- gh repo clone $(PRIVATE_REPO) ~/$(PRIVATE_REPO)
- cd ~/$(PRIVATE_REPO) && git commit --allow-empty -m "Empty commit" && \
- git remote add upstream https://github.com/six2dez/reconftw && \
- git fetch upstream && \
- git rebase upstream/main $(shell git symbolic-ref refs/remotes/origin/HEAD | sed 's@^refs/remotes/origin/@@') && \
- mkdir Recon && \
- git push origin $(shell git symbolic-ref refs/remotes/origin/HEAD | sed 's@^refs/remotes/origin/@@')
- @echo "Done!"
- @echo "Initialized private repo: $(PRIVATE_REPO)"
+ @if [ -z "$(GH_CLI)" ]; then echo "github cli is missing. please install"; exit 2; fi
+ @repo="$${PRIV_REPO:-reconftw-data}"; \
+ case "$$repo" in (*[!A-Za-z0-9._/-]*|'') echo "invalid repo name: $$repo"; exit 2;; esac; \
+ gh repo create "$$repo" --private && \
+ gh repo clone "$$repo" "$$HOME/$$repo" && \
+ cd "$$HOME/$$repo" && \
+ branch=$$(git symbolic-ref --short refs/remotes/origin/HEAD | sed 's@^origin/@@') && \
+ git check-ref-format --branch "$$branch" >/dev/null || { echo "invalid default branch: $$branch"; exit 1; } && \
+ git commit --allow-empty -m "Empty commit" && \
+ git remote add upstream https://github.com/six2dez/reconftw && \
+ git fetch upstream && \
+ git rebase upstream/main "$$branch" && \
+ mkdir Recon && \
+ git push origin "$$branch" && \
+ echo "Done!" && \
+ echo "Initialized private repo: $$repo"
+ @echo "bootstrap complete"
rm:
- gh repo delete $(PRIVATE_REPO) --yes
- rm -rf ~/$(PRIVATE_REPO)
+ @repo="$${PRIV_REPO:-reconftw-data}"; \
+ case "$$repo" in (*[!A-Za-z0-9._/-]*|'') echo "invalid repo name: $$repo"; exit 2;; esac; \
+ gh repo delete "$$repo" --yes && \
+ rm -rf -- "$$HOME/$$repo"
sync:
- cd ~/$(PRIVATE_REPO) && git fetch upstream && git rebase upstream/main $(shell git symbolic-ref refs/remotes/origin/HEAD | sed 's@^refs/remotes/origin/@@')
+ @repo="$${PRIV_REPO:-reconftw-data}"; \
+ case "$$repo" in (*[!A-Za-z0-9._/-]*|'') echo "invalid repo name: $$repo"; exit 2;; esac; \
+ cd "$$HOME/$$repo" && \
+ git fetch upstream && \
+ branch=$$(git symbolic-ref --short refs/remotes/origin/HEAD | sed 's@^origin/@@') && \
+ git check-ref-format --branch "$$branch" >/dev/null || { echo "invalid default branch: $$branch"; exit 1; } && \
+ git rebase upstream/main "$$branch"
upload:
- cd ~/$(PRIVATE_REPO) && \
- git add . && \
- git commit -m "Data upload" && \
- git push origin $(shell git symbolic-ref refs/remotes/origin/HEAD | sed 's@^refs/remotes/origin/@@')
+ @repo="$${PRIV_REPO:-reconftw-data}"; \
+ case "$$repo" in (*[!A-Za-z0-9._/-]*|'') echo "invalid repo name: $$repo"; exit 2;; esac; \
+ cd "$$HOME/$$repo" && \
+ git add . && \
+ git commit -m "Data upload" && \
+ branch=$$(git symbolic-ref --short refs/remotes/origin/HEAD | sed 's@^origin/@@') && \
+ git check-ref-format --branch "$$branch" >/dev/null || { echo "invalid default branch: $$branch"; exit 1; } && \
+ git push origin "$$branch"
lint:
@if command -v shellcheck >/dev/null 2>&1; then \
diff --git a/Proxmox/reconftw_prox_deploy.sh b/Proxmox/reconftw_prox_deploy.sh
index 1b743c0..fde1b2b 100644
--- a/Proxmox/reconftw_prox_deploy.sh
+++ b/Proxmox/reconftw_prox_deploy.sh
@@ -8,7 +8,12 @@ YELLOW='\033[1;33m'
NC='\033[0m'
# Logging configuration
+# umask 077 so only root can read the log (it records container hostname,
+# IDs and other metadata). The container root password is NOT written to this
+# log; it is printed only to the controlling TTY (fd 3) at the end of the run.
+umask 077
LOGFILE="/var/log/reconftw_deploy_$(date +%Y%m%d_%H%M%S).log"
+exec 3>/dev/tty 2>/dev/null || exec 3>&1
exec 1> >(tee -a "$LOGFILE") 2>&1
# Logging function
@@ -146,5 +151,8 @@ log "${GREEN}Installation completed${NC}"
echo "Container information:"
echo "ID: $CONTAINER_ID"
echo "Hostname: $HOSTNAME"
-echo "Password: $PASSWORD"
+# Print the root password only to the controlling TTY (fd 3) so it is not
+# captured by the global `tee -a "$LOGFILE"` stdout redirection above.
+printf 'Password: %s\n' "$PASSWORD" >&3
+echo "Password: [printed to terminal only; not stored in log]"
echo "Log file: $LOGFILE"
\ No newline at end of file
diff --git a/Terraform/.terraform.lock.hcl b/Terraform/.terraform.lock.hcl
new file mode 100644
index 0000000..51d3d37
--- /dev/null
+++ b/Terraform/.terraform.lock.hcl
@@ -0,0 +1,25 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/aws" {
+ version = "6.41.0"
+ constraints = ">= 5.0.0"
+ hashes = [
+ "h1:P5G2OYd40P7QUS0dM2uw3WJ0y+VzOoIO7RvRRqA62D0=",
+ "zh:01835476adda6d93095e37fdf782f14e6709f6922dc62e88994f9684627deb69",
+ "zh:0b9bc5eda9def53df19e1a37562dcb67c1fba8452803e1b5601e75653c986255",
+ "zh:196f81d97ea2951d2c6667709445d7c36b5fd8603890c774495806b4da0743aa",
+ "zh:1ba36118b0146e5c3603020509b34d09e693db50ec29f9be5badc9f2a4fd95b8",
+ "zh:4253c2f6066ce279e5ce48849c78fc193f222dc42a929e6876b9563ed5c23fcf",
+ "zh:457ed8609680338dbcb9809e263638a530c06ba43208af9e660803133dc5e1e6",
+ "zh:4e8de5f3fcc3e3f41b6703ad4c0fa62175d0c29afcf56ac82eb16c604bb6dafa",
+ "zh:583ae622cfe4633fabca071ded738e9ef3398d9e9916cb26350aad28068462c5",
+ "zh:5b8bf11070c13e21a0fef05713a25be0392c7d064bd2199c2f99939cc345e8c5",
+ "zh:6aa7ae41d4fff4e95cedcbeaa143b2af9ad3e3a4b40208801b701d77a738b2c7",
+ "zh:8143670bca48af3b873b0db83443dbdcb868442f1e789ffba356d6adf4dbd7b9",
+ "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
+ "zh:c22951cf0a4b169607ea066717dd499f46221af035903497b97f24590fb79d2c",
+ "zh:dbd9cd975206a41a9c12006d3a30c77b95c0e660fcba2cc3bb0ee5779431c107",
+ "zh:e6ea2e0f142001b7f2229e8d39eb7f8037084723861be9d53478005196cd7f9e",
+ ]
+}
diff --git a/Terraform/reconFTW.yml b/Terraform/reconFTW.yml
index 38da797..359332c 100644
--- a/Terraform/reconFTW.yml
+++ b/Terraform/reconFTW.yml
@@ -5,6 +5,8 @@
reconftw_repo: "https://github.com/six2dez/reconftw.git"
reconftw_dest: "/opt/reconftw"
reconftw_branch_default: "main"
+ reconftw_commit: "" # passed from Terraform; "" means branch-based deploy
+ reconftw_owner: "admin" # OS user that owns reconftw.cfg (not root)
tasks:
- name: Update apt cache
ansible.builtin.apt:
@@ -16,22 +18,57 @@
name: git
state: present
- - name: Clone reconFTW
+ # If reconftw_commit is supplied we pin to that exact SHA. Otherwise fall
+ # back to branch-based deploys (backwards compatible). Pinned deploys also
+ # run a follow-up assert to fail fast if the ref resolved to anything other
+ # than the expected commit.
+ - name: Clone reconFTW (pinned commit)
+ when: reconftw_commit | length > 0
+ ansible.builtin.git:
+ repo: "{{ reconftw_repo }}"
+ dest: "{{ reconftw_dest }}"
+ version: "{{ reconftw_commit }}"
+ update: false
+ depth: 1
+
+ - name: Clone reconFTW (branch)
+ when: reconftw_commit | length == 0
ansible.builtin.git:
repo: "{{ reconftw_repo }}"
dest: "{{ reconftw_dest }}"
version: "{{ reconftw_branch | default(reconftw_branch_default) }}"
- update: yes
+ update: true
- - name: Install reconFTW dependencies and tools
- ansible.builtin.command: ./install.sh --verbose
+ - name: Verify checked-out HEAD matches reconftw_commit
+ when: reconftw_commit | length > 0
+ ansible.builtin.command: git rev-parse HEAD
args:
chdir: "{{ reconftw_dest }}"
+ register: _reconftw_head
+ changed_when: false
- - name: Copy reconftw.cfg
+ - name: Abort on unexpected revision
+ when: reconftw_commit | length > 0
+ ansible.builtin.assert:
+ that:
+ - _reconftw_head.stdout == reconftw_commit
+ fail_msg: >-
+ Checked out reconFTW revision {{ _reconftw_head.stdout }} does not
+ match the pinned commit {{ reconftw_commit }}.
+
+ # Copy the operator-controlled reconftw.cfg BEFORE running install.sh so
+ # install.sh sources the trusted config (not whatever was in the cloned
+ # repo). Mode 0600 + owner reconftw_owner means only that account — and
+ # root — can read any API keys the operator placed in the file.
+ - name: Copy reconftw.cfg with restricted permissions
ansible.builtin.copy:
src: files/reconftw.cfg
dest: "{{ reconftw_dest }}/reconftw.cfg"
- owner: root
- group: root
- mode: "0644"
+ owner: "{{ reconftw_owner }}"
+ group: "{{ reconftw_owner }}"
+ mode: "0600"
+
+ - name: Install reconFTW dependencies and tools
+ ansible.builtin.command: ./install.sh --verbose
+ args:
+ chdir: "{{ reconftw_dest }}"
diff --git a/Terraform/terraform-reconFTW.tf b/Terraform/terraform-reconFTW.tf
index cfa8901..61d01e9 100644
--- a/Terraform/terraform-reconFTW.tf
+++ b/Terraform/terraform-reconFTW.tf
@@ -84,7 +84,12 @@ resource "aws_instance" "reconftw" {
provisioner "local-exec" {
working_dir = path.module
- command = "ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook -i '${self.public_ip},' -u ${var.ssh_user} --private-key ${var.private_key_path} -e reconftw_branch=${var.reconftw_branch} reconFTW.yml"
+ # Quote extra-vars values so they reach ansible-playbook as a single argv
+ # token. reconftw_branch is regex-validated in variables.tf, so shell
+ # metacharacters cannot appear here, but quoting keeps the guarantee
+ # future-proof. reconftw_commit is passed too; the playbook ignores it
+ # when empty.
+ command = "ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook -i '${self.public_ip},' -u ${var.ssh_user} --private-key ${var.private_key_path} -e 'reconftw_branch=${var.reconftw_branch}' -e 'reconftw_commit=${var.reconftw_commit}' reconFTW.yml"
}
}
diff --git a/Terraform/variables.tf b/Terraform/variables.tf
index 4b51898..a2f55eb 100644
--- a/Terraform/variables.tf
+++ b/Terraform/variables.tf
@@ -43,6 +43,26 @@ variable "reconftw_branch" {
description = "Git branch/tag to deploy on the instance via Ansible."
type = string
default = "main"
+
+ validation {
+ # Restrict to the characters Git ref names commonly use. Blocks shell
+ # metacharacters (space, ;, |, &, $, backtick, newline, quotes) that would
+ # otherwise break out of the `-e reconftw_branch=...` arg to ansible-playbook
+ # in the local-exec provisioner.
+ condition = can(regex("^[0-9A-Za-z._/-]+$", var.reconftw_branch))
+ error_message = "reconftw_branch may contain only letters, digits, dot, underscore, slash, and dash."
+ }
+}
+
+variable "reconftw_commit" {
+ description = "Optional immutable reconFTW commit SHA (40 hex chars). If set, the Ansible playbook checks out this exact commit and aborts if HEAD does not match, mitigating unpinned-ref supply-chain drift. Leave empty to keep branch-based deploys."
+ type = string
+ default = ""
+
+ validation {
+ condition = var.reconftw_commit == "" || can(regex("^[0-9a-fA-F]{40}$", var.reconftw_commit))
+ error_message = "reconftw_commit must be empty or a 40-character hex commit SHA."
+ }
}
variable "tags" {
diff --git a/install.sh b/install.sh
index 3120409..c616c15 100755
--- a/install.sh
+++ b/install.sh
@@ -79,6 +79,28 @@ run_to() {
if [[ -n $TIMEOUT_CMD ]]; then "$TIMEOUT_CMD" "$secs" "$@"; else "$@"; fi
}
+# verify_sha256 <file> <expected-hex>
+# Returns 0 if the file's SHA-256 matches expected, 1 otherwise. Works with
+# both GNU sha256sum (Linux) and shasum -a 256 (macOS).
+verify_sha256() {
+ local file="$1"
+ local expected="$2"
+ local actual=""
+
+ [[ -n "$expected" ]] || return 0 # nothing to verify
+ [[ -s "$file" ]] || return 1
+
+ if command -v sha256sum >/dev/null 2>&1; then
+ actual=$(sha256sum "$file" | awk '{print $1}')
+ elif command -v shasum >/dev/null 2>&1; then
+ actual=$(shasum -a 256 "$file" | awk '{print $1}')
+ else
+ return 0 # no hashing tool available; skip (don't block install)
+ fi
+
+ [[ -n "$actual" && "$actual" == "$expected" ]]
+}
+
# Helper: optionally dry-run
run_cmd() {
if [[ $DRY_RUN == "true" ]]; then
@@ -473,9 +495,9 @@ function install_tools() {
local go_ok=0 go_skip=0 go_fail=0
for gotool in "${!gotools[@]}"; do
((++go_step))
- local _go_cmd="go install -v ${gotools[$gotool]}@latest"
# Always run go install so already-present binaries also get updated.
- if q bash -lc "$_go_cmd"; then
+ # argv form (not bash -lc) so arr values are data, not shell syntax.
+ if q go install -v "${gotools[$gotool]}@latest"; then
((++go_ok))
msg_ok "[$go_step/$total_go] ${gotool} installed"
else
@@ -1188,7 +1210,20 @@ function initial_setup() {
header "Downloading required files"
mkdir -p ${HOME}/.config/notify
- # Download required files with error handling
+ # Download required files with error handling.
+ #
+ # Supply-chain note: helpers fetched from third-party repos or gists run
+ # inside reconFTW (axiom_config.sh is chmod +x, getjswords.py is invoked
+ # per-target). A compromise of `m4ll0k/Bug-Bounty-Toolz@master` or of the
+ # gist revision = code execution during install.
+ #
+ # You can pin any entry by exporting an expected SHA-256 before running
+ # install.sh:
+ # export GETJSWORDS_SHA256=<64 hex chars>
+ # export AXIOM_CONFIG_SHA256=<64 hex chars>
+ # If set, each download is verified with `verify_sha256` and the install
+ # aborts on mismatch. If unset the previous upstream-trusting behaviour is
+ # preserved for backwards compatibility.
declare -A downloads=(
["notify_provider_config"]="https://gist.githubusercontent.com/six2dez/23a996bca189a11e88251367e6583053/raw ${HOME}/.config/notify/provider-config.yaml"
["getjswords"]="https://raw.githubusercontent.com/m4ll0k/Bug-Bounty-Toolz/master/getjswords.py ${tools}/getjswords.py"
@@ -1198,6 +1233,13 @@ function initial_setup() {
["axiom_config"]="https://gist.githubusercontent.com/six2dez/6e2d9f4932fd38d84610eb851014b26e/raw ${tools}/axiom_config.sh"
)
+ # Map of optional pinned checksums (env-var driven). Add more entries here
+ # to extend integrity checks. Empty string means "no verification".
+ declare -A download_sha256=(
+ ["getjswords"]="${GETJSWORDS_SHA256:-}"
+ ["axiom_config"]="${AXIOM_CONFIG_SHA256:-}"
+ )
+
local dl_step=0
local total_dl=${#downloads[@]}
for key in "${!downloads[@]}"; do
@@ -1215,7 +1257,19 @@ function initial_setup() {
mkdir -p "$(dirname "$destination")" 2>/dev/null || true
if with_spinner "[$dl_step/$total_dl] Fetching $key" retry 3 3 q_to 120 wget -q -O "$destination" "$url"; then
- msg_ok "[$dl_step/$total_dl] $key fetched"
+ # Optional integrity check (active only when a SHA is pinned).
+ local _expected="${download_sha256[$key]:-}"
+ if [[ -n "$_expected" ]]; then
+ if verify_sha256 "$destination" "$_expected"; then
+ msg_ok "[$dl_step/$total_dl] $key fetched (sha256 verified)"
+ else
+ msg_err "[$dl_step/$total_dl] $key sha256 mismatch for $url; refusing to install"
+ rm -f "$destination"
+ continue
+ fi
+ else
+ msg_ok "[$dl_step/$total_dl] $key fetched"
+ fi
else
msg_err "[$dl_step/$total_dl] Failed to download $key from $url"
continue
diff --git a/lib/parallel.sh b/lib/parallel.sh
index f758ba2..563faf1 100644
--- a/lib/parallel.sh
+++ b/lib/parallel.sh
@@ -23,6 +23,24 @@ PARALLEL_TRACE_SLOW_SECONDS="${PARALLEL_TRACE_SLOW_SECONDS:-30}"
declare -a _PARALLEL_PIDS=()
_PARALLEL_LAST_BADGE=""
+###############################################################################
+# Background-job throttle used by modules/{web,vulns,osint}.sh to parallelise
+# per-target loops that replaced `interlace -c` (removed in the Naxus audit
+# fix for command injection). Each iteration launches a subshell in the
+# background; this helper blocks until the running-job count falls below the
+# requested cap. Requires bash 4.3+ for `wait -n`.
+###############################################################################
+_throttle_jobs() {
+ local max="${1:-4}"
+ # Clamp to sane minimum so a misconfigured var doesn't busy-spawn.
+ [[ "$max" =~ ^[0-9]+$ ]] || max=4
+ (( max < 1 )) && max=1
+
+ while (( $(jobs -rp | wc -l) >= max )); do
+ wait -n 2>/dev/null || break
+ done
+}
+
_parallel_live_break() {
if declare -F ui_live_progress_break >/dev/null 2>&1; then
ui_live_progress_break
diff --git a/lib/validation.sh b/lib/validation.sh
index ea6bec8..51d2897 100644
--- a/lib/validation.sh
+++ b/lib/validation.sh
@@ -183,6 +183,110 @@ function sanitize_interlace_input() {
return 0
}
+# is_in_scope_host()
+# Description: Returns 0 if $1 (hostname) is exactly $2 (domain) or a proper
+# subdomain of it. Prevents substring false positives (exampleXcom.evil) used by
+# raw "grep -a $domain" checks.
+# Arguments: $1 - hostname, $2 - in-scope domain
+# Returns: 0 if in scope, 1 otherwise
+function is_in_scope_host() {
+ local host="${1,,}"
+ local domain="${2,,}"
+ host="${host%.}"
+ domain="${domain%.}"
+ [[ -n "$host" && -n "$domain" ]] || return 1
+ [[ "$host" == "$domain" || "$host" == *.".$domain" || "$host" == *."$domain" ]]
+}
+
+# filter_in_scope_urls()
+# Description: Reads URLs from stdin, writes to stdout only those whose parsed
+# hostname is exactly $1 or a subdomain of $1. Rejects URLs containing userinfo
+# (user:pass@) to block SSRF-via-basic-auth tricks, non-http(s) schemes, and
+# substring look-alikes. Depends on python3.
+# Arguments: $1 - in-scope domain
+function filter_in_scope_urls() {
+ local domain="$1"
+ [[ -z "$domain" ]] && return 0
+
+ # Use `python3 -c` (not `- <<HEREDOC`) so sys.stdin stays wired to the
+ # incoming pipe; heredoc consumes fd 0 and breaks the pipeline.
+ #
+ # "Multi" is a sentinel used by list-mode (-l) runs where there is no single
+ # in-scope root. In that case, accept any URL with an http/https scheme and
+ # no userinfo — the scope is enforced upstream by the list itself.
+ python3 -c '
+import sys
+from urllib.parse import urlsplit
+
+domain = sys.argv[1].rstrip(".").lower()
+multi_mode = domain == "multi"
+if not domain:
+ sys.exit(0)
+
+for raw in sys.stdin:
+ url = raw.strip()
+ if not url:
+ continue
+ try:
+ parsed = urlsplit(url)
+ except ValueError:
+ continue
+ if parsed.scheme not in ("http", "https"):
+ continue
+ if parsed.username is not None or parsed.password is not None:
+ continue
+ host = (parsed.hostname or "").rstrip(".").lower()
+ if not host:
+ continue
+ if multi_mode or host == domain or host.endswith("." + domain):
+ print(url)
+' "$domain"
+}
+
+# filter_in_scope_hosts()
+# Description: Reads hostnames (one per line) from stdin, writes only those
+# equal to $1 or a subdomain of $1 to stdout. Strips wildcard and trailing
+# dots. Unlike `grep -a "$domain"` this anchors the match so `exampleXcom.evil`
+# does NOT pass.
+# Arguments: $1 - in-scope domain
+function filter_in_scope_hosts() {
+ local domain="$1"
+ [[ -z "$domain" ]] && return 0
+
+ python3 -c '
+import sys
+
+domain = sys.argv[1].rstrip(".").lower()
+if not domain:
+ sys.exit(0)
+
+for raw in sys.stdin:
+ host = raw.strip().rstrip(".")
+ if not host:
+ continue
+ if host.startswith("*."):
+ host = host[2:]
+ host = host.lower()
+ if host == domain or host.endswith("." + domain):
+ print(host)
+' "$domain"
+}
+
+# is_in_scope_url()
+# Description: Single-URL wrapper around filter_in_scope_urls. Prints the URL
+# to stdout if it passes all scope checks (scheme, no userinfo, host match),
+# otherwise exits non-zero.
+# Arguments: $1 - URL, $2 - in-scope domain
+# Returns: 0 if in scope, 1 otherwise
+function is_in_scope_url() {
+ local url="$1"
+ local domain="$2"
+ local result
+ result=$(printf '%s\n' "$url" | filter_in_scope_urls "$domain")
+ [[ -n "$result" ]] || return 1
+ printf '%s\n' "$result"
+}
+
# is_empty()
# Description: Checks if a value is empty
# Arguments: $1 - Value to check
diff --git a/modules/axiom.sh b/modules/axiom.sh
index 0ac153e..15609a9 100644
--- a/modules/axiom.sh
+++ b/modules/axiom.sh
@@ -126,9 +126,18 @@ function axiom_launch() {
# Check to see if we have a fleet already, if so, SKIP THIS!
print_notice RUN "axiom_launch" "launching axiom fleet"
- NUMOFNODES=$(timeout 30 axiom-ls 2>>"$LOGFILE" | grep -c "$AXIOM_FLEET_NAME" || true)
+ # Count nodes whose name starts with our fleet prefix so we don't
+ # mis-count sibling fleets that happen to share this prefix (e.g.
+ # "reconftw" vs "reconftw-dev").
+ NUMOFNODES=$(timeout 30 axiom-ls 2>>"$LOGFILE" | grep -c "^${AXIOM_FLEET_NAME}" || true)
if [[ $NUMOFNODES -ge $AXIOM_FLEET_COUNT ]]; then
- axiom-select "$AXIOM_FLEET_NAME*" 2>>"$LOGFILE" >/dev/null
+ if ! axiom-select "$AXIOM_FLEET_NAME*" 2>>"$LOGFILE" >/dev/null; then
+ axiom_disable_runtime \
+ "axiom-select failed for existing fleet ${AXIOM_FLEET_NAME}; see ${LOGFILE}" \
+ "axiom-select"
+ end_func "Fleet select failed; see ${LOGFILE}" "${FUNCNAME[0]}" error
+ return 1
+ fi
end_func "" "${FUNCNAME[0]}"
else
if [[ $NUMOFNODES -eq 0 ]]; then
@@ -141,14 +150,29 @@ function axiom_launch() {
# Regions intentionally disabled here per original comment
# To re-enable: [[ -n ${AXIOM_FLEET_REGIONS-} ]] && AXIOM_FLEET_ARGS+=( --regions="${AXIOM_FLEET_REGIONS}" )
- # Show the exact command with proper quoting
- axiom-fleet2 "${AXIOM_FLEET_NAME}" "${AXIOM_FLEET_ARGS[@]}" >/dev/null 2>&1
- axiom-select "$AXIOM_FLEET_NAME*" 2>>"$LOGFILE" >/dev/null
+ # Route fleet2 output to $LOGFILE so failures (missing provider token,
+ # billing issues, quota errors) are visible rather than silently swallowed.
+ local fleet_rc=0
+ axiom-fleet2 "${AXIOM_FLEET_NAME}" "${AXIOM_FLEET_ARGS[@]}" >>"$LOGFILE" 2>&1 || fleet_rc=$?
+ if ! axiom-select "$AXIOM_FLEET_NAME*" 2>>"$LOGFILE" >/dev/null; then
+ axiom_disable_runtime \
+ "axiom-select failed after fleet launch (${AXIOM_FLEET_NAME}); see ${LOGFILE}" \
+ "axiom-select"
+ end_func "Fleet select failed after launch; see ${LOGFILE}" "${FUNCNAME[0]}" error
+ return 1
+ fi
if [[ -n $AXIOM_POST_START ]]; then
bash -lc "$AXIOM_POST_START" 2>>"$LOGFILE" >/dev/null
fi
- NUMOFNODES=$(timeout 30 axiom-ls 2>>"$LOGFILE" | grep -c "$AXIOM_FLEET_NAME" || true)
+ NUMOFNODES=$(timeout 30 axiom-ls 2>>"$LOGFILE" | grep -c "^${AXIOM_FLEET_NAME}" || true)
+ if [[ $fleet_rc -ne 0 ]] || [[ $NUMOFNODES -lt $AXIOM_FLEET_COUNT ]]; then
+ axiom_disable_runtime \
+ "fleet launch incomplete (${NUMOFNODES}/${AXIOM_FLEET_COUNT} nodes, rc=${fleet_rc}); see ${LOGFILE}" \
+ "axiom-fleet2"
+ end_func "Fleet launch failed; see ${LOGFILE}" "${FUNCNAME[0]}" error
+ return 1
+ fi
end_func "" "${FUNCNAME[0]}"
fi
fi
@@ -166,7 +190,7 @@ function axiom_shutdown() {
return
fi
axiom-rm -f "$AXIOM_FLEET_NAME*" 2>>"$LOGFILE" >/dev/null || true
- axiom-ls 2>>"$LOGFILE" | grep "$AXIOM_FLEET_NAME" >>"$LOGFILE" || true
+ axiom-ls 2>>"$LOGFILE" | grep "^${AXIOM_FLEET_NAME}" >>"$LOGFILE" || true
[[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && notification "Axiom fleet $AXIOM_FLEET_NAME shutdown" info
fi
}
@@ -174,7 +198,12 @@ function axiom_shutdown() {
function axiom_selected() {
start_func "${FUNCNAME[0]}" "Checking Axiom connectivity"
- if [[ ! $(axiom-ls 2>>"${LOGFILE:-/dev/null}" | tail -n +2 | sed '$ d' | wc -l) -gt 0 ]]; then
+ # Count nodes whose name starts with our fleet prefix. Avoids the fragile
+ # `tail -n +2 | sed '$ d'` header/footer stripping which under-counts by one
+ # when axiom-ls has no trailing footer line.
+ local running_nodes
+ running_nodes=$(axiom-ls 2>>"${LOGFILE:-/dev/null}" | grep -c "^${AXIOM_FLEET_NAME}" 2>/dev/null || echo 0)
+ if [[ "$running_nodes" -le 0 ]]; then
end_func "No axiom instances running" "${FUNCNAME[0]}" warn
axiom_disable_runtime "no axiom instances running" "axiom-ls"
return 0
diff --git a/modules/core.sh b/modules/core.sh
index bae5273..4d6b968 100644
--- a/modules/core.sh
+++ b/modules/core.sh
@@ -18,56 +18,110 @@ pt_header() { :; }
# If LOGFILE is unset or empty, send logs to /dev/null until later initialization
: "${LOGFILE:=/dev/null}"
-# List of environment variables containing secrets that should be redacted in logs
+# List of environment-variable names whose VALUES should be redacted from logs
+# and dry-run previews. Values resolved via ${!var} indirection from the
+# current shell scope.
REDACT_VARS=(
"SHODAN_API_KEY"
"WHOISXML_API"
"PDCP_API_KEY"
"GITHUB_TOKEN"
+ "GH_TOKEN"
"GITLAB_TOKEN"
"DISCORD_WEBHOOK_URL"
"SLACK_WEBHOOK_URL"
+ "SLACK_BOT_TOKEN"
+ "TELEGRAM_BOT_TOKEN"
"slack_auth"
+ "telegram_key"
+ "telegram_api_key"
+ "discord_url"
+ "discord_webhook_url"
"XSS_SERVER"
"COLLAB_SERVER"
)
+# Values (not names) registered at runtime from config files or CLI parsing.
+# Used when a secret lives in a local var that `${!var}` can't reach from
+# inside redact_secrets, e.g. telegram_key parsed inside sendToNotify().
+REGISTERED_SECRETS=()
+
+# register_secret()
+# Description: Adds a raw secret VALUE to REGISTERED_SECRETS so later redaction
+# passes will replace it with [REDACTED]. Safe to call repeatedly (deduped).
+# Arguments: $1 - secret value
+function register_secret() {
+ local value="$1"
+ [[ -z "$value" || ${#value} -le 4 ]] && return 0
+
+ local existing
+ for existing in "${REGISTERED_SECRETS[@]}"; do
+ [[ "$existing" == "$value" ]] && return 0
+ done
+ REGISTERED_SECRETS+=("$value")
+}
+
# redact_secrets()
-# Description: Redacts sensitive values from a string
+# Description: Redacts sensitive values from a string. Consults both the
+# REDACT_VARS list (env-var-name lookups) and REGISTERED_SECRETS (raw values
+# from config files).
# Arguments: $1 - String to redact
# Returns: Redacted string via stdout
function redact_secrets() {
local text="$1"
local redacted="$text"
-
+ local var value secret
+
for var in "${REDACT_VARS[@]}"; do
- local value="${!var:-}"
+ value="${!var:-}"
if [[ -n "$value" && ${#value} -gt 4 ]]; then
- # Replace the secret with [REDACTED]
redacted="${redacted//$value/[REDACTED]}"
fi
done
-
+
+ for secret in "${REGISTERED_SECRETS[@]}"; do
+ if [[ -n "$secret" && ${#secret} -gt 4 ]]; then
+ redacted="${redacted//$secret/[REDACTED]}"
+ fi
+ done
+
echo "$redacted"
}
+# _trace_redact_stream()
+# Description: Reads xtrace lines from stdin, runs each through redact_secrets()
+# and appends the scrubbed line to $LOGFILE. Used as the sink for
+# BASH_XTRACEFD so SHOW_COMMANDS=true never writes raw secrets to disk.
+_trace_redact_stream() {
+ local line
+ while IFS= read -r line; do
+ if declare -F redact_secrets >/dev/null 2>&1; then
+ line=$(redact_secrets "$line")
+ fi
+ printf '%s\n' "$line"
+ done >>"$LOGFILE"
+}
+
enable_command_trace() {
- # Enable bash xtrace to the current LOGFILE when SHOW_COMMANDS=true
- # WARNING: This may log sensitive data. Use redact_secrets() when reviewing logs.
+ # Enable bash xtrace to the current LOGFILE when SHOW_COMMANDS=true.
+ # xtrace output is routed through _trace_redact_stream so registered
+ # secrets (REDACT_VARS + REGISTERED_SECRETS) are replaced with [REDACTED]
+ # before the line reaches disk.
if [[ ${SHOW_COMMANDS:-false} != true ]]; then
return
fi
[[ -z ${LOGFILE:-} ]] && return
- _print_status WARN "Command tracing enabled" "Logs may contain sensitive data"
+ _print_status WARN "Command tracing enabled" "Logs routed through redact_secrets"
# Close any previous trace descriptor (native bash {varname} redirect syntax)
if [[ -n ${TRACE_FD:-} ]]; then
exec {TRACE_FD}>&- 2>/dev/null || true
fi
- # Open a new descriptor against the active log
- if ! exec {TRACE_FD}>>"$LOGFILE"; then
+ # Open a new descriptor against a redaction filter (process substitution).
+ # Each xtrace line passes through _trace_redact_stream before hitting LOGFILE.
+ if ! exec {TRACE_FD}> >(_trace_redact_stream); then
return
fi
export BASH_XTRACEFD=$TRACE_FD
@@ -1346,11 +1400,13 @@ function sendToNotify {
notification "Sending ${domain} data over Telegram" info
telegram_chat_id=$(sed -n '/^telegram:/,/^[^ ]/p' ${NOTIFY_CONFIG} | sed -n 's/^[ ]*telegram_chat_id:[ ]*"\([^"]*\)".*/\1/p')
telegram_key=$(sed -n '/^telegram:/,/^[^ ]/p' ${NOTIFY_CONFIG} | sed -n 's/^[ ]*telegram_api_key:[ ]*"\([^"]*\)".*/\1/p')
+ register_secret "$telegram_key"
run_command curl -F "chat_id=${telegram_chat_id}" -F "document=@${1}" https://api.telegram.org/bot${telegram_key}/sendDocument 2>>"$LOGFILE" >/dev/null
fi
if grep -q '^ discord\|^discord\|^ discord' $NOTIFY_CONFIG; then
notification "Sending ${domain} data over Discord" info
discord_url=$(sed -n '/^discord:/,/^[^ ]/p' ${NOTIFY_CONFIG} | sed -n 's/^[ ]*discord_webhook_url:[ ]*"\([^"]*\)".*/\1/p')
+ register_secret "$discord_url"
run_command curl -v -i -H "Accept: application/json" -H "Content-Type: multipart/form-data" -X POST -F 'payload_json={"username": "test", "content": "hello"}' -F file1=@${1} $discord_url 2>>"$LOGFILE" >/dev/null
fi
if [[ -n $slack_channel ]] && [[ -n $slack_auth ]]; then
diff --git a/modules/modes.sh b/modules/modes.sh
index 82f298c..0acede2 100644
--- a/modules/modes.sh
+++ b/modules/modes.sh
@@ -162,6 +162,127 @@ function start() {
}
+# Prepare the web-mode output dir for a clean run. Behavior differs between
+# `-l` (shared "Multi" dir, scope redefined per invocation) and `-d`
+# (per-domain dir, scope stable across invocations):
+#
+# The Multi dir is created only by the `-l -w` flow, so nothing in it predates
+# a `-w` invocation — the entire contents (other than cross-run tracking) can
+# be wiped to guarantee scope isolation between distinct lists.
+#
+# Per-domain dirs may have been populated by prior `-a`/`-s`/`-p` runs, so
+# non-web state must survive.
+#
+# Always wiped (pure web-pipeline artifacts that anew-accumulate):
+# webs/*, .tmp/*, js/*, fuzzing/*, gf/*, screenshots/*, assets.jsonl
+#
+# Additionally wiped in `-l` Multi mode (no prior non-web state to protect):
+# vulns/* — nuclei/cms/swagger/graphql findings
+# subdomains/* — subtakeover, sub_js_extract, tls_ip_pivots artifacts
+# hosts/* — grpc_reflection.txt and any hosts/ writer in webs_menu
+# osint/* — defensive wipe (no webs_menu writers today, but prevents
+# accidental leakage if a module grows an osint/ output path)
+#
+# Always preserved (cross-run tracking):
+# .log/*, .incremental/*, .called_fn/*, debug.log
+#
+# For `-d`: vulns/, subdomains/ (except subdomains.txt append), hosts/, osint/
+# are preserved so prior recon state survives. webs_menu modules using anew
+# will add new findings atop old ones (default incremental-but-untracked
+# behavior).
+#
+# subdomains/subdomains.txt handling:
+# -l Multi: OVERWRITE with sanitized hostnames from the list (scope reset)
+# -d <dom>: APPEND the target via anew so enumerated subs from prior
+# `-a`/`-s` remain as probe input
+#
+# Used by: reconftw.sh -w dispatch, monitor_mode 'w' case (every cycle).
+function prepare_web_mode_scope() {
+ if [[ -n $list ]]; then
+ # Defense-in-depth for the user's input file: treat $flist as strictly
+ # read-only. One initial `cp` to a tmp snapshot outside $dir; every
+ # subsequent operation uses the snapshot. The original is never written,
+ # deleted, or re-referenced — so even if -path matching has edge cases
+ # (trailing slash, symlinks, case folding) the original is untouched.
+ local _list_snapshot
+ if ! _list_snapshot=$(mktemp -t reconftw_web_list.XXXXXX); then
+ return 1
+ fi
+ if ! cp "$flist" "$_list_snapshot" 2>/dev/null; then
+ rm -f "$_list_snapshot" 2>/dev/null
+ return 1
+ fi
+
+ # Sanitize + stage OUTSIDE $dir first. Only wipe $dir after we know we
+ # have a non-empty, validated new scope — otherwise a fail-close path
+ # would destroy prior output dirs and leave the caller with no scope
+ # AND no prior state. `| sort -u > file` hides the while-loop rc, so
+ # we capture `$?` of the whole pipeline (which is `sort`'s + the
+ # redirection) and also check the staged file is non-empty.
+ local _subs_stage
+ if ! _subs_stage=$(mktemp -t reconftw_web_subs.XXXXXX); then
+ rm -f "$_list_snapshot" 2>/dev/null
+ return 1
+ fi
+ local _t _write_rc
+ while IFS= read -r _t <&3 || [[ -n "$_t" ]]; do
+ _t=${_t%$'\r'}
+ [[ -z "$_t" ]] && continue
+ _t=$(_sanitize_list_entry "$_t") || continue
+ printf '%s\n' "$_t"
+ done 3<"$_list_snapshot" | sort -u >"$_subs_stage"
+ _write_rc=$?
+
+ rm -f "$_list_snapshot" 2>/dev/null
+
+ if (( _write_rc != 0 )) || [[ ! -s "$_subs_stage" ]]; then
+ # Empty/failed staging → every entry rejected by sanitization, or
+ # the write failed. Abort WITHOUT wiping $dir so prior state
+ # survives the failed prep.
+ rm -f "$_subs_stage" 2>/dev/null
+ return 1
+ fi
+
+ # Validation passed → safe to wipe and install the new scope atomically.
+ # -l Multi: the dir is only ever created/populated by -l -w flow, so no
+ # non-web state needs protecting. Wipe every file under $dir except
+ # cross-run tracking and $flist (belt-and-suspenders in case the user's
+ # list path lives inside $dir).
+ find "$dir" -mindepth 1 -type f \
+ -not -path "$dir/.log/*" \
+ -not -path "$dir/.incremental/*" \
+ -not -path "$dir/.called_fn/*" \
+ -not -path "$dir/debug.log" \
+ -not -path "$flist" \
+ -delete 2>/dev/null || true
+
+ mkdir -p "$dir/subdomains" 2>/dev/null
+ if ! mv "$_subs_stage" "$dir/subdomains/subdomains.txt"; then
+ rm -f "$_subs_stage" 2>/dev/null
+ return 1
+ fi
+ return 0
+ else
+ # -d <domain>: the per-domain dir may hold prior -a/-s/-p recon state
+ # that must survive (hosts/ips.txt for virtualhosts, subdomains enum,
+ # vulns/ findings history, osint/, etc.). Wipe only pure web-pipeline
+ # artifacts that modules append via anew/>> and would otherwise mix
+ # prior-run data with this run's probe output.
+ local _p
+ for _p in webs .tmp js fuzzing gf screenshots; do
+ [[ -d "$dir/$_p" ]] && find "$dir/$_p" -mindepth 1 -type f -delete 2>/dev/null || true
+ done
+ : >"$dir/assets.jsonl" 2>/dev/null || true
+
+ mkdir -p "$dir/subdomains" 2>/dev/null
+ if ! printf '%s\n' "$domain" | anew -q "$dir/subdomains/subdomains.txt"; then
+ return 1
+ fi
+ [[ -s "$dir/subdomains/subdomains.txt" ]] || return 1
+ return 0
+ fi
+}
+
function end() {
if [[ $opt_ai ]]; then
@@ -1162,22 +1283,30 @@ function multi_custom() {
custom_function_list=$(echo "$custom_function" | tr ',' '\n')
func_total=$(echo "$custom_function_list" | wc -l)
+ # Iterate each target line through _sanitize_list_entry instead of loading
+ # the whole file into $domain. Loading the whole file exposed every sink
+ # that expands $domain unquoted (urlfinder, EmailHarvester) to CLI option
+ # injection and accidental word-splitting across multiple targets.
func_count=0
- domain=$(cat "$flist")
- for custom_f in $custom_function_list; do
- ((func_count = func_count + 1))
+ while IFS= read -r raw_domain || [[ -n "$raw_domain" ]]; do
+ [[ -z "$raw_domain" ]] && continue
+ domain=$(_sanitize_list_entry "$raw_domain") || continue
- loopstart=$(date +%s)
+ for custom_f in $custom_function_list; do
+ ((func_count = func_count + 1))
- run_module_with_axiom_failover "$custom_f"
+ loopstart=$(date +%s)
- currently=$(date +"%H:%M:%S")
- loopend=$(date +%s)
- local duration=$((loopend - loopstart))
- _print_status OK "$custom_f" "${duration}s"
- [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \
- print_notice INFO "$custom_f" "entries ${entries} (${func_count}/${func_total}) at ${currently}"
- done
+ run_module_with_axiom_failover "$custom_f"
+
+ currently=$(date +"%H:%M:%S")
+ loopend=$(date +%s)
+ local duration=$((loopend - loopstart))
+ _print_status OK "$custom_f" "${duration}s (${domain})"
+ [[ "${OUTPUT_VERBOSITY:-1}" -ge 2 ]] && \
+ print_notice INFO "$custom_f" "entries ${entries} (${func_count}/${func_total}) at ${currently}"
+ done
+ done <"$flist"
if [[ $AXIOM == true ]]; then
axiom_shutdown
@@ -1350,18 +1479,25 @@ function monitor_mode() {
all
;;
'w')
- if [[ -n $list ]]; then
- start
- if [[ $list == /* ]]; then
- cp "$list" "$dir/webs/webs.txt"
- else
- cp "${SCRIPTPATH}/$list" "$dir/webs/webs.txt"
- fi
- webs_menu
- else
+ if [[ -z $list ]]; then
notification "Web mode in monitor requires -l list file" error
return 1
fi
+ # Same scope-prep as the one-shot -w dispatch: wipe stale module
+ # output, preserve cross-run state (.log/.incremental/.called_fn/
+ # debug.log), and populate subdomains/subdomains.txt with sanitized
+ # hostnames so webprobe_full picks them up. DIFF=true is already
+ # set by monitor_mode above. If scope prep fails (mktemp / cp
+ # error) skip webs_menu so we don't probe a stale scope, but
+ # fall through to the normal post-case (monitor_snapshot,
+ # max-cycles check, sleep) — `continue` here would busy-loop and
+ # bypass MONITOR_MAX_CYCLES.
+ start
+ if prepare_web_mode_scope; then
+ webs_menu
+ else
+ notification "Web-scope preparation failed this cycle; skipping webs_menu until next interval" warn
+ fi
;;
'n')
PRESERVE=true
diff --git a/modules/osint.sh b/modules/osint.sh
index df1f8a7..9188c6e 100644
--- a/modules/osint.sh
+++ b/modules/osint.sh
@@ -77,6 +77,7 @@ function github_repos() {
fi
GH_TOKEN=$(head -n 1 "$GITHUB_TOKENS")
+ register_secret "$GH_TOKEN"
echo "$domain" | unfurl format %r >.tmp/company_name.txt
# Use temp file for token to avoid exposing it in process list
@@ -97,9 +98,27 @@ function github_repos() {
ensure_dirs .tmp/github_repos .tmp/github
if [[ -s ".tmp/company_repos_url.txt" ]]; then
- if ! run_command interlace -tL .tmp/company_repos_url.txt -threads "$INTERLACE_THREADS" -c "git clone _target_ .tmp/github_repos/_cleantarget_" 2>>"$LOGFILE" >/dev/null; then
- _print_error "interlace git clone command failed"
- return 1
+ # argv-safe parallel replacement for interlace -c. Each clone runs in a
+ # background subshell; failures are recorded via a marker file so the
+ # caller can log a summary after wait.
+ local _clone_fail_dir
+ _clone_fail_dir=$(mktemp -d)
+ while IFS= read -r _repo_target; do
+ [[ -z "$_repo_target" ]] && continue
+ (
+ _repo_safe=$(printf '%s' "$_repo_target" | sha256sum | cut -d' ' -f1)
+ if ! git clone "$_repo_target" ".tmp/github_repos/${_repo_safe}" 2>>"$LOGFILE" >/dev/null; then
+ : >"$_clone_fail_dir/$_repo_safe"
+ fi
+ ) &
+ _throttle_jobs "${INTERLACE_THREADS:-10}"
+ done <.tmp/company_repos_url.txt
+ wait
+ local _clone_failures
+ _clone_failures=$(find "$_clone_fail_dir" -type f 2>/dev/null | wc -l | tr -d ' ')
+ rm -rf "$_clone_fail_dir"
+ if [[ "$_clone_failures" -gt 0 ]]; then
+ _print_error "git clone failed for $_clone_failures repo(s); continuing with successful clones"
fi
else
log_note "No GitHub repos found for ${domain}; continuing" "${FUNCNAME[0]}" "${LINENO}"
@@ -137,11 +156,32 @@ function github_repos() {
end_func "Results are saved in $domain/osint/github_company_secrets.json" "${FUNCNAME[0]}" WARN
return 0
fi
- local np_scan_opts=""
- [[ "${SECRETS_SCAN_GIT_HISTORY:-false}" == "true" ]] && np_scan_opts="--git-history"
- if ! run_command interlace -tL .tmp/github_repos_folders.txt -threads "$INTERLACE_THREADS" -c "noseyparker scan ${np_scan_opts} --datastore .tmp/github/np_ds__cleantarget_ .tmp/github_repos/_target_ >/dev/null 2>>${LOGFILE}; noseyparker report --datastore .tmp/github/np_ds__cleantarget_ --format json > .tmp/github/nosey__cleantarget_.json" 2>>"$LOGFILE" >/dev/null; then
- _print_error "interlace noseyparker command failed"
- return 1
+ local -a np_scan_opts=()
+ [[ "${SECRETS_SCAN_GIT_HISTORY:-false}" == "true" ]] && np_scan_opts+=(--git-history)
+ # argv-safe parallel replacement for interlace -c.
+ local _np_fail_dir
+ _np_fail_dir=$(mktemp -d)
+ while IFS= read -r _np_target; do
+ [[ -z "$_np_target" ]] && continue
+ (
+ _np_safe=$(printf '%s' "$_np_target" | sha256sum | cut -d' ' -f1)
+ if ! noseyparker scan "${np_scan_opts[@]}" \
+ --datastore ".tmp/github/np_ds_${_np_safe}" \
+ ".tmp/github_repos/${_np_target}" >/dev/null 2>>"$LOGFILE"; then
+ : >"$_np_fail_dir/$_np_safe"
+ elif ! noseyparker report --datastore ".tmp/github/np_ds_${_np_safe}" --format json \
+ >".tmp/github/nosey_${_np_safe}.json" 2>>"$LOGFILE"; then
+ : >"$_np_fail_dir/$_np_safe"
+ fi
+ ) &
+ _throttle_jobs "${INTERLACE_THREADS:-10}"
+ done <.tmp/github_repos_folders.txt
+ wait
+ local _np_failures
+ _np_failures=$(find "$_np_fail_dir" -type f 2>/dev/null | wc -l | tr -d ' ')
+ rm -rf "$_np_fail_dir"
+ if [[ "$_np_failures" -gt 0 ]]; then
+ _print_error "noseyparker failed for $_np_failures repo(s); continuing"
fi
;;
titus|*)
@@ -151,21 +191,54 @@ function github_repos() {
end_func "Results are saved in $domain/osint/github_company_secrets.json" "${FUNCNAME[0]}" WARN
return 0
fi
- local titus_opts=""
- [[ "${SECRETS_SCAN_GIT_HISTORY:-false}" == "true" ]] && titus_opts="${titus_opts} --git"
- [[ "${SECRETS_VALIDATE:-false}" == "true" ]] && titus_opts="${titus_opts} --validate"
- if ! run_command interlace -tL .tmp/github_repos_folders.txt -threads "$INTERLACE_THREADS" -c "\"${titus_bin}\" scan --format json ${titus_opts} .tmp/github_repos/_target_ > .tmp/github/titus__cleantarget_.json" 2>>"$LOGFILE" >/dev/null; then
- _print_error "interlace titus command failed"
- return 1
+ local -a titus_opts=()
+ [[ "${SECRETS_SCAN_GIT_HISTORY:-false}" == "true" ]] && titus_opts+=(--git)
+ [[ "${SECRETS_VALIDATE:-false}" == "true" ]] && titus_opts+=(--validate)
+ # argv-safe parallel replacement for interlace -c.
+ local _titus_fail_dir
+ _titus_fail_dir=$(mktemp -d)
+ while IFS= read -r _titus_target; do
+ [[ -z "$_titus_target" ]] && continue
+ (
+ _titus_safe=$(printf '%s' "$_titus_target" | sha256sum | cut -d' ' -f1)
+ "$titus_bin" scan --format json "${titus_opts[@]}" \
+ ".tmp/github_repos/${_titus_target}" \
+ >".tmp/github/titus_${_titus_safe}.json" 2>>"$LOGFILE" \
+ || : >"$_titus_fail_dir/$_titus_safe"
+ ) &
+ _throttle_jobs "${INTERLACE_THREADS:-10}"
+ done <.tmp/github_repos_folders.txt
+ wait
+ local _titus_failures
+ _titus_failures=$(find "$_titus_fail_dir" -type f 2>/dev/null | wc -l | tr -d ' ')
+ rm -rf "$_titus_fail_dir"
+ if [[ "$_titus_failures" -gt 0 ]]; then
+ _print_error "titus scan failed for $_titus_failures repo(s); continuing"
fi
;;
esac
# Keep trufflehog enrichment (all engines).
if [[ -s ".tmp/company_repos_url.txt" ]]; then
- if ! run_command interlace -tL .tmp/company_repos_url.txt -threads "$INTERLACE_THREADS" -c "trufflehog git _target_ -j 2>&1 | jq -c > _output_/_cleantarget_" -o .tmp/github/ 2>>"$LOGFILE" >/dev/null; then
- _print_error "interlace trufflehog command failed"
- return 1
+ # argv-safe parallel replacement for interlace -c.
+ local _th_fail_dir
+ _th_fail_dir=$(mktemp -d)
+ while IFS= read -r _th_target; do
+ [[ -z "$_th_target" ]] && continue
+ (
+ _th_safe=$(printf '%s' "$_th_target" | sha256sum | cut -d' ' -f1)
+ if ! trufflehog git "$_th_target" -j 2>&1 | jq -c >".tmp/github/${_th_safe}" 2>>"$LOGFILE"; then
+ : >"$_th_fail_dir/$_th_safe"
+ fi
+ ) &
+ _throttle_jobs "${INTERLACE_THREADS:-10}"
+ done <.tmp/company_repos_url.txt
+ wait
+ local _th_failures
+ _th_failures=$(find "$_th_fail_dir" -type f 2>/dev/null | wc -l | tr -d ' ')
+ rm -rf "$_th_fail_dir"
+ if [[ "$_th_failures" -gt 0 ]]; then
+ _print_error "trufflehog failed for $_th_failures repo(s); continuing"
fi
fi
@@ -201,6 +274,7 @@ function github_leaks() {
if [[ -s $GITHUB_TOKENS ]]; then
GH_TOKEN=$(head -n 1 "$GITHUB_TOKENS")
+ register_secret "$GH_TOKEN"
local -a ghleaks_cmd=(
"${tools}/ghleaks/ghleaks"
-q "$domain"
@@ -468,7 +542,7 @@ function emails() {
start_func "${FUNCNAME[0]}" "Searching for emails/users/passwords leaks"
- run_command env PYTHONWARNINGS=ignore "${tools}/EmailHarvester/venv/bin/python3" "${tools}/EmailHarvester/EmailHarvester.py" -d ${domain} -e all -l 20 2>>"$LOGFILE" | anew -q .tmp/EmailHarvester.txt || true
+ run_command env PYTHONWARNINGS=ignore "${tools}/EmailHarvester/venv/bin/python3" "${tools}/EmailHarvester/EmailHarvester.py" -d "$domain" -e all -l 20 2>>"$LOGFILE" | anew -q .tmp/EmailHarvester.txt || true
# Process emailfinder results
if [[ -s ".tmp/EmailHarvester.txt" ]]; then
diff --git a/modules/subdomains.sh b/modules/subdomains.sh
index 9363eea..e17b2f7 100644
--- a/modules/subdomains.sh
+++ b/modules/subdomains.sh
@@ -1177,11 +1177,12 @@ function sub_scraping() {
subdomains_count=$(wc -l <"$dir/subdomains/subdomains.txt")
if [[ $subdomains_count -le $DEEP_LIMIT ]] || [[ $DEEP == true ]]; then
- run_command urlfinder -d $domain -all -o .tmp/url_extract_tmp.txt 2>>"$LOGFILE" >/dev/null
+ run_command urlfinder -d "$domain" -all -o .tmp/url_extract_tmp.txt 2>>"$LOGFILE" >/dev/null
if [[ -s ".tmp/url_extract_tmp.txt" ]]; then
- cat .tmp/url_extract_tmp.txt | grep -a "$domain" \
- | grep -aEo 'https?://[^ ]+' \
+ # Anchored scope filter (filter_in_scope_urls) replaces grep -a "$domain".
+ cat .tmp/url_extract_tmp.txt | grep -aEo 'https?://[^ ]+' \
+ | filter_in_scope_urls "$domain" \
| sed "s/^\*\.//" | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt || true
fi
@@ -1191,8 +1192,8 @@ function sub_scraping() {
log_note "sub_scraping: waymore failed or timed out; continuing" "${FUNCNAME[0]}" "${LINENO}"
fi
if [[ -s ".tmp/waymore_urls_subs.txt" ]]; then
- cat .tmp/waymore_urls_subs.txt | grep -a "$domain" \
- | grep -aEo 'https?://[^ ]+' \
+ cat .tmp/waymore_urls_subs.txt | grep -aEo 'https?://[^ ]+' \
+ | filter_in_scope_urls "$domain" \
| sed "s/^\*\.//" | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt || true
fi
else
@@ -1209,8 +1210,8 @@ function sub_scraping() {
if [[ -s ".tmp/web_full_info1.txt" ]]; then
cat .tmp/web_full_info1.txt | jq -r 'try .url' 2>/dev/null \
- | grep -a "$domain" \
| grep -aEo 'https?://[^ ]+' \
+ | filter_in_scope_urls "$domain" \
| sed "s/^\*\.//" \
| anew .tmp/probed_tmp_scrap.txt \
| unfurl -u domains 2>>"$LOGFILE" \
@@ -1218,7 +1219,8 @@ function sub_scraping() {
fi
if [[ -s ".tmp/probed_tmp_scrap.txt" ]]; then
- cat .tmp/probed_tmp_scrap.txt | run_command csprecon -s | grep -a "$domain" | sed "s/^\*\.//" | sort -u \
+ # csprecon output is hostnames, not URLs -> filter_in_scope_hosts
+ cat .tmp/probed_tmp_scrap.txt | run_command csprecon -s | sed "s/^\*\.//" | filter_in_scope_hosts "$domain" | sort -u \
| unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt || true
fi
@@ -1232,8 +1234,8 @@ function sub_scraping() {
if [[ -s ".tmp/web_full_info1.txt" ]]; then
cat .tmp/web_full_info1.txt | jq -r 'try .url' 2>/dev/null \
- | grep -a "$domain" \
| grep -aEo 'https?://[^ ]+' \
+ | filter_in_scope_urls "$domain" \
| sed "s/^\*\.//" \
| anew .tmp/probed_tmp_scrap.txt \
| unfurl -u domains 2>>"$LOGFILE" \
@@ -1241,7 +1243,7 @@ function sub_scraping() {
fi
if [[ -s ".tmp/probed_tmp_scrap.txt" ]]; then
- cat .tmp/probed_tmp_scrap.txt | run_command csprecon -s | grep -a "$domain" | sed "s/^\*\.//" | sort -u \
+ cat .tmp/probed_tmp_scrap.txt | run_command csprecon -s | sed "s/^\*\.//" | filter_in_scope_hosts "$domain" | sort -u \
| unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt || true
fi
@@ -1278,8 +1280,8 @@ function sub_scraping() {
if [[ -s ".tmp/web_full_info3.txt" ]]; then
cat .tmp/web_full_info3.txt | jq -r 'try .url' 2>/dev/null \
- | grep -a "$domain" \
| grep -aEo 'https?://[^ ]+' \
+ | filter_in_scope_urls "$domain" \
| sed "s/^\*\.//" \
| anew .tmp/probed_tmp_scrap.txt \
| unfurl -u domains 2>>"$LOGFILE" \
diff --git a/modules/utils.sh b/modules/utils.sh
index 1b0facd..b5be718 100644
--- a/modules/utils.sh
+++ b/modules/utils.sh
@@ -721,21 +721,53 @@ function run_with_adaptive_rate() {
####################################### SECURITY ################################################
###############################################################################################################
-# Sanitize domain input to prevent command injection
-# Usage: sanitize_domain <domain>
-# Returns: sanitized domain string (lowercase)
+# Sanitize domain input to prevent command injection.
+# Accepts bare domains, URLs (strips scheme/userinfo/path/query/fragment/port),
+# and IPv4 addresses (validated inline — octets > 255 rejected).
+# Usage: sanitize_domain <domain_or_url>
+# Returns: sanitized domain (lowercase) or IPv4
function sanitize_domain() {
local input_domain="$1"
+ local working="$input_domain"
- # Remove any characters that are not alphanumeric, dots, or hyphens
- # This prevents command injection via malicious domain names
- local sanitized
- sanitized=$(echo "$input_domain" | tr -cd 'a-zA-Z0-9.-')
+ # 1) Strip scheme (scheme://) — RFC 3986 scheme charset
+ if [[ "$working" =~ ^[a-zA-Z][a-zA-Z0-9+.-]*:// ]]; then
+ working="${working#*://}"
+ fi
- # Convert to lowercase for consistency
- sanitized=$(echo "$sanitized" | tr '[:upper:]' '[:lower:]')
+ # 2) Cut at first / ? # (path / query / fragment).
+ # Must happen BEFORE userinfo strip: an @ inside a path is not userinfo.
+ working="${working%%[/?#]*}"
+
+ # 3) Strip userinfo (user:pass@host)
+ if [[ "$working" == *@* ]]; then
+ working="${working##*@}"
+ fi
+
+ # 4) Strip :port (bare IPv6 literals are not supported elsewhere in reconFTW)
+ working="${working%%:*}"
+
+ # 5) IPv4 post-normalization: redirect to inline octet validation.
+ if [[ "$working" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+ local IFS_bak="$IFS"
+ IFS='.'
+ local -a _octs=($working)
+ IFS="$IFS_bak"
+ local _o
+ for _o in "${_octs[@]}"; do
+ if (( _o > 255 )); then
+ print_errorf "Invalid IP octet in '%s'" "$input_domain"
+ return 1
+ fi
+ done
+ echo "$working"
+ return 0
+ fi
- # Remove leading/trailing dots and hyphens
+ # 6) Hardening: char whitelist + lowercase + trim leading/trailing dots/hyphens
+ local sanitized
+ sanitized=$(echo "$working" | tr -cd 'a-zA-Z0-9.-')
+ sanitized=$(echo "$sanitized" | tr '[:upper:]' '[:lower:]')
sanitized=$(echo "$sanitized" | sed 's/^[.-]*//; s/[.-]*$//')
# Check if domain is empty after sanitization
@@ -751,8 +783,11 @@ function sanitize_domain() {
print_warnf "Domain '%s' has no TLD, may be invalid" "$sanitized" >&2
fi
- # If sanitization removed characters, warn user
- if [[ "$input_domain" != "$sanitized" ]]; then
+ # Log only when the tr whitelist actually removed characters beyond case/trim.
+ # URL normalization (scheme/userinfo/path/port strip) is silent — that's the expected path.
+ local working_trivial
+ working_trivial=$(echo "$working" | tr '[:upper:]' '[:lower:]' | sed 's/^[.-]*//; s/[.-]*$//')
+ if [[ "$working_trivial" != "$sanitized" ]]; then
print_notice INFO "sanitize_domain" "Domain sanitized from '${input_domain}' to '${sanitized}'" >&2
fi
@@ -944,8 +979,13 @@ function cached_download_typed() {
mkdir -p "$(dirname "$cache_file")"
# Download fresh copy
- printf "%b[%s] Downloading: %s%b\n" \
- "$bblue" "$(date +'%Y-%m-%d %H:%M:%S')" "$(basename "$url")" "$reset"
+ if _ui_human_output_enabled; then
+ printf "%b[%s] Downloading: %s%b\n" \
+ "$bblue" "$(date +'%Y-%m-%d %H:%M:%S')" "$(basename "$url")" "$reset"
+ elif [[ -n "${LOGFILE:-}" ]]; then
+ printf "[%s] Downloading: %s\n" \
+ "$(date +'%Y-%m-%d %H:%M:%S')" "$(basename "$url")" >>"$LOGFILE" 2>/dev/null || true
+ fi
curl_cmd=(curl -sL "$url" -o "$destination")
if [[ "$cache_type" == "resolvers" ]]; then
diff --git a/modules/vulns.sh b/modules/vulns.sh
index 1142693..c25b9ec 100644
--- a/modules/vulns.sh
+++ b/modules/vulns.sh
@@ -339,7 +339,6 @@ function lfi() {
_print_msg INFO "Running: LFI Fuzzing with FFUF"
local lfi_interlace_threads="${LFI_INTERLACE_THREADS:-4}"
- local lfi_interlace_timeout="${LFI_INTERLACE_TIMEOUT:-180}"
local lfi_ffuf_threads="${LFI_FFUF_THREADS:-20}"
local lfi_ffuf_rate="${LFI_FFUF_RATELIMIT:-50}"
local lfi_ffuf_timeout="${LFI_FFUF_TIMEOUT:-10}"
@@ -348,10 +347,37 @@ function lfi() {
local lfi_redirect_flag=""
[[ "$lfi_follow_redirects" == true ]] && lfi_redirect_flag="-r"
- # Use Interlace to parallelize FFUF scanning
- run_command interlace -tL ".tmp/tmp_lfi.txt" -threads "$lfi_interlace_threads" -timeout "$lfi_interlace_timeout" \
- -c "ffuf -v -noninteractive ${lfi_redirect_flag} -t ${lfi_ffuf_threads} -rate ${lfi_ffuf_rate} -timeout ${lfi_ffuf_timeout} -maxtime ${lfi_ffuf_maxtime} -H \"${HEADER}\" -w \"${lfi_wordlist}\" -u \"_target_\" -mr \"root:\" " 2>>"$LOGFILE" \
+ # argv-safe parallel replacement for interlace -c. Each ffuf runs in a
+ # background subshell, writing its output to a per-target temp file;
+ # outputs are merged once all workers complete. Concurrency capped at
+ # lfi_interlace_threads (same as the removed `interlace -threads`).
+ local -a _lfi_ffuf_args
+ _lfi_ffuf_args=(-v -noninteractive)
+ [[ -n "$lfi_redirect_flag" ]] && _lfi_ffuf_args+=("$lfi_redirect_flag")
+ _lfi_ffuf_args+=(
+ -t "$lfi_ffuf_threads"
+ -rate "$lfi_ffuf_rate"
+ -timeout "$lfi_ffuf_timeout"
+ -maxtime "$lfi_ffuf_maxtime"
+ -H "$HEADER"
+ -w "$lfi_wordlist"
+ -mr "root:"
+ )
+ local _lfi_parts
+ _lfi_parts=$(mktemp -d)
+ while IFS= read -r _lfi_target; do
+ [[ -z "$_lfi_target" ]] && continue
+ (
+ _lfi_hash=$(printf '%s' "$_lfi_target" | sha256sum | cut -d' ' -f1)
+ ffuf "${_lfi_ffuf_args[@]}" -u "$_lfi_target" \
+ >"$_lfi_parts/$_lfi_hash" 2>>"$LOGFILE" || true
+ ) &
+ _throttle_jobs "$lfi_interlace_threads"
+ done <".tmp/tmp_lfi.txt"
+ wait
+ cat "$_lfi_parts"/* 2>/dev/null \
| grep "URL" | sed 's/| URL | //' | anew -q "vulns/lfi.txt"
+ rm -rf "$_lfi_parts"
end_func "Results are saved in vulns/lfi.txt" "${FUNCNAME[0]}"
else
@@ -511,8 +537,20 @@ function sqli() {
if [[ $GHAURI == true ]]; then
_print_msg INFO "Running: Ghauri for SQLi Checks"
mkdir -p .tmp/ghauri_parts
- run_command interlace -tL ".tmp/tmp_sqli.txt" -threads "$INTERLACE_THREADS" -c "ghauri -u _target_ --batch -H \"${HEADER}\" --force-ssl >> .tmp/ghauri_parts/_cleantarget_.txt" 2>>"$LOGFILE" >/dev/null
- # Merge per-target outputs to avoid concurrent write corruption
+ # argv-safe parallel replacement for interlace -c. Each target runs
+ # ghauri in a background subshell writing to its own per-target file.
+ while IFS= read -r _sqli_target; do
+ [[ -z "$_sqli_target" ]] && continue
+ (
+ _sqli_safe=$(printf '%s' "$_sqli_target" | sha256sum | cut -d' ' -f1)
+ ghauri -u "$_sqli_target" --batch -H "$HEADER" --force-ssl \
+ >".tmp/ghauri_parts/${_sqli_safe}.txt" 2>>"$LOGFILE" || true
+ ) &
+ _throttle_jobs "${INTERLACE_THREADS:-10}"
+ done <".tmp/tmp_sqli.txt"
+ wait
+ # Merge per-target outputs (each process writes its own file, so
+ # there is no concurrent-write corruption).
cat .tmp/ghauri_parts/*.txt 2>/dev/null | anew -q vulns/ghauri_log.txt || true
rm -rf .tmp/ghauri_parts
fi
diff --git a/modules/web.sh b/modules/web.sh
index 849b09c..accab42 100644
--- a/modules/web.sh
+++ b/modules/web.sh
@@ -35,6 +35,9 @@ _validate_probe_jsonl() {
}
# Extract in-scope URLs from a JSONL probe file.
+# Uses filter_in_scope_urls (lib/validation.sh) to anchor the host match and
+# reject URLs with userinfo (`user@host`), which previously allowed off-scope
+# hosts like `http://[email protected]:8080/` to slip into webs_all.txt.
# Usage: _extract_probe_urls_from_json <input_file> <domain_filter> <output_file>
_extract_probe_urls_from_json() {
local input_file="$1"
@@ -44,7 +47,7 @@ _extract_probe_urls_from_json() {
[[ ! -s "$input_file" ]] && return 0
jq -r 'try (.url // empty)' "$input_file" 2>/dev/null \
- | awk -v dom="$dom_filter" 'index($0, dom) && $0 ~ /^https?:\/\// {print}' \
+ | filter_in_scope_urls "$dom_filter" \
| _normalize_probe_urls \
| sed '/^$/d' \
| anew_q_safe "$output_file"
@@ -59,9 +62,16 @@ _write_plain_webinfo() {
[[ ! -s "$json_file" ]] && return 0
- jq -r 'try . | "\(.url) [\(.status_code)] [\(.title)] [\(.webserver)] \(.tech)"' "$json_file" \
- | awk -v dom="$dom_filter" 'index($0, dom)' \
- | anew_q_safe "$output_file"
+ # "Multi" is the sentinel for list-mode runs (no single in-scope root);
+ # accept every line in that case. Otherwise require the domain substring.
+ if [[ "$dom_filter" == "Multi" ]]; then
+ jq -r 'try . | "\(.url) [\(.status_code)] [\(.title)] [\(.webserver)] \(.tech)"' "$json_file" \
+ | anew_q_safe "$output_file"
+ else
+ jq -r 'try . | "\(.url) [\(.status_code)] [\(.title)] [\(.webserver)] \(.tech)"' "$json_file" \
+ | awk -v dom="$dom_filter" 'index($0, dom)' \
+ | anew_q_safe "$output_file"
+ fi
}
# Send URLs to proxy if enabled
@@ -207,7 +217,7 @@ function webprobe_full() {
elif ((.url // "") | startswith("http://")) then "80"
else ""
end;
- select((.url // "") | contains($dom))
+ select($dom == "Multi" or ((.url // "") | contains($dom)))
| select(effective_port == "80" or effective_port == "443")
' "$probe_out" 2>>"$LOGFILE" >"$common_json_tmp"
@@ -219,7 +229,7 @@ function webprobe_full() {
elif ((.url // "") | startswith("http://")) then "80"
else ""
end;
- select((.url // "") | contains($dom))
+ select($dom == "Multi" or ((.url // "") | contains($dom)))
| select(effective_port != "80" and effective_port != "443")
' "$probe_out" 2>>"$LOGFILE" >"$uncommon_json_tmp"
fi
@@ -1654,9 +1664,15 @@ function swagger_check() {
cp webs/webs_all.txt "$brute_targets"
fi
- run_command interlace -tL "$brute_targets" -threads "$INTERLACE_THREADS" \
- -c "sj brute -u _target_ -qi -t ${SJ_TIMEOUT:-30} 2>/dev/null | grep -o 'Definition file found: .*' > _output_/_cleantarget_.txt" \
- -o .tmp/sj_brute 2>>"$LOGFILE" >/dev/null || true
+ # Wrap the whole interlace+sj pool in run_with_heartbeat_shell so the
+ # UI shows "Running: sj brute (swagger) | elapsed Xm Ys | ETA: --"
+ # every 20s — otherwise swagger_check appears to hang in -l/-z modes
+ # where sj brute can run for tens of minutes against 200 targets.
+ local _brute_targets_count
+ _brute_targets_count=$(wc -l <"$brute_targets" 2>/dev/null | tr -d ' ' || echo "?")
+ run_with_heartbeat_shell "sj brute (swagger, ${_brute_targets_count} targets)" \
+ "interlace -tL \"$brute_targets\" -threads \"$INTERLACE_THREADS\" -c \"sj brute -u _target_ -qi -t ${SJ_TIMEOUT:-30} 2>/dev/null | grep -o 'Definition file found: .*' > _output_/_cleantarget_.txt\" -o .tmp/sj_brute 2>>\"$LOGFILE\" >/dev/null" \
+ || true
find .tmp/sj_brute/ -type f -name "*.txt" -print0 2>/dev/null \
| xargs -0 grep -hoP 'Definition file found: \K\S+' 2>/dev/null \
@@ -1983,13 +1999,32 @@ function urlchecks() {
fi
if [[ -s ".tmp/url_extract_tmp.txt" ]]; then
- grep -a "$domain" .tmp/url_extract_tmp.txt | grep -aEo 'https?://[^ ]+' | grep -iE '\.js([?#].*)?$|\.js([/?&].*)' | anew -q .tmp/url_extract_js.txt || true
- grep -a "$domain" .tmp/url_extract_tmp.txt | grep -aEo 'https?://[^ ]+' | grep -iE '\.map([/?#].*)?$' | anew -q .tmp/url_extract_jsmap.txt || true
+ # Use filter_in_scope_urls (validation.sh) to anchor host match and
+ # reject userinfo/off-scope look-alikes. Replaces fragile grep -a "$domain".
+ grep -aEo 'https?://[^ ]+' .tmp/url_extract_tmp.txt | filter_in_scope_urls "$domain" | grep -iE '\.js([?#].*)?$|\.js([/?&].*)' | anew -q .tmp/url_extract_js.txt || true
+ grep -aEo 'https?://[^ ]+' .tmp/url_extract_tmp.txt | filter_in_scope_urls "$domain" | grep -iE '\.map([/?#].*)?$' | anew -q .tmp/url_extract_jsmap.txt || true
if [[ $DEEP == true ]] && [[ -s ".tmp/url_extract_js.txt" ]]; then
- run_command interlace -tL .tmp/url_extract_js.txt -threads 10 -c "${tools}/JSA/venv/bin/python3 ${tools}/JSA/jsa.py -f _target_ | anew -q .tmp/url_extract_tmp.txt" &>/dev/null
+ # argv-safe parallel replacement for interlace -c. Each target runs
+ # in a background subshell; outputs are written to per-target temp
+ # files (anew is not safe for concurrent writes) then merged once
+ # all workers finish.
+ local _jsa_parts
+ _jsa_parts=$(mktemp -d)
+ while IFS= read -r _jsa_target; do
+ [[ -z "$_jsa_target" ]] && continue
+ (
+ _jsa_hash=$(printf '%s' "$_jsa_target" | sha256sum | cut -d' ' -f1)
+ "${tools}/JSA/venv/bin/python3" "${tools}/JSA/jsa.py" -f "$_jsa_target" \
+ >"$_jsa_parts/$_jsa_hash" 2>/dev/null || true
+ ) &
+ _throttle_jobs "${INTERLACE_THREADS:-10}"
+ done <.tmp/url_extract_js.txt
+ wait
+ cat "$_jsa_parts"/* 2>/dev/null | anew -q .tmp/url_extract_tmp.txt
+ rm -rf "$_jsa_parts"
fi
- grep -a "$domain" .tmp/url_extract_tmp.txt | grep -aEo 'https?://[^ ]+' | grep "=" | qsreplace -a 2>>"$LOGFILE" | grep -aEiv "\.(eot|jpg|jpeg|gif|css|tif|tiff|png|ttf|otf|woff|woff2|ico|pdf|svg)$" | anew -q .tmp/url_extract_tmp2.txt || true
+ grep -aEo 'https?://[^ ]+' .tmp/url_extract_tmp.txt | filter_in_scope_urls "$domain" | grep "=" | qsreplace -a 2>>"$LOGFILE" | grep -aEiv "\.(eot|jpg|jpeg|gif|css|tif|tiff|png|ttf|otf|woff|woff2|ico|pdf|svg)$" | anew -q .tmp/url_extract_tmp2.txt || true
if [[ -s ".tmp/url_extract_tmp2.txt" ]]; then
urless <.tmp/url_extract_tmp2.txt | anew -q .tmp/url_extract_uddup.txt 2>>"$LOGFILE" >/dev/null || true
@@ -2254,15 +2289,30 @@ function jschecks() {
print_warnf "Failed to create sourcemapper directory."
fi
if [[ -s "js/js_livelinks.txt" ]]; then
- run_command interlace -tL js/js_livelinks.txt -threads "$INTERLACE_THREADS" \
- -c "sourcemapper -jsurl '_target_' -output _output_/_cleantarget_" \
- -o .tmp/sourcemapper 2>>"$LOGFILE" >/dev/null
+ # argv-safe parallel replacement for interlace -c. sourcemapper writes
+ # its own per-target output dir so no post-merge is needed.
+ while IFS= read -r _sm_target; do
+ [[ -z "$_sm_target" ]] && continue
+ (
+ _sm_safe=$(printf '%s' "$_sm_target" | sha256sum | cut -d' ' -f1)
+ sourcemapper -jsurl "$_sm_target" -output ".tmp/sourcemapper/${_sm_safe}" 2>>"$LOGFILE" >/dev/null || true
+ ) &
+ _throttle_jobs "${INTERLACE_THREADS:-10}"
+ done <js/js_livelinks.txt
+ wait
fi
if [[ -s ".tmp/url_extract_jsmap.txt" ]]; then
- run_command interlace -tL .tmp/url_extract_jsmap.txt -threads "$INTERLACE_THREADS" \
- -c "sourcemapper -url '_target_' -output _output_/_cleantarget_" \
- -o .tmp/sourcemapper 2>>"$LOGFILE" >/dev/null
+ # argv-safe parallel replacement for interlace -c.
+ while IFS= read -r _smu_target; do
+ [[ -z "$_smu_target" ]] && continue
+ (
+ _smu_safe=$(printf '%s' "$_smu_target" | sha256sum | cut -d' ' -f1)
+ sourcemapper -url "$_smu_target" -output ".tmp/sourcemapper/${_smu_safe}" 2>>"$LOGFILE" >/dev/null || true
+ ) &
+ _throttle_jobs "${INTERLACE_THREADS:-10}"
+ done <.tmp/url_extract_jsmap.txt
+ wait
fi
find .tmp/sourcemapper/ \( -name "*.js" -o -name "*.ts" \) -type f \
@@ -2307,8 +2357,22 @@ function jschecks() {
# shellcheck source=/dev/null
source "${GETJSWORDS_VENV}/bin/activate"
if python3 -c "import jsbeautifier, requests" 2>/dev/null; then
- run_command interlace -tL js/js_livelinks.txt -threads "$INTERLACE_THREADS" \
- -c "python3 ${tools}/getjswords.py '_target_' | anew -q webs/dict_words.txt" 2>>"$LOGFILE" >/dev/null
+ # argv-safe parallel replacement for interlace -c. Per-target
+ # output is written to a temp dir, then merged into dict_words.txt
+ # (anew is not concurrent-write safe).
+ _gjs_parts=$(mktemp -d)
+ while IFS= read -r _gjs_target; do
+ [[ -z "$_gjs_target" ]] && continue
+ (
+ _gjs_hash=$(printf '%s' "$_gjs_target" | sha256sum | cut -d' ' -f1)
+ python3 "${tools}/getjswords.py" "$_gjs_target" \
+ >"$_gjs_parts/$_gjs_hash" 2>>"$LOGFILE" || true
+ ) &
+ _throttle_jobs "${INTERLACE_THREADS:-10}"
+ done <js/js_livelinks.txt
+ wait
+ cat "$_gjs_parts"/* 2>/dev/null | anew -q webs/dict_words.txt
+ rm -rf "$_gjs_parts"
else
log_note "jschecks: jsbeautifier/requests missing in venv ${GETJSWORDS_VENV}; skipping getjswords wordlist step" "${FUNCNAME[0]}" "${LINENO}"
fi
@@ -2322,8 +2386,20 @@ function jschecks() {
if ! command -v "$getjs_py" >/dev/null 2>&1; then
log_note "getjswords python not found: ${getjs_py}; skipping wordlist step" "${FUNCNAME[0]}" "${LINENO}"
elif "$getjs_py" -c "import jsbeautifier, requests" 2>/dev/null; then
- run_command interlace -tL js/js_livelinks.txt -threads "$INTERLACE_THREADS" \
- -c "${getjs_py} ${tools}/getjswords.py '_target_' | anew -q webs/dict_words.txt" 2>>"$LOGFILE" >/dev/null
+ # argv-safe parallel replacement for interlace -c.
+ _gjs_parts=$(mktemp -d)
+ while IFS= read -r _gjs_target; do
+ [[ -z "$_gjs_target" ]] && continue
+ (
+ _gjs_hash=$(printf '%s' "$_gjs_target" | sha256sum | cut -d' ' -f1)
+ "$getjs_py" "${tools}/getjswords.py" "$_gjs_target" \
+ >"$_gjs_parts/$_gjs_hash" 2>>"$LOGFILE" || true
+ ) &
+ _throttle_jobs "${INTERLACE_THREADS:-10}"
+ done <js/js_livelinks.txt
+ wait
+ cat "$_gjs_parts"/* 2>/dev/null | anew -q webs/dict_words.txt
+ rm -rf "$_gjs_parts"
else
log_note "jschecks: jsbeautifier/requests missing in ${getjs_py}; skipping getjswords wordlist step" "${FUNCNAME[0]}" "${LINENO}"
fi
@@ -2449,6 +2525,10 @@ function well_known_pivots() {
head -n "$max_targets" webs/webs_all.txt | while IFS= read -r url; do
[[ -z "$url" ]] && continue
+ # Defense-in-depth: even if something bypasses _extract_probe_urls_from_json,
+ # re-validate each URL here and skip if it contains userinfo or is off-scope.
+ url=$(is_in_scope_url "$url" "$domain") || continue
+
# security.txt (RFC 9116)
local resp
for path in "/.well-known/security.txt" "/security.txt"; do
diff --git a/reconftw.sh b/reconftw.sh
index 5ce38a0..482b67d 100755
--- a/reconftw.sh
+++ b/reconftw.sh
@@ -770,17 +770,19 @@ case $opt_mode in
fi
;;
'w')
- if [[ -n $list ]]; then
- start
- if [[ $list == /* ]]; then
- cp "$list" "$dir/webs/webs.txt"
- else
- cp "${SCRIPTPATH}/$list" "$dir/webs/webs.txt"
- fi
- else
- _print_error "Web mode needs a website list file as target (./reconftw.sh -l target.txt -w)"
+ if [[ -z $list ]] && [[ -z $domain ]]; then
+ _print_error "Web mode needs a target: use -d <domain|url> or -l <file>"
exit $E_INVALID_INPUT
fi
+ # DIFF=true forces every web function to re-run; the shared output dir is
+ # reused across runs and without this the checkpoint from a prior -w would
+ # skip the probe leaving the (just-cleaned) output files empty.
+ export DIFF=true
+ start
+ if ! prepare_web_mode_scope; then
+ _print_error "Web mode scope preparation failed (tmpfile/cp error); aborting to avoid running against stale scope."
+ exit 1
+ fi
webs_menu
exit 0
;;
diff --git a/tests/security/test_redact_secrets.bats b/tests/security/test_redact_secrets.bats
new file mode 100644
index 0000000..eb3541d
--- /dev/null
+++ b/tests/security/test_redact_secrets.bats
@@ -0,0 +1,61 @@
+#!/usr/bin/env bats
+
+# Regression tests for redact_secrets() + register_secret() — Naxus finding #3
+# (dry-run/xtrace redaction of GH_TOKEN, telegram_key, discord_url).
+
+setup() {
+ local project_root
+ project_root="$(cd "$(dirname "$BATS_TEST_FILENAME")/../.." && pwd)"
+
+ # Extract only the redaction helpers from core.sh so the test does not
+ # need reconftw.cfg, logfile, or the full reconftw.sh bootstrap.
+ awk '/^REDACT_VARS=\(/,/^\)/ {print}
+ /^REGISTERED_SECRETS=\(\)$/ {print}
+ /^function register_secret\(\)/,/^}$/ {print}
+ /^function redact_secrets\(\)/,/^}$/ {print}' \
+ "$project_root/modules/core.sh" > "$BATS_TMPDIR/_redact.sh"
+ # shellcheck source=/dev/null
+ source "$BATS_TMPDIR/_redact.sh"
+}
+
+@test "redact_secrets replaces registered values with [REDACTED]" {
+ register_secret "ghp_supersecrettokenvalue_abc"
+ result=$(redact_secrets "ghleaks --token ghp_supersecrettokenvalue_abc")
+ [[ "$result" == *"[REDACTED]"* ]]
+ [[ "$result" != *"ghp_supersecrettokenvalue_abc"* ]]
+}
+
+@test "redact_secrets replaces env-var values via REDACT_VARS indirection" {
+ GH_TOKEN="ghp_envvar_aaaaaaaaaaaaaaaaaaaaaaaa"
+ result=$(redact_secrets "curl -H 'Authorization: token ghp_envvar_aaaaaaaaaaaaaaaaaaaaaaaa'")
+ [[ "$result" == *"[REDACTED]"* ]]
+ [[ "$result" != *"ghp_envvar_aaaaaaaaaaaaaaaaaaaaaaaa"* ]]
+}
+
+@test "redact_secrets redacts telegram_key from local scope via REDACT_VARS" {
+ telegram_key="bot123:AAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+ result=$(redact_secrets "https://api.telegram.org/botbot123:AAAAAAAAAAAAAAAAAAAAAAAAAAAA/sendMessage")
+ [[ "$result" == *"[REDACTED]"* ]]
+}
+
+@test "redact_secrets redacts discord_url" {
+ discord_url="https://discord.com/api/webhooks/1234567890/very-secret-token-xyz"
+ result=$(redact_secrets "curl $discord_url")
+ [[ "$result" == *"[REDACTED]"* ]]
+ [[ "$result" != *"very-secret-token-xyz"* ]]
+}
+
+@test "redact_secrets leaves short strings untouched" {
+ register_secret "x"
+ result=$(redact_secrets "benign short text x stays")
+ [[ "$result" == *"x stays"* ]]
+}
+
+@test "register_secret deduplicates repeated values" {
+ local before=${#REGISTERED_SECRETS[@]}
+ register_secret "some-repeated-value-1234"
+ register_secret "some-repeated-value-1234"
+ register_secret "some-repeated-value-1234"
+ local after=${#REGISTERED_SECRETS[@]}
+ [ "$((after - before))" -eq 1 ]
+}
diff --git a/tests/security/test_scope_validation.bats b/tests/security/test_scope_validation.bats
new file mode 100644
index 0000000..aadd47f
--- /dev/null
+++ b/tests/security/test_scope_validation.bats
@@ -0,0 +1,93 @@
+#!/usr/bin/env bats
+
+# Regression tests for the scope/SSRF helpers added to lib/validation.sh.
+# Covers Naxus audit findings #7 (substring bypass) and #8 (userinfo SSRF).
+
+setup() {
+ local project_root
+ project_root="$(cd "$(dirname "$BATS_TEST_FILENAME")/../.." && pwd)"
+ # shellcheck source=/dev/null
+ source "$project_root/lib/validation.sh"
+}
+
+@test "is_in_scope_host accepts the apex domain itself" {
+ run is_in_scope_host "example.com" "example.com"
+ [ "$status" -eq 0 ]
+}
+
+@test "is_in_scope_host accepts a subdomain" {
+ run is_in_scope_host "api.example.com" "example.com"
+ [ "$status" -eq 0 ]
+}
+
+@test "is_in_scope_host accepts a deep subdomain" {
+ run is_in_scope_host "foo.bar.example.com" "example.com"
+ [ "$status" -eq 0 ]
+}
+
+@test "is_in_scope_host is case-insensitive" {
+ run is_in_scope_host "SUB.Example.COM" "example.com"
+ [ "$status" -eq 0 ]
+}
+
+@test "is_in_scope_host rejects substring look-alike (exampleXcom.evil)" {
+ run is_in_scope_host "exampleXcom.evil" "example.com"
+ [ "$status" -ne 0 ]
+}
+
+@test "is_in_scope_host rejects prefix look-alike (badexample.com)" {
+ run is_in_scope_host "badexample.com" "example.com"
+ [ "$status" -ne 0 ]
+}
+
+@test "is_in_scope_host rejects unrelated domain" {
+ run is_in_scope_host "attacker.test" "example.com"
+ [ "$status" -ne 0 ]
+}
+
+@test "filter_in_scope_urls passes http and https targets on the apex" {
+ result=$(printf '%s\n' "http://example.com/" "https://example.com/x" | filter_in_scope_urls "example.com")
+ [[ "$result" == *"http://example.com/"* ]]
+ [[ "$result" == *"https://example.com/x"* ]]
+}
+
+@test "filter_in_scope_urls drops URLs with userinfo (SSRF bypass block)" {
+ result=$(printf '%s\n' "http://[email protected]:8080/" | filter_in_scope_urls "example.com")
+ [ -z "$result" ]
+}
+
+@test "filter_in_scope_urls drops URLs with user:pass@host" {
+ result=$(printf '%s\n' "http://user:[email protected]/" | filter_in_scope_urls "example.com")
+ [ -z "$result" ]
+}
+
+@test "filter_in_scope_urls drops off-scope with target as query (attacker.test/?cb=example.com)" {
+ result=$(printf '%s\n' "http://attacker.test/?cb=example.com" | filter_in_scope_urls "example.com")
+ [ -z "$result" ]
+}
+
+@test "filter_in_scope_urls drops non-http schemes" {
+ result=$(printf '%s\n' "ftp://example.com/" "file:///etc/passwd" | filter_in_scope_urls "example.com")
+ [ -z "$result" ]
+}
+
+@test "filter_in_scope_urls drops substring look-alike hosts" {
+ result=$(printf '%s\n' "https://exampleXcom.evil/" | filter_in_scope_urls "example.com")
+ [ -z "$result" ]
+}
+
+@test "filter_in_scope_hosts rejects substring look-alike on raw hostnames" {
+ result=$(printf '%s\n' "exampleXcom.evil" "api.example.com" | filter_in_scope_hosts "example.com")
+ [[ "$result" == "api.example.com" ]]
+}
+
+@test "is_in_scope_url returns the sanitized URL on match" {
+ run is_in_scope_url "https://api.example.com/path" "example.com"
+ [ "$status" -eq 0 ]
+ [[ "$output" == *"api.example.com"* ]]
+}
+
+@test "is_in_scope_url rejects userinfo" {
+ run is_in_scope_url "http://[email protected]:8080/" "example.com"
+ [ "$status" -ne 0 ]
+}
diff --git a/tests/unit/test_sanitize.bats b/tests/unit/test_sanitize.bats
index fd51cea..d94a3a6 100644
--- a/tests/unit/test_sanitize.bats
+++ b/tests/unit/test_sanitize.bats
@@ -71,6 +71,56 @@ setup() {
[ "$result" = "example.com" ]
}
+@test "sanitize_domain strips https:// scheme" {
+ result=$(sanitize_domain "https://example.com")
+ [ "$result" = "example.com" ]
+}
+
+@test "sanitize_domain strips http:// scheme with subdomain" {
+ result=$(sanitize_domain "http://sub.example.com")
+ [ "$result" = "sub.example.com" ]
+}
+
+@test "sanitize_domain strips path" {
+ result=$(sanitize_domain "https://example.com/path/to/x")
+ [ "$result" = "example.com" ]
+}
+
+@test "sanitize_domain strips port" {
+ result=$(sanitize_domain "https://example.com:8443")
+ [ "$result" = "example.com" ]
+}
+
+@test "sanitize_domain strips query and fragment" {
+ result=$(sanitize_domain "https://example.com?q=1#frag")
+ [ "$result" = "example.com" ]
+}
+
+@test "sanitize_domain strips userinfo" {
+ result=$(sanitize_domain "https://user:[email protected]/")
+ [ "$result" = "example.com" ]
+}
+
+@test "sanitize_domain handles @ in path (not userinfo)" {
+ result=$(sanitize_domain "https://example.com/path@x")
+ [ "$result" = "example.com" ]
+}
+
+@test "sanitize_domain rejects URL with invalid IP octet" {
+ run sanitize_domain "https://1.2.3.999/"
+ [ "$status" -ne 0 ]
+}
+
+@test "sanitize_domain accepts URL with valid IP" {
+ result=$(sanitize_domain "https://1.2.3.4:8080/")
+ [ "$result" = "1.2.3.4" ]
+}
+
+@test "sanitize_domain normalizes URL with all parts" {
+ result=$(sanitize_domain "https://user:[email protected]:8443/path?q=1#f")
+ [ "$result" = "sub.example.com" ]
+}
+
@test "sanitize_interlace_input removes shell metacharacters" {
local tmpfile
tmpfile=$(mktemp)