~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
test: enforce Brave Gitleaks regressions (#6075)
LearningCircuitbb64bde45716
diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.ymlindex 831a0f37d5..9e06b7180f 100644--- a/.github/workflows/docker-tests.yml+++ b/.github/workflows/docker-tests.yml@@ -220,6 +220,27 @@ jobs:run: |cd tests/infrastructure_tests && npm ci+ - name: Install Gitleaks for secret-rule regression tests+ # The Brave rule regression tests (tests/security/test_gitleaks_brave_rule.py)+ # run real-engine assertions inside the container below. This job backs+ # the required "All Pytest Tests + Coverage" check, so installing the+ # binary here — into the workspace, which is mounted at /app — makes+ # the regressions load-bearing for merge: with REQUIRE_GITLEAKS=1 a+ # missing binary fails the test instead of skipping it.+ env:+ # Keep this in sync with gitleaks-rule-tests.yml and gitleaks.yml.+ GITLEAKS_SHA256: 79a3ab579b53f71efd634f3aaf7e04a0fa0cf206b7ed434638d1547a2470a66e+ run: |+ GITLEAKS_ARCHIVE="${RUNNER_TEMP}/gitleaks_8.30.0_linux_x64.tar.gz"+ curl -sSfL --retry 3 --retry-all-errors \+ --connect-timeout 10 --max-time 120 \+ --output "${GITLEAKS_ARCHIVE}" \+ https://github.com/gitleaks/gitleaks/releases/download/v8.30.0/gitleaks_8.30.0_linux_x64.tar.gz+ echo "${GITLEAKS_SHA256} ${GITLEAKS_ARCHIVE}" | sha256sum --check --strict+ mkdir -p "${PWD}/.ci-gitleaks-bin"+ tar -xzf "${GITLEAKS_ARCHIVE}" -C "${PWD}/.ci-gitleaks-bin" gitleaks+ chmod +x "${PWD}/.ci-gitleaks-bin/gitleaks"+- name: Run all pytest tests with coverageid: main_pytest# Tests are always blocking - failures should not be silently ignored.@@ -258,6 +279,8 @@ jobs:-e PYTHONUNBUFFERED=1 \-e LDR_NEWS_SCHEDULER_ENABLED=false \-e PYTHONPATH=/app/src \+ -e REQUIRE_GITLEAKS=1 \+ -e GITLEAKS_PATH=/app/.ci-gitleaks-bin/gitleaks \-e OPENROUTER_API_KEY="${OPENROUTER_API_KEY}" \-w /app \${{ env.TEST_IMAGE }} \diff --git a/.github/workflows/gitleaks-rule-tests.yml b/.github/workflows/gitleaks-rule-tests.ymlnew file mode 100644index 0000000000..601c7ee67f--- /dev/null+++ b/.github/workflows/gitleaks-rule-tests.yml@@ -0,0 +1,76 @@+name: Gitleaks Rule Regressions++on:+ pull_request:+ branches:+ - main+ - refactor/fastapi-migration-phase1+ paths:+ - .gitleaks.toml+ - tests/security/test_gitleaks_*.py+ - .github/workflows/gitleaks-rule-tests.yml+ push:+ branches:+ - main+ - refactor/fastapi-migration-phase1+ paths:+ - .gitleaks.toml+ - tests/security/test_gitleaks_*.py+ - .github/workflows/gitleaks-rule-tests.yml+ workflow_dispatch:++permissions:+ contents: read++concurrency:+ group: gitleaks-rule-regressions-${{ github.event.pull_request.number || github.ref }}+ cancel-in-progress: true++jobs:+ gitleaks-rule-regressions:+ runs-on: ubuntu-latest+ timeout-minutes: 5++ steps:+ - name: Harden the runner+ uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0+ with:+ egress-policy: audit++ - name: Checkout repository+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false++ - name: Set up Python 3.12+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0+ with:+ python-version: '3.12'++ - name: Install focused test dependency+ run: python -m pip install pytest==9.1.1++ - name: Install Gitleaks 8.30.0+ env:+ # Keep this in sync with the repository's Gitleaks scan workflows.+ GITLEAKS_SHA256: 79a3ab579b53f71efd634f3aaf7e04a0fa0cf206b7ed434638d1547a2470a66e+ run: |+ GITLEAKS_ARCHIVE="${RUNNER_TEMP}/gitleaks_8.30.0_linux_x64.tar.gz"+ GITLEAKS_BIN_DIR="${RUNNER_TEMP}/gitleaks-bin"+ curl -sSfL --retry 3 --retry-all-errors \+ --connect-timeout 10 --max-time 120 \+ --output "${GITLEAKS_ARCHIVE}" \+ https://github.com/gitleaks/gitleaks/releases/download/v8.30.0/gitleaks_8.30.0_linux_x64.tar.gz+ echo "${GITLEAKS_SHA256} ${GITLEAKS_ARCHIVE}" | sha256sum --check --strict+ mkdir -p "${GITLEAKS_BIN_DIR}"+ tar -xzf "${GITLEAKS_ARCHIVE}" -C "${GITLEAKS_BIN_DIR}" gitleaks+ echo "${GITLEAKS_BIN_DIR}" >> "${GITHUB_PATH}"++ - name: Run real-engine rule regressions+ env:+ PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1"+ REQUIRE_GITLEAKS: "1"+ run: |+ python -m pytest --confcutdir=tests/security -q \+ tests/security/test_gitleaks_brave_rule.py \+ tests/security/test_gitleaks_credential_rule.pydiff --git a/.gitleaks.toml b/.gitleaks.tomlindex 314af9ad62..87e8dd9b00 100644--- a/.gitleaks.toml+++ b/.gitleaks.toml@@ -35,9 +35,11 @@ description = "Brave Search API key"id = "brave-api-key"# Require token boundaries so ordinary identifiers containing a 34-character# `BS...` substring (for example `NEWS_SUBSCRIPTION_...`) are not secrets.-# The key itself is capture group 1; Gitleaks reports only that group rather-# than the surrounding delimiter used to express boundaries in RE2.-regex = '''(?:^|[^a-zA-Z0-9_-])(BS[a-zA-Z0-9_-]{32})(?:$|[^a-zA-Z0-9_-])'''+# `:-` is an explicit left boundary for shell parameter defaults because `-`+# is otherwise part of the key alphabet. Multiline anchors avoid consuming+# the newline between standalone keys on consecutive lines.+# The key itself is capture group 1; Gitleaks reports only that group.+regex = '''(?m)(?:^|:-|[^a-zA-Z0-9_-])(BS[a-zA-Z0-9_-]{32})(?:$|[^a-zA-Z0-9_-])'''secretGroup = 1keywords = ["BS"]diff --git a/docs/ci/workflow-status.md b/docs/ci/workflow-status.mdindex d06bb47bb2..c7707eed6f 100644--- a/docs/ci/workflow-status.md+++ b/docs/ci/workflow-status.md@@ -29,7 +29,7 @@<!-- BEGIN GENERATED -->-**67 workflows:** 1 disabled · 3 stale · 1 manual-only · 62 active+**68 workflows:** 1 disabled · 3 stale · 1 manual-only · 63 active## ⚠ Disabled workflows@@ -41,8 +41,8 @@| Workflow | Cron | Last successful run ||---|---|---|-| `osv-scanner.yml` | `39 12 * * 1` | never || `puppeteer-e2e-tests.yml` | `0 2 * * 0` | never |+| `release-gate.yml` | `0 2 * * *` | never || `update-precommit-hooks.yml` | `0 8 * * 5` | never |## ℹ Manual-only by design@@ -104,11 +104,11 @@| `gitleaks.yml` | last 30 days | PR, manual, schedule(0 3 * * *) | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/gitleaks.yml?query=event%3Aschedule) || `ossf-scorecard.yml` | last 30 days | branch_protection_rule, schedule(0 8 * * 1), manual, push:main | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/ossf-scorecard.yml?query=event%3Aschedule) || `osv-scanner-scheduled.yml` | last 30 days | push:main, schedule(41 21 * * 1), manual | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/osv-scanner-scheduled.yml?query=event%3Aschedule) |-| `release-gate.yml` | last 30 days | workflow_call, manual, schedule(0 2 * * *) | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/release-gate.yml?query=event%3Aschedule) |+| `osv-scanner.yml` | last 30 days | PR, merge_group, schedule(39 12 * * 1), manual | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/osv-scanner.yml?query=event%3Aschedule) || `sbom.yml` | last 30 days | manual, schedule(0 10 * * 3), release | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/sbom.yml?query=event%3Aschedule) || `update-dependencies.yml` | last 30 days | workflow_call, manual, schedule(0 8 * * 3) | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/update-dependencies.yml?query=event%3Aschedule) || `update-npm-dependencies.yml` | last 30 days | workflow_call, manual, schedule(0 8 * * 4) | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/update-npm-dependencies.yml?query=event%3Aschedule) |-| `update-oldest-auto-merge-pr.yml` | never | schedule(17,47 * * * *), manual | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/update-oldest-auto-merge-pr.yml?query=event%3Aschedule) |+| `update-oldest-auto-merge-pr.yml` | last 30 days | schedule(17,47 * * * *), manual | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/update-oldest-auto-merge-pr.yml?query=event%3Aschedule) |## PR / push checks@@ -122,6 +122,7 @@| `dependency-review.yml` | last 30 days | PR, manual | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/dependency-review.yml) || `e2e-research-test.yml` | last 30 days | PR | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/e2e-research-test.yml) || `fastapi-migration-quality.yml` | last 30 days | PR, manual | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/fastapi-migration-quality.yml) |+| `gitleaks-rule-tests.yml` | last 30 days | PR, push:main,refactor/fastapi-migration-phase1, manual | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/gitleaks-rule-tests.yml) || `labels-sync.yml` | last 30 days | push:main, manual | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/labels-sync.yml) || `mcp-tests.yml` | last 30 days | push:main, PR, manual | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/mcp-tests.yml) || `pr-triage.yml` | last 30 days | PR, pull_request_review | [](https://github.com/LearningCircuit/local-deep-research/actions/workflows/pr-triage.yml) |diff --git a/tests/security/test_gitleaks_brave_rule.py b/tests/security/test_gitleaks_brave_rule.pyindex c7e305920e..4b2c416dbd 100644--- a/tests/security/test_gitleaks_brave_rule.py+++ b/tests/security/test_gitleaks_brave_rule.py@@ -54,8 +54,19 @@ def test_brave_rule_rejects_keys_embedded_in_larger_tokens(joining_character):assert _detected_secret(BRAVE_KEY + joining_character) is None+def test_brave_rule_detects_a_key_in_shell_default_syntax():+ shell_default = f'BRAVE_API_KEY="${{BRAVE_API_KEY:-{BRAVE_KEY}}}"'+ assert _detected_secret(shell_default) == BRAVE_KEY+++def test_brave_rule_detects_consecutive_standalone_keys():+ pattern = re.compile(_brave_rule()["regex"])+ matches = list(pattern.finditer(f"{BRAVE_KEY}\n{BRAVE_KEY}"))+ assert [match.group(1) for match in matches] == [BRAVE_KEY, BRAVE_KEY]++def test_brave_rule_with_gitleaks_cli(tmp_path):- gitleaks = shutil.which("gitleaks")+ gitleaks = os.environ.get("GITLEAKS_PATH") or shutil.which("gitleaks")if gitleaks is None:if os.environ.get("REQUIRE_GITLEAKS") == "1":pytest.fail("REQUIRE_GITLEAKS=1 but gitleaks is not installed")@@ -70,6 +81,9 @@ def test_brave_rule_with_gitleaks_cli(tmp_path):"NEWS_SUBSCRIPTION_MAX_QUESTIONS_PER_ITERATION","_" + BRAVE_KEY,BRAVE_KEY + "-suffix",+ f'BRAVE_API_KEY="${{BRAVE_API_KEY:-{BRAVE_KEY}}}"',+ BRAVE_KEY,+ BRAVE_KEY,]),encoding="utf-8",@@ -99,4 +113,4 @@ def test_brave_rule_with_gitleaks_cli(tmp_path):for finding in json.loads(report.read_text(encoding="utf-8"))if finding["RuleID"] == "brave-api-key"]- assert {finding["StartLine"] for finding in findings} == {1, 2}+ assert {finding["StartLine"] for finding in findings} == {1, 2, 6, 7, 8}diff --git a/tests/security/test_gitleaks_credential_rule.py b/tests/security/test_gitleaks_credential_rule.pyindex fb591b1029..af185ead22 100644--- a/tests/security/test_gitleaks_credential_rule.py+++ b/tests/security/test_gitleaks_credential_rule.py@@ -62,7 +62,7 @@ def test_credential_literal_rule_ignores_code_and_placeholders():def test_credential_literal_rule_with_gitleaks_cli(tmp_path):- gitleaks = shutil.which("gitleaks")+ gitleaks = os.environ.get("GITLEAKS_PATH") or shutil.which("gitleaks")if gitleaks is None:if os.environ.get("REQUIRE_GITLEAKS") == "1":pytest.fail("REQUIRE_GITLEAKS=1 but gitleaks is not installed")