Skip to content

The agent that grows with you

70.1 KB
Raw Blame
"""Tests for agent.redact -- secret masking in logs and output."""

import ast
import logging
import time

import pytest

from agent.redact import mask_secret, redact_cdp_url, redact_sensitive_text, RedactingFormatter


@pytest.fixture(autouse=True)
def _ensure_redaction_enabled(monkeypatch):
    """Ensure HERMES_REDACT_SECRETS is not disabled by prior test imports."""
    monkeypatch.delenv("HERMES_REDACT_SECRETS", raising=False)
    # Also patch the module-level snapshot so it reflects the cleared env var
    monkeypatch.setattr("agent.redact._REDACT_ENABLED", True)


class TestKnownPrefixes:

    def test_dotted_sk_and_prefixless_zhipu_keys_fully_masked_on_every_surface(self):
        """A key whose body carries dots must never leave a cleartext tail, and the
        prefix-less Zhipu ``id.secret`` shape must mask at all: on the terminal
        ``cat``/``grep`` path (code_file=True, the reporter's surface) and on the
        file-read path, where the mask must be the non-reusable sentinel."""
        from agent.redact import redact_terminal_output

        dotted_sk = "sk-sp-" + "ABCDEFGH1234567890" + "." + "abcdefgh1234567890_XYZ-0987654321"
        multi_dot_sk = "sk-ws-" + "H.EEPXREE.pFau.MEUCIQC6UnD-jj2a" + "ABCDEFGHIJKLMNOP"
        zhipu = "50aaed1234567890abcdef1234567890" + "." + "ZpSh99AbCdEfGh12"
        yaml = f"a:\n  api_key: {dotted_sk}\nb:\n  api_key: {zhipu}\nc:\n  api_key: {multi_dot_sk}\n"

        term = redact_terminal_output(yaml, "cat /tmp/keys.yaml")
        for secret_tail in ("abcdefgh1234567890_XYZ", "ZpSh99", "MEUCIQC6UnD", "EEPXREE"):
            assert secret_tail not in term, term
        # Display masks (``sk-pro...EFGH``) contain ``..``, which no real key does:
        # a second pass over an already-masked token must be a no-op, not ``***``
        # (tool_executor redacts browser_type args, then build_tool_preview
        # redacts them again).
        assert redact_sensitive_text("sk-pro...EFGH", force=True) == "sk-pro...EFGH"

        read = redact_sensitive_text(yaml, file_read=True)
        assert "«redacted:sk-…»" in read and "«redacted-secret»" in read, read
        for secret_tail in ("abcdefgh1234567890_XYZ", "ZpSh99", "MEUCIQC6UnD", "EEPXREE"):
            assert secret_tail not in read, read

    def test_dotted_and_prefixless_matchers_leave_benign_tokens_alone(self):
        """The Zhipu matcher is provider-shaped, not a generic dotted-token sweep:
        content-hash filenames (incl. ``<sha>.bundle`` / ``<md5>.sqlite3``), bare
        git shas, short ``sk-`` fragments and a 31-char id must stay byte-identical."""
        from agent.redact import redact_terminal_output

        benign = (
            "blob 0123456789abcdef0123456789abcdef.png\n"
            "git bundle create 0123456789abcdef0123456789abcdef01234567.bundle\n"
            "/cache/0123456789abcdef0123456789abcdef.sqlite3\n"
            "cp " + "a1" * 18 + ".example\n"
            "commit 0123456789abcdef0123456789abcdef01234567\n"
            "sk-short sk-abc.def\n"
            "release=" + "a" * 31 + ".ZpSh99AbCdEfGh12\n"
        )
        assert redact_terminal_output(benign, "git log --stat") == benign
        assert redact_sensitive_text(benign, file_read=True) == benign



    def test_gitlab_token_prefixes(self):
        """GitLab token families redact via their literal prefixes.

        Ported from openclaw/openclaw#112954; follow-up invited in #4541.
        """
        tokens = [
            # NOTE: every token is prefix + suffix CONCATENATION so no
            # contiguous token literal exists in this file — GitHub push
            # protection blocks realistic GitLab-token-shaped literals.
            "glpat-" + "Zx9AbCdEfGhIjKlMnOpQ",       # personal access token
            "gloas-" + "a" * 64,                     # OAuth application secret
            "gldt-" + "AbCdEfGhIjKlMnOpQrSt",        # deploy token
            "glrt-" + "t1_AbCdEfGhIjKlMnOpQrSt",     # runner auth token
            "glrt-" + "A" * 27 + ".01." + "a" * 9,   # routable (dotted) runner token
            "glrtr-" + "B" * 27 + ".01." + "b" * 9,  # routable runner registration
            "glcbt-" + "a1B2_AbCdEfGhIjKlMnOpQ",     # CI/CD job token
            "glptt-" + "c" * 40,                     # pipeline trigger token
            "glft-" + "AbCdEfGhIjKlMnOp",            # feed token
            "glimt-" + "AbCdEfGhIjKlMnOpQrStUvWxY",  # incoming mail token
            "glagent-" + "d" * 50,                   # agent (KAS) token
            "glsoat-" + "AbCdEfGhIjKlMnOpQrSt",      # service-account token
            "glffct-" + "AbCdEfGhIjKlMnOpQrSt",      # feature-flags client token
            "glwt-" + "AbCdEfGhIjKlMnOpQrSt",        # workspace token
            "GR1348941" + "E" * 20,                  # legacy runner registration
        ]
        for token in tokens:
            result = redact_sensitive_text(f"leaked {token} in output")
            secret_body = token.split("-", 1)[-1] if "-" in token else token[9:]
            assert secret_body not in result, f"{token!r} survived redaction: {result!r}"

    def test_gitlab_prefix_requires_word_boundary_and_length(self):
        """Prose and embedded identifiers must not false-positive."""
        for benign in [
            "the glossary explains gitlab tokens",   # no prefix at all
            "glpat-short",                            # suffix under 10 chars
            "myglpat-AbCdEfGhIjKlMnOpQrSt",           # embedded — lookbehind blocks
        ]:
            assert redact_sensitive_text(benign) == benign

    def test_context7_api_key_prefix(self):
        """Context7 keys carry a fixed vendor prefix (``ctx7sk-``) and must redact on every surface.

        The generic ``sk-`` family cannot catch them: the matcher's lookbehind
        rejects a prefix preceded by a word char, and here the ``sk-`` sits
        INSIDE ``ctx7sk-`` - so a bare Context7 key passed through unmasked
        (#142576).
        """
        # NOTE: prefix + suffix CONCATENATION so no contiguous token literal
        # exists in this file - GitHub push protection blocks realistic
        # vendor-token-shaped literals.
        token = ("ctx7sk-" + "0" * 8 + "-" + "0" * 4 + "-" + "0" * 4
                 + "-" + "0" * 4 + "-" + "0" * 12)
        for surface in (
            f"key is {token}\n",
            f'["{token}"]\n',
            f"headers:\n  CONTEXT7_API_KEY: {token}\nurl: https://mcp.context7.com/mcp\n",
            f"CONTEXT7_API_KEY={token}\n",
        ):
            assert token not in redact_sensitive_text(surface, force=True), surface

    def test_context7_prefix_requires_word_boundary_and_length(self):
        """Prose and embedded identifiers must not false-positive."""
        for benign in [
            "the glossary explains context7 keys",
            "ctx7sk-short",
            "myctx7sk-" + "A" * 20,
        ]:
            assert redact_sensitive_text(benign) == benign

    def test_agentmail_prefix_needs_a_key_shaped_hex_suffix(self):
        """``am_`` is a common identifier prefix; only the documented hex key body is a secret (#10983)."""
        for benign in ["schema.am_example_identifier_123", "path/to/am_monthly_report.sql"]:
            assert redact_sensitive_text(benign) == benign
        for key in ("am_" + "0123456789abcdef" * 2, "am_" + "Ab9" * 8, "am_org_" + "Zq7k" * 6):
            assert key[-12:] not in redact_sensitive_text(f"leaked {key} in output"), key

    def test_slack_token(self):
        token = "xoxb-" + "0" * 12 + "-" + "a" * 14
        result = redact_sensitive_text(token)
        assert "a" * 14 not in result





    def test_fireworks_keys(self):
        samples = [
            "fw-" + "A" * 40,
            "fw_" + "B" * 40,
            "fpk_" + "C" * 40,
        ]

        for token in samples:
            result = redact_sensitive_text(f"provider error {token}")
            assert token not in result
            assert "..." in result

    def test_short_fireworks_like_words_unchanged(self):
        text = "fw-tooshort fw_tooshort fpk_tooshort"
        assert redact_sensitive_text(text) == text




class TestEnvAssignments:
    def test_export_api_key(self):
        text = "export OPENAI_API_KEY=sk-proj-abc123def456ghi789jkl012"
        result = redact_sensitive_text(text)
        assert "OPENAI_API_KEY=" in result
        assert "abc123def456" not in result


    def test_non_secret_env_unchanged(self):
        text = "HOME=/home/user"
        result = redact_sensitive_text(text)
        assert result == text

    @pytest.mark.parametrize(
        "text",
        [
            'IDENTITY_TOKEN="bailu"',
            "--override-tensor per_layer_token_embd.weight=CPU",
            'runtime.token="local"',
            '{"token": "CPU"}',
            "token: CPU",
        ],
    )
    def test_ambiguous_key_preserves_obviously_noncredential_value(self, text):
        assert redact_sensitive_text(text, force=True) == text

    @pytest.mark.parametrize(
        "text, cleartext",
        [
            ("PASSWORD=hunter2", "hunter2"),
            ("SECRET_TOKEN=bailu", "bailu"),
            ("id_token=local", "local"),
            ("CUSTOM_TOKEN=opaqueValue123456789", "opaqueValue123456789"),
            ('{"token": "opaqueValue123456789"}', "opaqueValue123456789"),
            ('{"key_material": "CPU"}', "CPU"),
            ('{"bearer": "local"}', "local"),
            ("TOKEN=" + "sk-" + "a" * 30, "a" * 20),
        ],
    )
    def test_strong_key_or_credential_shaped_value_still_redacts(
        self, text, cleartext
    ):
        assert cleartext not in redact_sensitive_text(text, force=True)

    @pytest.mark.parametrize(
        "text, cleartext",
        [
            # AWS secret access keys are 40 chars of base64: ~1 in 64 begin with '/'.
            ("AWS_SECRET_ACCESS_KEY=/wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY",
             "wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY"),
            # argon2/bcrypt digests always begin with '$'.
            ("API_SECRET=$argon2id$v=19$m=65536,t=3,p=4$c29tZXNhbHQ$aGFzaHZhbHVl",
             "$argon2id$v=19$m=65536,t=3,p=4$c29tZXNhbHQ$aGFzaHZhbHVl"),
            ("SESSION_SECRET=/8f3kd9sKd0alsKDJ2mfkeisl3kdMc9dksla",
             "8f3kd9sKd0alsKDJ2mfkeisl3kdMc9dksla"),
            # A second '/' (~1 in 3 of the '/'-led AWS secrets) must not turn it into a "path".
            ("AWS_SECRET_ACCESS_KEY=/wJalrXUtnFEMIK7MDENG/bPxRfiCYEXAMPLEKEY",
             "wJalrXUtnFEMIK7MDENG/bPxRfiCYEXAMPLEKEY"),
            ("API_SECRET=~wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY",
             "wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY"),
        ],
    )
    def test_secret_that_only_starts_like_a_path_or_var_still_redacts(
        self, text, cleartext
    ):
        # The rc-readability exemption must fire only on a value that IS a complete
        # $VAR / ~/path / /abs/path reference — not on a secret that merely begins with
        # one of those characters. Before the exemption was anchored it returned ahead
        # of the strong-key and opaque-credential checks, leaking these verbatim.
        assert cleartext not in redact_sensitive_text(text, force=True)

    @pytest.mark.parametrize(
        "text",
        [
            "SSH_AUTH_SOCK=$HOME/.ssh/agent.sock",
            "SSH_AUTH_SOCK=$XDG_RUNTIME_DIR/ssh-agent.$USER.sock",
            "SSH_AUTH_SOCK=/run/user/$UID/keyring/ssh",
            "export SSH_AUTH_SOCK=$(gpgconf --list-dirs agent-ssh-socket)",
            "DOCKER_AUTH_CONFIG=/home/u/.docker",
            "DOCKER_AUTH_CONFIG=/home/u/.docker/MyProject2024Build.d/config.json",
            "MY_KEY_PATH=~/.ssh/id_rsa",
            "SECRET_DIR=/etc",
        ],
    )
    def test_real_path_and_var_references_stay_readable(self, text):
        assert redact_sensitive_text(text, force=True) == text




    def test_export_whitespace_preserved(self):
        # Regression: #4367 — whitespace before uppercase env var must be preserved
        text = "export SECRET_TOKEN=mypassword"
        result = redact_sensitive_text(text)
        assert result.startswith("export ")
        assert "SECRET_TOKEN=" in result
        assert "mypassword" not in result


class TestBareSecretEnvSuffixes:
    """Bare *_KEY / *_PASS / *_PW env suffixes mask, incl. lowercase — #77484."""

    def test_upper_suffix_keys_mask(self):
        for text in ("FAL_KEY=sk-abc123def456", "OPENAI_KEY=sk-abc123def456",
                     "MYSQL_PASS=ghi789", "DB_PW=jkl012"):
            result = redact_sensitive_text(text, force=True)
            assert "=" in result and result.split("=", 1)[1] != text.split("=", 1)[1]

    def test_lowercase_env_name_masks(self):
        # Opaque value with NO known prefix: only the env-name regex can
        # catch it — a sk-/ghp_ value would be masked by _PREFIX_RE on old
        # code too, making this test false-pass (review finding).
        result = redact_sensitive_text("openai_key=xyzzyplugh1234567890abcd", force=True)
        assert "xyzzyplugh1234567890abcd" not in result

    def test_prose_words_with_keyword_unchanged(self):
        # KEYBOARD / PASSAGE embed the bare keyword but are prose, not creds
        for text in ("KEYBOARD=notsecret", "PASSAGE=notsecret"):
            result = redact_sensitive_text(text, force=True)
            assert result == text

    def test_form_body_not_swallowed(self):
        # A bare `password=`/`token=` in a form body must not be eaten greedily
        text = "password=mysecret&username=bob&token=opaqueValue"
        result = redact_sensitive_text(text, force=True)
        assert "mysecret" not in result
        assert "opaqueValue" not in result
        assert "username=bob" in result


class TestControlCharSplitTokens:
    """Tokens split by control/zero-width chars must still mask — #77484."""

    def _assert_split_masked(self, text, tok):
        # Bare token (no KEY= context). Assert the LONGEST FRAGMENT is gone
        # from the result: the token is split in the input, so `tok` (whole,
        # contiguous) is absent from ANY output — even one that leaks every
        # fragment verbatim (review finding: whole-token assertion is a false
        # negative; the tail fragment appears contiguously in the leak).
        result = redact_sensitive_text(text, force=True)
        longest = max(tok[10:], tok[:10], key=len)
        assert longest not in result

    def test_newline_split_token_masks(self):
        tok = "ghp_abcdef1234567890ABCDEF1234567890abcdef"
        self._assert_split_masked(f"{tok[:10]}\n{tok[10:]}", tok)

    def test_esc_split_token_masks(self):
        tok = "ghp_abcdef1234567890ABCDEF1234567890abcdef"
        self._assert_split_masked(f"{tok[:10]}\x1b{tok[10:]}", tok)

    def test_zero_width_split_token_masks(self):
        tok = "ghp_abcdef1234567890ABCDEF1234567890abcdef"
        self._assert_split_masked(f"{tok[:10]}\u200b{tok[10:]}", tok)

    def test_complete_token_does_not_swallow_next_line(self):
        # A COMPLETE token at end-of-line followed by ordinary text must not
        # be joined across the newline — the ordinary prefix pass masks the
        # token; joining would swallow the adjacent line (browser
        # accessibility annotations regressed this way: "button [ref=e3]"
        # disappeared into the mask).
        tok = "ghp_" + "F" * 29
        text = f"text: Token: {tok}\nbutton [ref=e3]: Copy\n"
        result = redact_sensitive_text(text, force=True)
        assert "F" * 20 not in result
        assert "button" in result
        assert "ref=e3" in result

    def test_selfmatching_head_esc_split_tail_masked(self):
        # A split where the HEAD fragment alone already matches _PREFIX_RE
        # (>= 10 body chars) but the tail doesn't: the join must still run
        # for non-newline controls, or the tail leaks in cleartext. Only
        # LINE-crossing spans skip the join (see the annotation test).
        head = "sk-" + "a" * 15
        tail = "b" * 25
        result = redact_sensitive_text(head + "\x1b" + tail, force=True)
        assert tail not in result
        assert "a" * 12 not in result

    def test_env_dump_lines_not_joined(self):
        # Control-stripping must not join unrelated env lines into one match
        env_dump = (
            "HOME=/home/user\n"
            "ELEVENLABS_API_KEY=sk_abc123def456ghi789jkl\n"
            "EXA_API_KEY=exa_XY789abcdef01234\n"
            "SHELL=/bin/bash\n"
        )
        result = redact_sensitive_text(env_dump, force=True)
        assert "SHELL=/bin/bash" in result
        assert "HOME=/home/user" in result


class TestEnvLookupPreserved:
    """Programmatic env var lookups must not be corrupted (issue #2852)."""

    def test_os_getenv_single_quote_uppercase_key(self):
        text = "MY_API_KEY=os.getenv('OPENAI_API_KEY')"
        assert redact_sensitive_text(text, force=True) == text






    def test_real_env_value_still_redacted(self):
        text = "HOMEASSISTANT_TOKEN=eyJhbGciOiJIUzI1NiJ9.abc123.xyz"
        result = redact_sensitive_text(text, force=True)
        assert "eyJhbGciOiJIUzI1NiJ9" not in result


    def test_multiline_prose_with_code_snippet(self):
        text = """Set it up like this:
    HA_TOKEN=os.getenv('HOMEASSISTANT_TOKEN')
    if not HA_TOKEN:
        raise ValueError('Missing credentials')"""
        result = redact_sensitive_text(text, force=True)
        assert "os.getenv('HOMEASSISTANT_TOKEN')" in result







class TestJsonFields:
    def test_json_api_key(self):
        text = '{"apiKey": "sk-proj-abc123def456ghi789jkl012"}'
        result = redact_sensitive_text(text)
        assert "abc123def456" not in result


    def test_json_non_secret_unchanged(self):
        text = '{"name": "John", "model": "gpt-4"}'
        result = redact_sensitive_text(text)
        assert result == text

    @pytest.mark.parametrize("depth", [1, 2])
    def test_json_field_inside_json_encoded_string_is_masked_and_stays_valid(self, depth):
        # A tool result wrapping a config dump escapes its quotes; the plain JSON rule never saw the value.
        import json
        key = "AQ.Ab8RN6Jx2kq9Zs0VwT4yLm3PbQe7HcUfGdA1nX5oIr"
        text = json.dumps({"api_key": key, "token": "CPU", "model": "gemini"})
        for _ in range(depth):
            text = json.dumps({"output": text})
        result = redact_sensitive_text(text)
        assert key not in result and '\\"token\\": \\"CPU\\"'.replace("\\", "\\" * (2 ** depth - 1)) in result
        inner = result
        for _ in range(depth):
            inner = json.loads(inner)["output"]
        assert json.loads(inner)["model"] == "gemini"


class TestPythonReprFields:
    @pytest.mark.parametrize(
        "key",
        [
            "BRAVE_API_KEY",
            "SERVICE_TOKEN",
            "DB_PASSWORD",
            "api_key",
            "access_token",
            "client_secret",
            "id_token",
            "private_key",
        ],
    )
    def test_secret_field_in_nested_python_repr_is_redacted(self, key):
        secret = f"opaque-{key.lower()}-value-1234567890"
        text = f"kwargs={{'env': {{'{key}': '{secret}'}}}}"

        result = redact_sensitive_text(text, force=True)

        assert secret not in result
        assert f"'{key}': '***'" in result

    @pytest.mark.parametrize(
        "key",
        ["TOKEN_COUNT", "AUTH_METHOD", "PASSWORD_POLICY", "SECRET_NAME", "CREDENTIAL_TYPE"],
    )
    def test_uppercase_metadata_field_is_unchanged(self, key):
        text = f"{{'{key}': 'public-metadata-value'}}"
        assert redact_sensitive_text(text, force=True) == text

    @pytest.mark.parametrize(
        "key",
        ["UserPassword", "sessionToken", "clientApiKey", "gh_token", "webhookSecret"],
    )
    def test_mixed_case_credential_suffix_key_is_redacted(self, key):
        """Case-insensitive dict-entry class per OpenHands/software-agent-sdk#4508."""
        secret = f"opaque-{key.lower()}-value-1234567890"
        text = f"{{'{key}': '{secret}'}}"

        result = redact_sensitive_text(text, force=True)

        assert secret not in result
        assert f"'{key}': '***'" in result

    @pytest.mark.parametrize(
        "key",
        ["tokenizer", "secretary", "password_policy", "token_count", "keyring"],
    )
    def test_embedded_or_metadata_keyword_key_is_unchanged(self, key):
        text = f"{{'{key}': 'ordinary-value-1234567890'}}"
        assert redact_sensitive_text(text, force=True) == text

    @pytest.mark.parametrize(
        "value",
        [
            "apostrophe-near-head-'1234567890",
            "apostrophe-near-tail-1234567890'",
            b"bytes-apostrophe-near-head-'1234567890",
            b"bytes-apostrophe-near-tail-1234567890'",
        ],
    )
    def test_escaped_repr_value_remains_parseable(self, value):
        text = repr({"API_KEY": value})

        result = redact_sensitive_text(text, force=True)
        parsed = ast.literal_eval(result)

        assert parsed["API_KEY"] == (b"***" if isinstance(value, bytes) else "***")

    def test_double_quoted_repr_value_is_redacted(self):
        secret = "opaque-double-quoted-value-1234567890"
        text = f"payload={{'SERVICE_TOKEN': \"{secret}\"}}"

        result = redact_sensitive_text(text, force=True)

        assert secret not in result
        assert "'SERVICE_TOKEN': \"***\"" in result

    def test_programmatic_env_lookup_is_preserved(self):
        text = "{'API_KEY': \"os.getenv('OPENAI_API_KEY')\"}"
        assert redact_sensitive_text(text, force=True) == text

    def test_non_secret_python_repr_field_is_unchanged(self):
        text = "{'model': 'gpt-5', 'token_count': '123'}"
        assert redact_sensitive_text(text, force=True) == text

    def test_already_masked_repr_value_keeps_its_scheme_word(self):
        # An upstream scrub (MCP probe headers) leaves ``Digest ***``; the repr
        # pass must not collapse that to a bare ``***`` and lose the scheme.
        text = "headers={'Authorization': 'Digest ***'}"
        assert redact_sensitive_text(text, force=True) == text

    def test_code_file_preserves_secret_shaped_fixture(self):
        text = "CONFIG = {'BRAVE_API_KEY': 'fixture-value-1234567890'}"
        assert redact_sensitive_text(text, force=True, code_file=True) == text


class TestAuthHeaders:





    def test_authorization_prose_unchanged(self):
        # "authorization" without a colon-delimited value is plain prose.
        text = "the authorization model is fully open"
        assert redact_sensitive_text(text) == text

    def test_token_flush_against_double_quote_preserves_quote(self):
        # Regression for #43083: a token sitting flush against a closing
        # double quote must NOT pull that quote into the mask. Greedy \S+
        # used to eat it, turning value corruption into syntax corruption
        # (unterminated quote → shell EOF).
        text = 'curl -H "Authorization: Bearer sk-abcdef1234567890"'
        result = redact_sensitive_text(text)
        assert "sk-abcdef1234567890" not in result
        assert result.count('"') == 2, result  # both quotes survive
        assert result.endswith('"'), result



class TestApiKeyHeaders:
    def test_x_api_key_header_masked(self):
        text = "x-api-key: opaque-provider-key-1234567890"
        result = redact_sensitive_text(text)
        assert "x-api-key:" in result
        assert "opaque-provider-key" not in result

    def test_x_api_key_in_curl_command_masked(self):
        text = 'curl -H "x-api-key: sk-local-VERYsecret-999888" https://api.example.com'
        result = redact_sensitive_text(text)
        assert "VERYsecret" not in result
        assert "https://api.example.com" in result

    def test_api_key_header_masked(self):
        text = "api-key: anotherOpaqueSecret1234567"
        result = redact_sensitive_text(text)
        assert "anotherOpaqueSecret" not in result


class TestTelegramTokens:
    def test_bot_token(self):
        text = "bot123456789:ABCDEfghij-KLMNopqrst_UVWXyz12345"
        result = redact_sensitive_text(text)
        assert "ABCDEfghij" not in result
        assert "123456789:***" in result

    def test_raw_token(self):
        text = "12345678901:ABCDEfghijKLMNopqrstUVWXyz1234567890"
        result = redact_sensitive_text(text)
        assert "ABCDEfghij" not in result

    def test_long_digit_run_completes_fast(self):
        """A long digit run with no ``:<token>`` after it must stay linear.

        Without the run-start lookbehind, ``\\d{8,}`` retried from every digit
        of the run (quadratic while holding the GIL): a 2 MB Unity asset diff
        with a ``_typelessdata: 000...`` line pinned a core for hours inside
        terminal-output redaction. The colon is what routes the text through
        the Telegram pass.
        """
        import time

        text = "_typelessdata: " + "0" * 100_000 + "\n"
        t0 = time.perf_counter()
        assert redact_sensitive_text(text, force=True) == text
        assert time.perf_counter() - t0 < 2.0

    def test_token_after_long_digit_run_still_redacted(self):
        # The lookbehind must not change matching: a real token whose id
        # starts its own digit run is still masked, digits embedded in a
        # longer run are not an id.
        text = "0" * 1000 + " bot123456789:ABCDEfghij-KLMNopqrst_UVWXyz12345"
        result = redact_sensitive_text(text, force=True)
        assert "ABCDEfghij" not in result
        assert "123456789:***" in result


class TestPassthrough:
    def test_empty_string(self):
        assert redact_sensitive_text("") == ""



    def test_non_string_input_dict_coerced_and_redacted(self):
        result = redact_sensitive_text({"token": "sk-proj-abc123def456ghi789jkl012"})
        assert "abc123def456" not in result





class TestRedactingFormatter:
    def test_formats_and_redacts(self):
        formatter = RedactingFormatter("%(message)s")
        record = logging.LogRecord(
            name="test",
            level=logging.INFO,
            pathname="",
            lineno=0,
            msg="Key is sk-proj-abc123def456ghi789jkl012",
            args=(),
            exc_info=None,
        )
        result = formatter.format(record)
        assert "abc123def456" not in result
        assert "sk-pro" in result


class TestPrintenvSimulation:
    """Simulate what happens when the agent runs `env` or `printenv`."""

    def test_full_env_dump(self):
        env_dump = """HOME=/home/user
PATH=/usr/local/bin:/usr/bin
OPENAI_API_KEY=sk-proj-abc123def456ghi789jkl012mno345
OPENROUTER_API_KEY=sk-or-v1-reallyLongSecretKeyValue12345678
FIRECRAWL_API_KEY=fc-shortkey123456789012
TELEGRAM_BOT_TOKEN=bot987654321:ABCDEfghij-KLMNopqrst_UVWXyz12345
SHELL=/bin/bash
USER=teknium"""
        result = redact_sensitive_text(env_dump)
        # Secrets should be masked
        assert "abc123def456" not in result
        assert "reallyLongSecretKey" not in result
        assert "ABCDEfghij" not in result
        # Non-secrets should survive
        assert "HOME=/home/user" in result
        assert "SHELL=/bin/bash" in result
        assert "USER=teknium" in result


class TestSecretCapturePayloadRedaction:
    def test_secret_value_field_redacted(self):
        text = '{"success": true, "secret_value": "sk-test-secret-1234567890"}'
        result = redact_sensitive_text(text)
        assert "sk-test-secret-1234567890" not in result



class TestElevenLabsTavilyExaKeys:
    """Regression tests for ElevenLabs (sk_), Tavily (tvly-), and Exa (exa_) keys."""

    def test_elevenlabs_key_redacted(self):
        text = "ELEVENLABS_API_KEY=sk_abc123def456ghi789jklmnopqrstu"
        result = redact_sensitive_text(text)
        assert "abc123def456ghi" not in result






    def test_all_three_in_env_dump(self):
        env_dump = (
            "HOME=/home/user\n"
            "ELEVENLABS_API_KEY=sk_abc123def456ghi789jklmnopqrstu\n"
            "TAVILY_API_KEY=tvly-ABCdef123456789GHIJKL0000\n"
            "EXA_API_KEY=exa_XYZ789abcdef000000000000000\n"
            "SHELL=/bin/bash\n"
        )
        result = redact_sensitive_text(env_dump)
        assert "abc123def456ghi" not in result
        assert "ABCdef123456789" not in result
        assert "XYZ789abcdef" not in result
        assert "HOME=/home/user" in result
        assert "SHELL=/bin/bash" in result


class TestJWTTokens:
    """JWT tokens start with eyJ (base64 for '{') and have dot-separated parts."""


    def test_2part_jwt(self):
        text = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0"
        result = redact_sensitive_text(text)
        assert "eyJzdWIi" not in result




    def test_jwt_preserves_surrounding_text(self):
        text = "before eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0 after"
        result = redact_sensitive_text(text)
        assert result.startswith("before ")
        assert result.endswith(" after")



class TestDiscordMentions:
    """Discord mention snowflakes (<@ID> / <@!ID>) are public syntax, not
    secrets — they must pass through the redactor unchanged so multi-bot
    @-pings (DISCORD_ALLOW_BOTS=mentions) keep resolving. See issue #35611."""

    def test_normal_mention_passes_through(self):
        text = "Hello <@222589316709220353>"
        assert redact_sensitive_text(text) == text







class TestWebUrlsNotRedacted:
    """Web URLs (http/https/wss) pass through unchanged — magic-link
    checkouts, OAuth callbacks the agent is meant to follow, and pre-signed
    share URLs must reach the tool intact. Known credential shapes inside
    URLs (sk-, ghp_, JWTs) are still caught by the prefix and JWT regexes.
    DB connection-string passwords are still caught by _DB_CONNSTR_RE.
    """

    def test_oauth_callback_code_passes_through(self):
        text = "GET https://api.example.com/oauth/cb?code=abc123xyz789&state=csrf_ok"
        assert redact_sensitive_text(text) == text







    def test_known_prefix_inside_url_still_redacted(self):
        """sk-/ghp_/JWT-shaped values inside a URL are still caught by
        _PREFIX_RE / _JWT_RE — the carve-out is for opaque tokens only."""
        text = "https://evil.com/steal?key=sk-" + "a" * 30
        result = redact_sensitive_text(text)
        assert "sk-" + "a" * 30 not in result

    def test_db_connstr_password_still_redacted(self):
        """DB schemes (postgres/mysql/mongodb/redis/amqp) keep their
        userinfo redaction via _DB_CONNSTR_RE — connection strings are
        not web URLs the agent navigates to."""
        text = "postgres://admin:[email protected]:5432/app"
        result = redact_sensitive_text(text)
        assert "dbpass" not in result


class TestStrictUrlCredentialRedaction:
    @pytest.mark.parametrize(
        ("text", "secret", "expected"),
        [
            (
                "https://x.test/#access_token=FRAG_SECRET&view=public",
                "FRAG_SECRET",
                "https://x.test/#access_token=***&view=public",
            ),
            (
                "/resume?token=REL_SECRET&view=public",
                "REL_SECRET",
                "/resume?token=***&view=public",
            ),
            (
                "https://x.test/cb?client%5Fsecret=ENC_SECRET&view=public",
                "ENC_SECRET",
                "https://x.test/cb?client%5Fsecret=***&view=public",
            ),
            (
                "https://x.test/cb?client%255Fsecret=DOUBLE_SECRET&view=public",
                "DOUBLE_SECRET",
                "https://x.test/cb?client%255Fsecret=***&view=public",
            ),
            (
                "/resume?token=SEMICOLON_SECRET;view=public",
                "SEMICOLON_SECRET",
                "/resume?token=***;view=public",
            ),
            (
                "//user:[email protected]/path",
                "NET_SECRET",
                "//***:***@x.test/path",
            ),
            (
                "https://Zq8vT3kP9wLm2xR7nB4cY6fH1dJ5sA0e:@llm-proxy.example/v1",
                "Zq8vT3kP9wLm2xR7nB4cY6fH1dJ5sA0e",
                "https://***:***@llm-proxy.example/v1",
            ),
        ],
    )
    def test_masks_all_url_reference_forms_only_when_opted_in(
        self, text, secret, expected
    ):
        assert redact_sensitive_text(text) == text

        result = redact_sensitive_text(text, redact_url_credentials=True)

        assert secret not in result
        assert result == expected

    def test_similarly_named_public_params_remain_unchanged(self):
        text = "/metrics?token_count=17&session_id=public"
        assert redact_sensitive_text(text, redact_url_credentials=True) == text


class TestBareTokenUserinfoRedaction:
    """Regression tests for #6396 — a bare credential in URL userinfo
    (``scheme://TOKEN@host``, no ``user:pass`` colon) is redacted. This is the
    git-remote-with-embedded-password shape. The colon form ``user:pass@`` and
    query-string tokens are deliberately left to pass through (#34029) so
    magic-link / OAuth round-trip skills keep working — see
    TestWebUrlsNotRedacted for those invariants.
    """

    def test_git_remote_bare_password_redacted(self):
        """Exact bug scenario: password in a git remote URL."""
        text = (
            "git remote set-url origin "
            "https://[email protected]/unclehowell/FCUK.git"
        )
        result = redact_sensitive_text(text)
        assert "MYPASSWORDWASDISLAYEDHERE" not in result
        assert "@github.com" in result
        assert "unclehowell/FCUK.git" in result

    def test_ssh_bare_token_redacted(self):
        text = "ssh://[email protected]/project.git"
        result = redact_sensitive_text(text)
        assert "longtoken1234567" not in result
        assert "@gitlab.com" in result

    def test_ftp_bare_token_redacted(self):
        text = "ftp://[email protected]/files"
        result = redact_sensitive_text(text)
        assert "ftptoken123456" not in result


    def test_user_pass_form_still_passes_through(self):
        """The ``user:pass@`` colon form must NOT be redacted (#34029)."""
        text = "URL: https://user:[email protected]/path"
        assert redact_sensitive_text(text) == text

    def test_short_username_not_redacted(self):
        """Short userinfo (git, admin, deploy) below the 8-char floor passes."""
        for text in (
            "https://[email protected]/user/repo.git",
            "https://[email protected]/x",
            "https://[email protected]/y",
        ):
            assert redact_sensitive_text(text) == text

    def test_email_in_path_not_redacted(self):
        """An ``@`` in a path/query is not userinfo — the token class stops at
        ``/``, so emails after the first slash are never treated as a credential."""
        for text in (
            "https://example.com/[email protected]",
            "https://example.com/users/[email protected]/profile",
        ):
            assert redact_sensitive_text(text) == text




class TestFormBodyRedaction:
    """Form-urlencoded body redaction (k=v&k=v with no other text)."""

    def test_pure_form_body(self):
        text = "password=mysecret&username=bob&token=opaqueValue"
        result = redact_sensitive_text(text)
        assert "mysecret" not in result
        assert "opaqueValue" not in result
        assert "username=bob" in result


    def test_non_form_text_unchanged(self):
        """Sentences with `&` should NOT trigger form redaction."""
        text = "I have password=foo and other things"  # contains spaces
        result = redact_sensitive_text(text)
        # The space breaks the form regex; passthrough expected.
        assert "I have" in result

    def test_multiline_text_not_form(self):
        """Multi-line text is never treated as form body."""
        text = "first=1\nsecond=2"
        # Should pass through (still subject to other redactors)
        assert "first=1" in redact_sensitive_text(text)


class TestLowercaseDottedConfigKeys:
    """Issue #16413 — config-file passwords in lowercase/dotted/colon keys
    must be redacted. The uppercase _ENV_ASSIGN_RE missed these, leaking
    `spring.datasource.password=...` and `password: ...` from `cat`'d config
    files. Carve-outs: prose, code (#4367), and web URLs are left untouched.
    """







    def test_properties_file_dump(self):
        text = (
            "server.port=8080\n"
            "spring.datasource.username=admin\n"
            "spring.datasource.password=Sup3rS3cret!\n"
            "logging.level.root=INFO"
        )
        result = redact_sensitive_text(text)
        assert "Sup3rS3cret!" not in result
        assert "server.port=8080" in result  # non-secret keys preserved
        assert "username=admin" in result

    # --- carve-outs: must NOT redact ---

    def test_prose_mid_sentence_password_unchanged(self):
        # Not line-anchored, not dotted → conversational text, leave alone.
        text = "I have password=foo and other things"
        assert redact_sensitive_text(text) == text





class TestConfigKeyRedosResistance:
    """The dotted-key patterns must not backtrack exponentially (ReDoS).

    Before the possessive-quantifier rewrite, a non-matching run of ~40
    dotted segments took ~30ms and doubled every ~4 segments; 100 segments
    would effectively hang the redactor (it runs on every log line).
    """

    def test_long_dotted_run_completes_fast(self):
        import time

        # 100 dotted segments with no '=' — worst case for the old pattern.
        text = ".".join(["segment"] * 100) + " end"
        t0 = time.perf_counter()
        assert redact_sensitive_text(text) == text
        assert time.perf_counter() - t0 < 2.0

    def test_long_dotted_run_with_keyword_completes_fast(self):
        """Exercise _CFG_DOTTED_RE directly (bypasses the keyword pre-gate).

        The pre-gate skips the regex when no secret keyword is present, so
        test_long_dotted_run_completes_fast only guards the pre-gate.  This
        test includes a keyword but no '=' so the regex runs and must still
        complete quickly thanks to the possessive quantifiers.
        """
        import time

        text = ".".join(["segment"] * 100) + ".token end"
        t0 = time.perf_counter()
        assert redact_sensitive_text(text) == text
        assert time.perf_counter() - t0 < 2.0

    def test_long_dotted_secret_still_redacted(self):
        # Possessive quantifiers must not change matching behavior.
        text = ".".join(["seg"] * 50) + ".password=Sup3rS3cret!"
        result = redact_sensitive_text(text)
        assert "Sup3rS3cret!" not in result
        assert ".password=" in result

    def test_long_opaque_assignment_run_completes_fast(self):
        """Lowercase env scanning stays linear on compaction payload blobs."""
        import time

        # A serialized tool payload can contain a long opaque alphanumeric
        # value followed by '=' without containing a secret-key suffix.  The
        # old unanchored lowercase-env pattern retried its greedy prefix from
        # every byte, making this quadratic while holding the GIL.
        text = "a" * 20_000 + "=value"
        t0 = time.perf_counter()
        assert redact_sensitive_text(text, force=True) == text
        assert time.perf_counter() - t0 < 2.0

    def test_dotted_cfg_scan_stays_linear_with_keyword_elsewhere(self):
        """_CFG_DOTTED_RE must stay linear once the pre-gate passes.

        The ``_CFG_SECRET_WORD_RE`` pre-gate only skips secret-FREE text, so a
        payload that contains a real secret assignment AND a long opaque
        dotted run still reaches the backtrackable ``*`` prefix. Without the
        run-start lookbehind the sub retries that prefix from every byte of
        the run (quadratic while holding the GIL).
        """
        import time

        text = "password=hunter2\n" + "a." * 15_000 + "=value"
        t0 = time.perf_counter()
        result = redact_sensitive_text(text, force=True)
        assert "hunter2" not in result
        assert time.perf_counter() - t0 < 2.0

    def test_yaml_assign_redos_resistance(self):
        """_YAML_ASSIGN_RE must not backtrack excessively on long inputs."""
        import time

        # 100 lines of a long dotted key with a secret keyword but no
        # matching colon-value form — stresses the regex without matching.
        line = "a." * 50 + "token not_an_assignment"
        text = "\n".join([line] * 100)
        t0 = time.perf_counter()
        redact_sensitive_text(text)
        assert time.perf_counter() - t0 < 2.0

    def test_yaml_assign_secret_still_redacted(self):
        # Possessive quantifiers must not change YAML matching behavior.
        text = "spring.datasource.password: hunter2"
        result = redact_sensitive_text(text)
        assert "hunter2" not in result
        assert "password:" in result


class TestXaiToken:
    KEY = "xai-ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstu"

    def test_bare_token_masked(self):
        result = redact_sensitive_text(f"using key {self.KEY}", force=True)
        assert self.KEY not in result
        assert "xai-AB" in result


    def test_too_short_not_masked(self):
        short = "xai-tooshort"
        result = redact_sensitive_text(f"text {short} here", force=True)
        assert short in result




class TestDbConnstrCodeOutput:
    """Regression tests for issue #33801 — _DB_CONNSTR_RE corrupting code output.

    Two distinct flaws, both confined to displayed tool OUTPUT (read_file /
    terminal / execute_code), never the on-disk content:

    1. The password group ``[^@]+`` was greedy across newlines, so on a
       multi-line block it scanned past the DSN line to the next stray ``@``
       (e.g. a Python ``@decorator``), replacing everything in between with
       ``***`` — dropping lines and concatenating the next one.
    2. An f-string DSN template (``f"postgresql://{user}:{pass}@{host}"``) is
       not a live credential, but was redacted anyway. Under ``code_file=True``
       a pure ``{...}`` brace password is now preserved.
    """

    MULTILINE = (
        '            return f"postgresql://{auth}@{self.pg_host}:'
        '{self.pg_port}/{self.pg_database}"\n'
        "\n"
        '    @model_validator(mode="after")\n'
        '    def _validate_critical_settings(self) -> "Settings":'
    )





    def test_literal_connstr_still_redacted_with_code_file(self):
        """A real password in a literal DSN is still masked under code_file."""
        text = "postgresql://admin:[email protected]:5432/app"
        result = redact_sensitive_text(text, code_file=True, force=True)
        assert "realpassword" not in result
        assert "***" in result

    def test_literal_connstr_redacted_all_schemes(self):
        for scheme, secret in [
            ("postgres", "pgsecret1234"),
            ("mysql", "mysqlsecret99"),
            ("redis", "redissecret77"),
            ("mongodb+srv", "mongosecret55"),
            ("amqp", "amqpsecret33"),
        ]:
            text = f"{scheme}://user:{secret}@host:1234/db"
            result = redact_sensitive_text(text, code_file=True, force=True)
            assert secret not in result, scheme

    def test_literal_connstr_in_log_line_redacted(self):
        text = "connected via postgres://user:s3cr3tpw@host:5432/db ok"
        result = redact_sensitive_text(text, force=True)
        assert "s3cr3tpw" not in result


class TestTerminalOutputRedaction:
    """is_env_dump_command + redact_terminal_output — issue #43025.

    Terminal/process stdout must be redacted on every surface (foreground
    `terminal` AND background `process(poll/log/wait)`). Env-dump commands
    and commands that read ``.env`` files get the ENV-assignment pass so
    opaque tokens (no vendor prefix) are masked; other commands stay on
    the code_file path to avoid false positives.
    """

    def test_is_env_dump_command_detection(self):
        from agent.redact import is_env_dump_command
        assert is_env_dump_command("printenv")
        assert is_env_dump_command("env")
        assert is_env_dump_command("env | grep API")
        assert is_env_dump_command("set")
        assert is_env_dump_command("export")
        assert is_env_dump_command("declare -x")
        assert is_env_dump_command("cat /tmp/x && printenv")
        assert not is_env_dump_command("python app.py")
        assert not is_env_dump_command("cat config.py")
        assert not is_env_dump_command("printf 'TOKEN=x'")
        assert not is_env_dump_command("")
        assert not is_env_dump_command(None)

    # ── .env file detection (issue #61352 v2) ──

    def test_command_reads_secret_file_detection(self):
        from agent.redact import _command_reads_secret_file
        # Basic detection
        assert _command_reads_secret_file("cat .env")
        assert _command_reads_secret_file("cat .env.local")
        assert _command_reads_secret_file("cat .env.production")
        assert _command_reads_secret_file("cat .envrc")
        assert _command_reads_secret_file("head .env")
        assert _command_reads_secret_file("tail .env")
        assert _command_reads_secret_file("type .env")
        assert _command_reads_secret_file("nl .env")
        assert _command_reads_secret_file("bat .env")
        # With flags
        assert _command_reads_secret_file("cat -n .env")
        assert _command_reads_secret_file("cat -A .env")
        # With paths
        assert _command_reads_secret_file("cat ~/.hermes/.env")
        assert _command_reads_secret_file("cat /home/user/project/.env")
        assert _command_reads_secret_file("cat ./config/.env.local")
        # In a pipeline / sequence
        assert _command_reads_secret_file("cat .env | grep KEY")
        assert _command_reads_secret_file("echo '---' && cat .env")
        # Windows-style backslash paths
        assert _command_reads_secret_file("cat C:\\Users\\test\\.env")
        # Quoted paths (plain split leaves the quotes attached)
        assert _command_reads_secret_file('cat ".env"')
        assert _command_reads_secret_file("cat '.env'")
        # Case-insensitive basename (macOS/Windows filesystems)
        assert _command_reads_secret_file("cat .ENV")

    def test_command_reads_secret_file_excludes_templates(self):
        from agent.redact import _command_reads_secret_file
        # Templates/examples should NOT trigger
        assert not _command_reads_secret_file("cat .env.example")
        assert not _command_reads_secret_file("cat .env.sample")
        assert not _command_reads_secret_file("cat .env.template")
        assert not _command_reads_secret_file("cat .env.dist")

    def test_command_reads_secret_file_rejects_non_env_files(self):
        from agent.redact import _command_reads_secret_file
        assert not _command_reads_secret_file("cat config.py")
        assert not _command_reads_secret_file("cat README.md")
        assert not _command_reads_secret_file("cat .envrc.bak")  # .bak not in list
        assert not _command_reads_secret_file("python app.py")
        assert not _command_reads_secret_file("echo .env")  # echo is not a file-read cmd
        assert not _command_reads_secret_file("")
        assert not _command_reads_secret_file(None)

    def test_cat_env_file_masks_opaque_token(self):
        """cat .env → code_file=False → generic ENV pass redacts opaque keys."""
        from agent.redact import redact_terminal_output
        out = (
            "MISTRAL_API_KEY=abc123opaqueSecretValue\n"
            "NOUS_API_KEY=xyz789opaqueKey\n"
            "DEBUG=true\n"
        )
        red = redact_terminal_output(out, "cat .env")
        assert "abc123opaqueSecretValue" not in red
        assert "xyz789opaqueKey" not in red
        assert "DEBUG=true" in red  # non-secret key preserved

    def test_cat_env_file_with_flags_masks_opaque_token(self):
        """cat -n .env → still detected as .env read."""
        from agent.redact import redact_terminal_output
        out = "     1\tMISTRAL_API_KEY=abc123opaqueSecretValue\n"
        red = redact_terminal_output(out, "cat -n .env")
        assert "abc123opaqueSecretValue" not in red

    def test_cat_env_file_in_pipeline_masks_opaque_token(self):
        """cat .env | grep KEY → still detected as .env read."""
        from agent.redact import redact_terminal_output
        out = "MISTRAL_API_KEY=abc123opaqueSecretValue"
        red = redact_terminal_output(out, "cat .env | grep MISTRAL")
        assert "abc123opaqueSecretValue" not in red

    def test_cat_env_example_not_redacted_as_env(self):
        """cat .env.example → NOT treated as .env read (template file)."""
        from agent.redact import redact_terminal_output
        out = "MISTRAL_API_KEY=placeholder_value_here"
        red = redact_terminal_output(out, "cat .env.example")
        # Should NOT be redacted by the ENV-assignment pass (code_file=True).
        # The placeholder value should survive since it has no vendor prefix.
        assert "placeholder_value_here" in red

    def test_cat_env_local_masks_opaque_token(self):
        """cat .env.local → detected as .env read."""
        from agent.redact import redact_terminal_output
        out = "CUSTOM_API_KEY=opaquecustomkey123456"
        red = redact_terminal_output(out, "cat .env.local")
        assert "opaquecustomkey123456" not in red

    def test_cat_env_inline_comment_preserved(self):
        """Inline comments after env values are preserved (issue #61352 review)."""
        from agent.redact import redact_terminal_output
        out = "MISTRAL_API_KEY=abc123secret # used for tests"
        red = redact_terminal_output(out, "cat .env")
        assert "abc123secret" not in red
        assert "MISTRAL_API_KEY=*** # used for tests" in red

    def test_cat_env_export_inline_comment_preserved(self):
        """export KEY=VALUE # comment — comment preserved."""
        from agent.redact import redact_terminal_output
        out = "export MISTRAL_API_KEY=abc123secret # prod key"
        red = redact_terminal_output(out, "cat .env")
        assert "abc123secret" not in red
        assert "export MISTRAL_API_KEY=*** # prod key" in red

    @pytest.mark.parametrize(
        ("command", "output", "secret"),
        [
            ("cat ~/.hermes/config.yaml", "api_key: hermesConfigSecret123", "hermesConfigSecret123"),
            (
                "head ~/.hermes/profiles/work/config.yaml",
                "provider.token=profileConfigSecret456",
                "profileConfigSecret456",
            ),
            ("tail ~/.bashrc", "export SERVICE_TOKEN=bashRcSecret789", "bashRcSecret789"),
            ("grep TOKEN ~/.zshrc", "SERVICE_TOKEN=zshRcSecret123", "zshRcSecret123"),
            (
                "awk -F= '/TOKEN/ {print $2}' ~/.profile",
                "SERVICE_TOKEN=profileSecret456",
                "profileSecret456",
            ),
            ("sed -n '1,20p' ~/.zprofile", "api_key: zprofileSecret789", "zprofileSecret789"),
            (
                'cat "$HERMES_HOME/config.yaml"',
                "SERVICE_TOKEN=variablePathSecret123456789",
                "variablePathSecret123456789",
            ),
            (
                'cat "${HERMES_HOME}/config.yaml"',
                "SERVICE_TOKEN=variablePathSecret123456789",
                "variablePathSecret123456789",
            ),
            (
                "cat $HOME/.hermes/config.yaml",
                "SERVICE_TOKEN=homeVariablePathSecret123456",
                "homeVariablePathSecret123456",
            ),
            (
                "awk '{print $1; print $2}' ~/.bashrc",
                "export SERVICE_TOKEN=awkQuotedSecret123",
                "awkQuotedSecret123",
            ),
            (
                "grep 'foo|bar' ~/.hermes/config.yaml",
                "SERVICE_TOKEN=grepQuotedSecret456",
                "grepQuotedSecret456",
            ),
        ],
    )
    def test_secret_bearing_file_commands_mask_assignments(self, command, output, secret):
        from agent.redact import redact_terminal_output

        assert secret not in redact_terminal_output(output, command)

    @pytest.mark.parametrize(
        "command",
        [
            "cat config.yaml",
            "cat /project/config.yaml",
            "cat ~/.hermes/config.example.yaml",
            "cat ~/.hermes/config.template.yaml",
            "cat ~/.bashrc.example",
            'cat "$OTHER/config.yaml"',
            "grep TOKEN app.py",
            "grep .bashrc app.py",
            "grep -n .env src/settings.py",
            "awk '/TOKEN/' settings.yaml",
            "sed -n '1,20p' template.yaml",
        ],
    )
    def test_arbitrary_yaml_and_source_reads_stay_unredacted(self, command):
        """Only the known secret-bearing files flip the gate: the same opaque value read
        from project YAML / source code is left alone (code_file path), while the
        identical text under ``cat .env`` is masked."""
        from agent.redact import redact_terminal_output

        output = "SERVICE_TOKEN=3JcQ1UzX9vQ2mL7pR4tY8wA1sD5fG6hJ2kSbn7Q0"
        assert redact_terminal_output(output, command) == output
        assert "3JcQ1UzX9vQ2mL7pR4tY8wA1sD5fG6hJ2kSbn7Q0" not in redact_terminal_output(output, "cat .env")


    def test_disabled_passes_through(self, monkeypatch):
        from agent.redact import redact_terminal_output
        monkeypatch.setattr("agent.redact._REDACT_ENABLED", False)
        out = "CUSTOM_TOKEN=zzzopaque1234567890abcdef"
        red = redact_terminal_output(out, "printenv")
        assert "zzzopaque1234567890abcdef" in red

    @pytest.mark.parametrize(
        "command",
        [
            "python -m pytest",
            "uv run pytest",
            "poetry run pytest",
            "coverage run -m pytest",
            "PYTHONPATH=. pytest",
            "/usr/bin/env pytest",
            "bash -lc 'pytest tests/agent/test_redact.py'",
        ],
    )
    def test_pytest_diagnostic_masks_python_repr_secret_field(self, command):
        from agent.redact import redact_terminal_output

        secret = "opaque-brave-value-1234567890"
        out = f"E       kwargs={{'env': {{'BRAVE_API_KEY': '{secret}'}}}}"

        red = redact_terminal_output(out, command, force=True)

        assert secret not in red
        assert "'BRAVE_API_KEY': '***'" in red

    def test_exception_line_masks_python_repr_secret_field(self):
        from agent.redact import redact_terminal_output

        secret = "opaque-exception-value-1234567890"
        out = f"RuntimeError: payload={{'API_KEY': '{secret}'}}"

        red = redact_terminal_output(out, "python app.py", force=True)

        assert secret not in red
        assert "'API_KEY': '***'" in red

    def test_source_dump_preserves_python_repr_fixture(self):
        from agent.redact import redact_terminal_output

        out = "CONFIG = {'BRAVE_API_KEY': 'fixture-value-1234567890'}"
        assert redact_terminal_output(out, "cat config.py", force=True) == out
        assert redact_terminal_output(out, "python dump_source.py", force=True) == out

    def test_pytest_source_line_preserves_python_repr_fixture(self):
        from agent.redact import redact_terminal_output

        source = "    CONFIG = {'BRAVE_API_KEY': 'fixture-value-1234567890'}"
        out = source + "\nE       assert False"

        red = redact_terminal_output(out, "uv run pytest", force=True)

        assert source in red

    def test_disabled_pytest_diagnostic_passes_through(self, monkeypatch):
        from agent.redact import redact_terminal_output

        monkeypatch.setattr("agent.redact._REDACT_ENABLED", False)
        secret = "opaque-disabled-value-1234567890"
        out = f"E       payload={{'API_KEY': '{secret}'}}"

        assert redact_terminal_output(out, "uv run pytest") == out


class TestFileReadNonReusableRedaction:
    """#35519: prefix-matched credentials in FILE CONTENT (read_file /
    search_files / cat) must be redacted to a NON-REUSABLE sentinel — not a
    head/tail mask that looks like a real-but-truncated key and gets written
    back to config (corrupting the credential -> 401)."""

    GHP = "ghp_S1abcdefghijklmnopqrstuvwxyz0Pn2T"  # realistic GitHub PAT shape
    SK = "sk-proj-abcdefghijklmnopqrstuvwxyz0123456789"


    def test_file_read_does_not_leak_secret_body(self):
        """Crucial: file_read must NOT expose the real key (no un-redact)."""
        out = redact_sensitive_text(f"token: {self.GHP}", force=True, file_read=True)
        # No run of the secret body survives.
        assert "S1abcdefghij" not in out
        assert self.GHP not in out
        assert "Pn2T" not in out  # not even the tail (the old mask kept it)

    def test_file_read_sentinel_is_not_a_plausible_key(self):
        """The sentinel can't be mistaken for / written back as a usable key:
        the old mask was a 13-char `ghp_S1...Pn2T` that broke GitHub auth when
        an agent re-saved it. The sentinel is syntactically invalid as a token
        (contains « » … and ':'), so it can't round-trip into a dead key."""
        out = redact_sensitive_text(f"GITHUB_PERSONAL_ACCESS_TOKEN: {self.GHP}",
                                    force=True, file_read=True)
        masked = out.split(": ", 1)[1].strip()
        # Not a bare token: contains the sentinel delimiters.
        assert masked.startswith("«") and masked.endswith("»")
        assert "…" in masked





class TestSecretFileAssignmentRedaction:
    """#110567: ``file_read=True`` used to imply ``code_file=True`` and skip the assignment passes,
    so an opaque prefix-less credential in a secret-bearing file reached the model in cleartext
    while the terminal read of the same file masked it. ``secret_file=True`` re-enables the passes
    with the non-reusable sentinel (#35519); unclassified reads are byte-identical to before."""

    SYNTH = "3JcQ1UqZ8mNp4Rt6vWx2Yb9Ad0Ef7Gh5Ij2kS"  # 40-char opaque, no vendor prefix

    @pytest.mark.parametrize("template, sentinel", [
        ("ADS_API_TOKEN: {tok}", "«redacted-secret»"),          # YAML
        ("export FOO_TOKEN='{tok}'", "«redacted-secret»"),      # shell rc
        ("FOO_API_KEY={tok}", "«redacted-secret»"),             # dotenv
        ('{{"api_key": "{tok}"}}', "«redacted-secret»"),        # JSON
        ("5|      ADS_API_TOKEN: {tok}", "«redacted-secret»"),  # read_file line gutter
        ("108:ADS_API_TOKEN: {tok}", "«redacted-secret»"),      # grep -n gutter
        ("108-      ADS_API_TOKEN: {tok}", "«redacted-secret»"),  # grep -A/-B/-C context gutter
        ("   108\tADS_API_TOKEN: {tok}", "«redacted-secret»"),   # cat -n / nl gutter (number + TAB)
        ("   108\texport FOO_TOKEN={tok}", "«redacted-secret»"),
        ("GITHUB_TOKEN: ghp_S1abcdefghijklmnopqrstuvwxyz0Pn2T", "«redacted:ghp_…»"),  # prefix label kept
    ])
    def test_secret_file_masks_assignment_with_non_reusable_sentinel(self, template, sentinel):
        text = template.format(tok=self.SYNTH)
        out = redact_sensitive_text(text, force=True, code_file=True, file_read=True, secret_file=True)
        assert self.SYNTH not in out and "ghp_S1" not in out
        assert sentinel in out
        assert out.split(sentinel)[0].rstrip(" ='\"") in text  # key, gutter and quoting survive

    def test_unclassified_read_and_non_secret_scalars_are_untouched(self):
        for text in ("MAX_TOKENS: 100", '{"apiKey": "test"}', "api_key: test", f"5|ADS_API_TOKEN: {self.SYNTH}"):
            assert redact_sensitive_text(text, force=True, file_read=True) == text
        # Strong-key names whose value is a variable/path reference are shell-rc configuration, not
        # secrets; the agent must still be able to read and edit them (password-class keys mask anyway).
        rc = "export SSH_AUTH_SOCK=$HOME/.ssh/agent.sock\nexport DOCKER_AUTH_CONFIG=/home/u/.docker\n"
        out = redact_sensitive_text(rc + f"ADS_API_TOKEN: {self.SYNTH}\n5|MAX_TOKENS: 100\nDB_PASSWORD=~/pw\n",
                                    force=True, file_read=True, secret_file=True)
        assert out.startswith(rc) and self.SYNTH not in out and "5|MAX_TOKENS: 100" in out and "~/pw" not in out
        # The gutter-tolerant anchors must stay linear: a wide indented line is not a stall.
        wide = "1|" + " " * 20000 + "token:"
        started = time.perf_counter()
        assert redact_sensitive_text(wide, force=True, file_read=True, secret_file=True) == wide
        assert time.perf_counter() - started < 1.0


class TestHermesHomePathClassification:
    """``_is_secret_file_arg`` must see the RESOLVED Hermes home: a managed Windows home
    (``%LOCALAPPDATA%\\hermes``) has no ``.hermes`` segment and a resolved path never spells
    ``$HERMES_HOME``, so the literal test alone classified its ``config.yaml`` as ordinary YAML."""

    def test_resolved_home_config_is_secret_bearing_but_project_config_is_not(self, tmp_path, monkeypatch):
        from agent import file_safety
        from agent.redact import _is_secret_file_arg

        home = tmp_path / "hermes"  # no ".hermes" segment
        monkeypatch.setattr(file_safety, "_hermes_home_path", lambda: home)
        monkeypatch.setattr(file_safety, "_hermes_root_path", lambda: home)
        assert _is_secret_file_arg(str(home / "config.yaml"))
        assert _is_secret_file_arg(str(home / "profiles" / "coder" / "config.yaml"))
        assert _is_secret_file_arg(str(home / "backups" / "config" / "config.yaml.good.20260914-184559"))
        assert not _is_secret_file_arg(str(home / "config.yaml.pdf"))  # only the backups/config/ copies
        assert not _is_secret_file_arg(str(tmp_path / "proj" / "config.yaml"))
        assert not _is_secret_file_arg("config.yaml")  # relative, not resolvable to the home


class TestFireworksToken:
    KEY = "fw_" + "A" * 40

    def test_bare_token_masked(self):
        result = redact_sensitive_text(f"fireworks error: key {self.KEY}", force=True)
        assert self.KEY not in result
        assert "fw_AA" in result


    def test_too_short_not_masked(self):
        short = "fw_tooshort"
        result = redact_sensitive_text(f"text {short} here", force=True)
        assert short in result



class TestRedactCdpUrl:
    """redact_cdp_url() is the single chokepoint for CDP endpoint log redaction.

    Unlike the global pass (which deliberately lets web-URL query params and
    userinfo through for OAuth/magic-link workflows), CDP endpoint credentials
    are pure secrets and must always be masked. Both the browser tool's
    session/discovery logs and the supervisor's attach-timeout error route
    through this helper.
    """


    def test_masks_multiple_query_credentials(self):
        url = "wss://provider.example/session?token=aaa-secret&apikey=bbb-secret"
        out = redact_cdp_url(url)
        assert "aaa-secret" not in out
        assert "bbb-secret" not in out




    def test_none_returns_empty(self):
        assert redact_cdp_url(None) == ""


class TestKeywordWordBoundary:
    """Ported from nearai/ironclaw#6129 — a secret keyword embedded inside a
    larger prose word (``Secretary`` ⊃ ``secret``, ``tokenizer`` ⊃ ``token``,
    ``authored`` ⊃ ``auth``) must NOT trigger the lowercase/dotted/YAML config
    passes. Real key shapes (separators, camelCase, acronyms, plurals, common
    concatenated compounds, all-caps env style) must keep redacting.
    """

    # ── prose words embedding a keyword are preserved ──────────────────

    def test_secretary_yaml_value_preserved(self):
        text = "Secretary: JanetYellen1234567890"
        assert redact_sensitive_text(text) == text








    # ── real key shapes still redact ────────────────────────────────────

    def test_separator_keys_still_redacted(self):
        for text in (
            "client_secret: abc123def456ghi789jkl",
            "auth_token: xyz789xyz789xyz789xyz",
            "my_secret: topvalue123456789012345",
            "db.password=hunter2verylongpassword",
        ):
            result = redact_sensitive_text(text)
            assert result != text, text

    def test_camelcase_keys_still_redacted(self):
        for text in (
            "clientSecret: abc123def456ghi789jkl",
            "secretKey: abc123def456ghi789jklmno",
            "APIToken: abc123def456ghi789jklmn",
        ):
            result = redact_sensitive_text(text)
            assert result != text, text


    def test_plural_keys_still_redacted(self):
        text = "secrets: hunter2hunter2hunter2hh"
        result = redact_sensitive_text(text)
        assert "hunter2hunter2hunter2hh" not in result


class TestMaskSecretControlStripping:
    """Issue #55319/#55321: mask_secret() must not emit control bytes
    (newline, NUL, DEL, C1) in the visible head/tail of a masked secret —
    they corrupt config/status/dump display output."""

    def test_newline_stripped_from_mask(self):
        # The #55319 probe: a newline inside the preserved head.
        assert mask_secret("ab\ncd0123456789zzzz") == "abcd...zzzz"

    def test_c1_control_stripped_from_mask(self):
        assert mask_secret("abcd0123456789zz\x85q") == "abcd...9zzq"

    def test_printable_mask_unchanged(self):
        assert mask_secret("abcdef0123456789zzzz") == "abcd...zzzz"

    def test_all_control_value_returns_empty_fallback(self):
        assert mask_secret("\n\x85\u200b") == ""
        assert mask_secret("\n\x85\u200b", empty="(not set)") == "(not set)"


class TestValueAwareGatingCorpus:
    """Issue #96607: corpus-level before/after for value-aware gating.

    Redaction must mask a keyword-named assignment ONLY when the value has
    credential shape (vendor prefix, hex/base64/high-entropy, or a strong
    credential-specific key name). Bare technical vocabulary — ``token``,
    ``key``, ``cpu`` — in ordinary technical prose/config must pass through
    byte-for-byte, on every assignment family (ENV, dotted config, JSON,
    YAML).
    """

    # Realistic technical prose. On pre-fix main every line was corrupted
    # to ``***`` despite containing no secret.
    TECHNICAL_CORPUS = [
        'IDENTITY_TOKEN="bailu"',
        "--override-tensor per_layer_token_embd.weight=CPU",
        "MAX_TOKENS=4096",
        "runtime.token=local",
        "The tokenizer splits on whitespace; set max_new_tokens=256.",
        "num_key_value_heads=8",
        "token: CPU",
        "llm_load_tensors: per_layer_token_embd.weight=CPU buffer",
    ]

    # Obviously-fake but shape-realistic secrets: every one of these must
    # STAY masked after the gating change (fail-closed on credential shape
    # or strong key names).
    FAKE_SECRET_CORPUS = [
        ("API_KEY=sk-fakefakefakefakefake1234567890abcd", "fakefake"),
        ("GITHUB_TOKEN=ghp_FAKEfakeFAKEfake1234567890fake", "FAKEfake"),
        ("MY_SERVICE_TOKEN=A9f3kZq7Lm2Xw8Rt4Yv6", "A9f3kZq7"),
        ("TOKEN=6f1d2a9c8b3e4f5a6d7c8b9a0e1f2d3c", "6f1d2a9c"),
        ("password=hunter2", "hunter2"),
        ("db_password: hunter2", "hunter2"),
        ("auth_token: 9f8e7d6c5b4a39281706f5e4d3c2b1a0", "9f8e7d6c"),
        ('"token": "Zx9Qw8Er7Ty6Ui5Op4As3"', "Zx9Qw8Er"),
        ("SESSION_TOKEN=shrt", "shrt"),
        ("client_secret=abc", "abc"),
        ("spring.datasource.password=fakePass123", "fakePass123"),
    ]

    def test_technical_prose_survives_intact(self):
        for line in self.TECHNICAL_CORPUS:
            assert redact_sensitive_text(line, force=True) == line, line

    def test_technical_corpus_as_one_block_survives_intact(self):
        # The multi-line shape a model actually reads from tool output.
        block = "\n".join(self.TECHNICAL_CORPUS)
        assert redact_sensitive_text(block, force=True) == block

    def test_shape_realistic_fake_secrets_still_masked(self):
        for line, cleartext in self.FAKE_SECRET_CORPUS:
            result = redact_sensitive_text(line, force=True)
            assert result != line, line
            assert cleartext not in result, line

    def test_mixed_block_masks_only_the_secret_lines(self):
        # Precondition guard: both halves must actually exercise the gate.
        secret_line = "MY_SERVICE_TOKEN=A9f3kZq7Lm2Xw8Rt4Yv6"
        prose_line = 'IDENTITY_TOKEN="bailu"'
        block = f"{prose_line}\n{secret_line}"
        result = redact_sensitive_text(block, force=True)
        assert prose_line in result
        assert "A9f3kZq7Lm2Xw8Rt4Yv6" not in result


class TestRedactForEgress:
    """``redact_for_egress`` is the single scrub every remote-reader surface (gateway chat, A2A, monitoring)
    calls; there is no second pattern list to keep in sync."""

    def test_opaque_bearer_without_vendor_prefix_is_masked(self):
        from agent.redact import redact_for_egress
        out = redact_for_egress("curl -H 'Authorization: Bearer opaque0123456789abcdef' https://x.example")
        assert "opaque0123456789abcdef" not in out
        assert "https://x.example" in out

    def test_bearer_sweep_masks_real_tokens_not_the_english_word(self):
        from agent.redact import redact_for_egress
        prose = "I'm the bearer of bad news: the deploy failed"
        assert redact_for_egress(prose) == prose
        assert "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9" not in redact_for_egress("Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9")

    def test_fails_closed_when_the_redactor_raises(self, monkeypatch):
        from agent import redact as R
        monkeypatch.setattr(R, "redact_sensitive_text", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom")))
        assert R.redact_for_egress("sk-live-0123456789abcdef") == R.REDACTION_UNAVAILABLE


class TestWebappLaunchLinks:
    """``hermes webapp`` prints a link that signs a browser in as the user: one with the gateway's
    browser credential after ``#hermes-session=`` (what older gateways printed on macOS), or a one-use
    launch ticket after ``/webapp/launch#ticket=``. Either opens the user's Hermes until it is spent or
    replaced, so when Hermes runs the command through its terminal tool neither reaches the transcript.
    The rest of the link stays readable."""

    CREDENTIAL = "Zq7_" + "k3Lm" * 9 + "x9-"  # the shape secrets.token_urlsafe(32) prints: 43 characters

    def test_the_browser_credential_in_a_link_is_masked(self):
        from agent.redact import redact_terminal_output

        link = "http://127.0.0.1:9120/?profile=work#hermes-session=" + self.CREDENTIAL
        out = redact_terminal_output(f"Open this link in your browser:\n  {link}\n", "hermes webapp")
        assert self.CREDENTIAL not in out
        assert "http://127.0.0.1:9120/?profile=work#hermes-session=" in out

    def test_a_launch_ticket_is_masked(self):
        from agent.redact import redact_terminal_output

        link = "http://127.0.0.1:9120/webapp/launch#ticket=" + self.CREDENTIAL + "&query=%3Fprofile%3Dwork"
        out = redact_terminal_output(f"Open this one-use link:\n  {link}\n", "hermes webapp --no-open")
        assert self.CREDENTIAL not in out
        assert "http://127.0.0.1:9120/webapp/launch#ticket=" in out and "&query=%3Fprofile%3Dwork" in out
        assert self.CREDENTIAL not in redact_sensitive_text(link)

    def test_code_that_builds_such_a_link_is_left_alone(self):
        source = "target.hash = `hermes-session=${token}`\nconst FRAGMENT_KEY = 'hermes-session'\n"
        assert redact_sensitive_text(source, code_file=True) == source