Skip to content

Personal AI, On Personal Devices

Python
9.9 KB
Raw Blame
"""Tests for openjarvis.analytics.redaction.

Two layers of guarantee are tested here:
  - Value-level: ``redact()`` drops string values that match any PII
    pattern, exceed ``MAX_STR_LEN``, or carry composite types.
  - Structural: ``validate_event()`` from the catalog drops unknown
    events and properties whose values fail the per-spec validator.

Together they form fail-closed PII protection. Adding a new event or
property anywhere in the analytics module should require touching
this file as well.
"""

from __future__ import annotations

import pytest

from openjarvis.analytics.events import (
    REGISTRY,
    known_event_names,
    validate_event,
)
from openjarvis.analytics.redaction import (
    MAX_STR_LEN,
    hash_id,
    looks_like_pii,
    redact,
)

# ---------------------------------------------------------------------------
# Value-level PII detection
# ---------------------------------------------------------------------------


@pytest.mark.parametrize(
    "value",
    [
        # Emails
        "[email protected]",
        "[email protected]",
        "leak hidden in [email protected] sentence",
        # IPv4
        "127.0.0.1",
        "192.168.1.42",
        "trace from 10.0.0.5 here",
        # IPv6 (loose pattern)
        "fe80::1ff:fe23:4567:890a",
        # MAC
        "00:1A:2B:3C:4D:5E",
        "AA:BB:CC:DD:EE:FF",
        # Home paths
        "/Users/alice/Documents/secret.txt",
        "/home/bob/.ssh/id_rsa",
        "~/Library/Application Support",
        "$HOME/secrets",
        # File URLs
        "file:///etc/passwd",
        # API keys
        "sk-abcdefghijklmnop123456",
        "sk-proj-AbCdEfGhIjKlMnOpQrStUv",
        "xoxb-1234567890-abcdefghij-1234567890abcdef",
        "ghp_1234567890abcdefghij1234567890abcd",
        "gho_1234567890abcdefghij1234567890abcd",
        "AKIAIOSFODNN7EXAMPLE",
        "AIzaSyD-1234567890abcdefghij1234567890",
        "ya29.a0AfH6SMBabc123def456ghi789jkl",
        # JWT
        "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0In0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U",
        # Bearer tokens
        "Bearer abc123def456",
        "bearer xyz789",
        # Password / secret assignments
        "password=hunter2",
        "API_KEY: super-secret-value",
        "token=abc123",
        # .local hostnames
        "alice-macbook.local",
    ],
)
def test_pii_patterns_are_detected(value: str) -> None:
    assert looks_like_pii(value), f"expected to flag: {value!r}"


@pytest.mark.parametrize(
    "value",
    [
        "darwin",
        "arm64",
        "0.1.1",
        "qwen3.5:2b",  # model name with colon — must not look like IPv6
        "ok",
        "chat",
        "v1.2.3",
        "8754b77c7eee26e4",  # a hashed id
        "true",
    ],
)
def test_safe_values_pass(value: str) -> None:
    assert not looks_like_pii(value), f"false positive: {value!r}"


# ---------------------------------------------------------------------------
# redact() — full property dict cleaning
# ---------------------------------------------------------------------------


def test_redact_keeps_safe_values() -> None:
    cleaned = redact(
        {
            "os": "darwin",
            "arch": "arm64",
            "count": 42,
            "ratio": 0.5,
            "enabled": True,
        }
    )
    assert cleaned == {
        "os": "darwin",
        "arch": "arm64",
        "count": 42,
        "ratio": 0.5,
        "enabled": True,
    }


def test_redact_drops_pii_strings() -> None:
    _jwt = (
        "eyJhbGciOiJIUzI1NiJ9"
        ".eyJzdWIiOiIxMjM0In0"
        ".dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U"
    )
    cleaned = redact(
        {
            "ok": "chat",
            "leak_email": "[email protected]",
            "leak_path": "/Users/alice/file.txt",
            "leak_key": "sk-1234567890abcdef",
            "leak_jwt": _jwt,
            "leak_bearer": "Bearer abc123",
        }
    )
    assert cleaned == {"ok": "chat"}


def test_redact_drops_long_strings() -> None:
    huge = "x" * (MAX_STR_LEN + 1)
    cleaned = redact({"safe": "short", "huge": huge})
    assert cleaned == {"safe": "short"}


def test_redact_drops_empty_strings() -> None:
    # Empty strings carry no signal — drop them to keep the wire clean.
    cleaned = redact({"empty": "", "real": "value"})
    assert cleaned == {"real": "value"}


def test_redact_drops_composite_types() -> None:
    # Lists, dicts, sets, and tuples bypass redaction entirely. Sending
    # them would let PII smuggle through inside containers.
    cleaned = redact(
        {
            "good": 1,
            "list": ["a", "b"],
            "dict": {"k": "v"},
            "set": {"x", "y"},
            "tuple": ("p", "q"),
        }
    )
    assert cleaned == {"good": 1}


def test_redact_preserves_numeric_zero_and_false() -> None:
    # Falsy non-string values must survive — they're meaningful counts.
    cleaned = redact({"a": 0, "b": False, "c": 0.0})
    assert cleaned == {"a": 0, "b": False, "c": 0.0}


def test_redact_returns_new_dict_does_not_mutate_input() -> None:
    src = {"safe": "ok", "leak": "[email protected]"}
    out = redact(src)
    assert "leak" in src  # original unchanged
    assert "leak" not in out


# ---------------------------------------------------------------------------
# hash_id() — for cohorting without leaking identifiers
# ---------------------------------------------------------------------------


def test_hash_id_is_16_lowercase_hex() -> None:
    h = hash_id("llama3.1:8b")
    assert len(h) == 16
    assert all(c in "0123456789abcdef" for c in h)


def test_hash_id_is_deterministic() -> None:
    a = hash_id("some-model-name")
    b = hash_id("some-model-name")
    assert a == b


def test_hash_id_differs_for_different_inputs() -> None:
    assert hash_id("model-a") != hash_id("model-b")


def test_hash_id_empty_string_returns_empty() -> None:
    assert hash_id("") == ""


# ---------------------------------------------------------------------------
# Structural validation — events.py catalog
# ---------------------------------------------------------------------------


def test_unknown_event_is_rejected() -> None:
    assert validate_event("not_a_real_event", {"any": "thing"}) is None


def test_known_event_with_no_properties_is_accepted() -> None:
    # Empty result is still a valid event — we'd still record it.
    assert validate_event("first_chat_sent", {}) == {}


def test_unknown_properties_are_silently_dropped() -> None:
    cleaned = validate_event(
        "install_started",
        {
            "os": "darwin",
            "arch": "arm64",
            "installer_version": "0.1.1",
            "malicious_extra": "should-not-survive",
        },
    )
    assert cleaned == {
        "os": "darwin",
        "arch": "arm64",
        "installer_version": "0.1.1",
    }


def test_invalid_enum_value_is_dropped() -> None:
    # 'os' must be in the allowlist; a path-like value fails.
    cleaned = validate_event(
        "install_started",
        {"os": "/Users/alice", "arch": "arm64"},
    )
    assert cleaned == {"arch": "arm64"}


def test_invalid_int_property_is_dropped() -> None:
    cleaned = validate_event(
        "install_stage_completed",
        {"stage": "uv", "elapsed_ms": -5},  # negative ms fails _is_int_nonneg
    )
    assert cleaned == {"stage": "uv"}


def test_bool_is_not_int_for_int_validators() -> None:
    # Critical: bool is a subclass of int in Python; the catalog
    # validators must reject booleans where ints are expected
    # (otherwise True would count as elapsed_ms=1).
    cleaned = validate_event(
        "install_stage_completed",
        {"stage": "uv", "elapsed_ms": True},
    )
    assert "elapsed_ms" not in cleaned


def test_chat_session_ended_properties_pass() -> None:
    cleaned = validate_event(
        "chat_session_ended",
        {
            "turn_count": 5,
            "tokens_in": 1000,
            "tokens_out": 500,
            "latency_ms_p50": 250.0,
            "latency_ms_p95": 800.0,
            "tool_count": 3,
            "unique_tools": 2,
            "unique_models": 1,
            "error_count": 0,
            "model_hash": "8754b77c7eee26e4",
            "engine": "ollama",
            "duration_ms": 60000,
        },
    )
    assert cleaned["turn_count"] == 5
    assert cleaned["model_hash"] == "8754b77c7eee26e4"
    assert cleaned["engine"] == "ollama"


def test_chat_session_ended_rejects_bad_hash() -> None:
    cleaned = validate_event(
        "chat_session_ended",
        {
            "turn_count": 1,
            "model_hash": "not-a-hash-too-short",  # not 16 hex chars
        },
    )
    assert "turn_count" in cleaned
    assert "model_hash" not in cleaned


# ---------------------------------------------------------------------------
# Catalog sanity
# ---------------------------------------------------------------------------


def test_catalog_has_expected_lifecycle_events() -> None:
    names = set(known_event_names())
    must_exist = {
        "install_started",
        "install_stage_completed",
        "install_completed",
        "install_failed",
        "uninstall_started",
        "app_opened",
        "setup_completed",
        "first_chat_sent",
        "chat_session_ended",
        "tool_first_used",
        "model_changed",
        "feature_used",
        "error_shown_to_user",
        "feedback_submitted",
        "usage_daily_summary",
    }
    missing = must_exist - names
    assert not missing, f"catalog is missing required events: {sorted(missing)}"


def test_every_event_has_at_least_one_property_or_is_marker() -> None:
    # Either the spec has properties, OR it's intentionally a marker
    # event with no payload (e.g. first_chat_sent). We just assert
    # that no property dict is malformed.
    for name, spec in REGISTRY.items():
        assert isinstance(spec.properties, dict), f"{name}: properties must be dict"
        for prop, validator in spec.properties.items():
            assert callable(validator), f"{name}.{prop}: validator must be callable"