Skip to content

armgit / PS5LM

Public Imported from GitHub
0

llama.cpp on a jailbroken PS5: run any GGUF model natively on the console, starting with Qwen 3.8

8.4 KB
Raw Blame
#!/usr/bin/env python3
"""Which neuron in a layer moves the answer? Ask the running PS5LM.

    python3 examples/interpret/neuron_ablation.py --host <console> \
        --prompt "The capital of France is" --layer 20

The loop this closes is the whole point of the endpoint:

1. ask for the model's **logits** at the last prompt position, with the whole
   vocabulary row, and the **ffn_swiglu** activation of one layer at that position:
   gate x up, one channel per FFN neuron, the input of the down projection;
2. rank that layer's neurons by how strongly they fire (the ones that are doing
   something for this token);
3. for each candidate, send the same request again with that one channel **zeroed**
   in place, and measure what changed: the target token's probability, its rank,
   and the exact KL divergence of the next-token distribution over the whole
   vocabulary.

Everything except the ranking comes from one request per candidate. A layer whose
ablation barely moves anything is a layer that is not carrying this decision; the
channels at the top of the last column are the ones worth looking at next.

`--tensor ffn_out` asks a different question: its channels are the `n_embd`-wide
residual features that the down projection writes, not neurons.

The full vocabulary row is on by default because the KL is then exact (over all
`n_vocab` values); pass `--top-only` to trade that for a tenth of the traffic.
"""

from __future__ import annotations

import argparse
import base64
import json
import math
import sys
import time
import urllib.error
import urllib.request


def post(host: str, path: str, body: dict, timeout: float = 300.0) -> dict:
    data = json.dumps(body).encode()
    req = urllib.request.Request(f"http://{host}:8081{path}", data=data, headers={"Content-Type": "application/json"})
    try:
        with urllib.request.urlopen(req, timeout=timeout) as resp:
            return json.loads(resp.read())
    except urllib.error.HTTPError as e:
        raise SystemExit(f"{e.code} from the console: {e.read().decode('utf-8', 'replace')[:800]}")


def floats(b64: str):
    import array

    a = array.array("f")
    a.frombytes(base64.b64decode(b64))
    return a


def softmax(row):
    m = max(row)
    exps = [math.exp(v - m) for v in row]
    s = sum(exps)
    return [e / s for e in exps]


def request(host: str, args, interventions=None, want_activation=True):
    spec: dict = {
        "logits": {"top_k": args.top_k, "probs": True, "full": not args.top_only, "max_full_rows": 1},
        "positions": "last",
        "generated": False,
        "capture_generated": False,
    }
    if want_activation:
        spec["activations"] = {"tensors": [args.tensor], "layers": [args.layer]}
    if interventions:
        spec["interventions"] = interventions
    body = {"prompt": args.prompt, "n_predict": 0, "temperature": 0.0, "introspect": spec}
    t0 = time.time()
    doc = post(host, "/v1/introspect", body)
    return doc, time.time() - t0


def intro(doc):
    it = doc.get("introspection")
    if not it:
        raise SystemExit("no introspection in the answer: " + json.dumps(doc)[:400])
    entry = it["positions"][-1]
    return it, entry


def main() -> int:
    p = argparse.ArgumentParser()
    p.add_argument("--host", required=True)
    p.add_argument("--prompt", default="The capital of France is")
    p.add_argument("--layer", type=int, default=20)
    p.add_argument("--tensor", default="ffn_swiglu",
                   help="ffn_swiglu (one FFN neuron per channel), or ffn_out, l_out, attn_out (n_embd residual features)")
    p.add_argument("--top", type=int, default=12, help="how many channels to rank and ablate")
    p.add_argument("--top-k", type=int, default=20, help="candidates printed from the answer")
    p.add_argument("--target", help="token piece to track (default: the baseline argmax)")
    p.add_argument("--top-only", action="store_true", help="skip the full vocabulary row (approximate KL)")
    args = p.parse_args()

    print(f"{args.prompt!r} -> layer {args.layer}, tensor {args.tensor}")
    t0 = time.time()
    doc, dt = request(args.host, args)
    it, entry = intro(doc)
    acts = entry.get("activations") or {}
    key = [k for k in acts if k.startswith(args.tensor)]
    if not key:
        raise SystemExit(f"the answer carried no {args.tensor}-<layer>: {sorted(acts)}")
    key = key[0]
    row = floats(acts[key]["data"])
    logits = entry["logits"]
    n_vocab = logits["n_vocab"]

    # the target is tracked by its token id: a piece is found only while it is among
    # the printed candidates, and an ablation can push it out of them
    candidates = [logits["argmax"]] + (logits.get("top") or [])
    if args.target is None:
        target_id, target = logits["argmax"]["id"], logits["argmax"]["piece"]
    else:
        found = [t for t in candidates if t["piece"] == args.target]
        if not found:
            raise SystemExit(f"{args.target!r} is not among the baseline's top {args.top_k} candidates; raise --top-k")
        target_id, target = found[0]["id"], args.target

    print(f"  {it['meta']['n_layer']} layers, vocab {n_vocab}, prompt {it['meta']['n_prompt']} tokens; answer in {dt:.2f} s")
    print(f"  next token: {logits['argmax']['piece']!r} (id {logits['argmax']['id']}, logit {logits['argmax']['logit']:.2f}, "
          f"p {logits['argmax'].get('prob', float('nan')):.4f}), entropy {logits.get('entropy', float('nan')):.3f} nats")
    print(f"  {key}: {len(row)} channels, mean {sum(row)/len(row):.3f}, "
          f"largest |activation| {max(abs(v) for v in row):.3f}")

    def score(doc_logits):
        """The target's logit, probability and rank. Exact from the full row; from the
        printed candidates otherwise, and None there once the target drops out of them."""
        if "full" in doc_logits:
            row_i = floats(doc_logits["full"]["data"])
            probs = softmax(row_i)
            x = row_i[target_id]
            return x, probs[target_id], sum(1 for v in row_i if v > x) + 1, probs
        for t in [doc_logits["argmax"]] + (doc_logits.get("top") or []):
            if t["id"] == target_id:
                return t["logit"], t.get("prob"), None, None
        return None, None, None, None

    base_logit, base_p, base_rank, base_probs = score(logits)
    print(f"  tracking {target!r} (id {target_id}): baseline p={base_p:.6f}" + (f", rank {base_rank}" if base_rank else ""))

    def num(v, spec):
        return format(v, spec) if v is not None else format(float("nan"), spec)

    # candidate neurons: the channels that fire hardest at this position
    order = sorted(range(len(row)), key=lambda i: -abs(row[i]))[: args.top]
    print(f"\n  ablating the {len(order)} loudest channels one at a time (zeroing {args.tensor}-{args.layer} channel c):\n")
    print(f"  {'channel':>7}  {'activation':>11}  {'d logit':>8}  {'p(target)':>12}  {'rank':>7}  {'KL(base||abl)':>13}  argmax")
    results = []
    for c in order:
        doc2, dt2 = request(args.host, args, interventions=[
            {"tensor": args.tensor, "layers": [args.layer], "channels": [c], "op": "zero"},
        ], want_activation=False)
        it2, e2 = intro(doc2)
        lg2 = e2["logits"]
        mod = (it2.get("interventions") or [{}])[0].get("modified")
        logit2, p2, rank2, probs2 = score(lg2)
        dlogit = logit2 - base_logit if logit2 is not None else None
        if base_probs is not None and probs2 is not None:
            kl = sum(p * math.log(p / q) for p, q in zip(base_probs, probs2) if p > 0 and q > 0)
        else:
            kl = float("nan")
        # a target that left the printed candidates moved more than any that stayed
        effect = abs(dlogit) if dlogit is not None else math.inf
        results.append((effect, c, row[c], dlogit, p2, rank2, kl, lg2["argmax"]["piece"], mod))
        print(f"  {c:>7}  {row[c]:>11.4f}  {num(dlogit, '>8.3f')}  "
              f"{num(p2, '>12.6f')}  {rank2 if rank2 else '':>7}  {kl:>13.5f}  {lg2['argmax']['piece']!r}")

    results.sort(key=lambda r: r[0], reverse=True)
    if results:
        _, c, a, dlogit, p2, rank2, kl, am, mod = results[0]
        moved = f"logit {dlogit:+.3f}, p {p2:.6f}" if dlogit is not None else f"left the top {args.top_k}"
        print(f"\n  biggest single-neuron effect: {args.tensor}-{args.layer} channel {c} (activation {a:.3f}) "
              f"-> {target!r} {moved}, argmax {am!r}, KL {kl:.5f} ({mod} elements changed)")
    print(f"  total {time.time() - t0:.1f} s for {1 + len(order)} requests")
    return 0


if __name__ == "__main__":
    sys.exit(main())