Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
#!/usr/bin/env python3
"""Shared frozen-encoder pooling and trained-head application for jev.
Imported by BOTH `run_jev.py` (which applies an adopted head at inference) and
`train_jev_head.py` (which fits one). They must agree exactly on two things or
a head silently scores a different vector space than it was fit on:
* how a (premise, hypothesis) pair is pooled into one vector, and
* how a head's layers map that vector onto the checkpoint's three labels.
Both live here, once. The Node-side contract they mirror is
`server/lib/jevHead.js`; the constants below are repeated rather than imported
so this boundary still holds if either script is invoked directly.
Nothing here reaches the network, reads a repository, or executes model code:
`trust_remote_code` is never set, and a head is plain JSON validated before a
single number of it is used.
"""
import json
import math
import re
from pathlib import Path
# Mirrors of server/lib/jevHead.js.
HEAD_SCHEMA_VERSION = 1
POOLING = "last-token"
ARCHITECTURES = ("linear", "mlp1")
MAX_PARAMS = 2_000_000
MAX_HIDDEN = 256
# Mirrors JEV_LABELS in server/lib/jev.js, in the checkpoint's id2label order.
LABELS = ("contradiction", "entailment", "neutral")
# A head is addressed by slug, never by path: the sidecar resolves it inside a
# directory the Node service owns, so a request cannot name a file elsewhere.
SLUG = re.compile(r"^[a-z0-9][a-z0-9-]{0,63}$")
# Used only when the checkpoint's config omits its own `nli_template`.
FALLBACK_TEMPLATE = "Premise: {premise}\nHypothesis: {hypothesis}"
class HeadError(Exception):
"""A head failure the Node service can map to a code without reading text."""
def __init__(self, code: str) -> None:
super().__init__(code)
self.code = code
def resolve_template(config) -> str:
"""The checkpoint's own NLI template, or the fallback.
Shared by the sidecar and the trainer: a head fit under one template and
applied under another is fit on a different vector space than it is scored
in, which is the exact drift this module exists to prevent.
"""
template = getattr(config, "nli_template", None)
if not isinstance(template, str) or "{premise}" not in template or "{hypothesis}" not in template:
return FALLBACK_TEMPLATE
return template
def resolve_window(tokenizer, config) -> int:
"""The largest token count one pair may occupy.
Transformers uses a sentinel-sized `model_max_length` for tokenizers that
declare none, which would read as "everything fits" — so an implausible
value falls through to the text config's position count.
"""
candidates = []
for value in (getattr(tokenizer, "model_max_length", None),
getattr(getattr(config, "text_config", None), "max_position_embeddings", None),
getattr(config, "max_position_embeddings", None)):
if isinstance(value, int) and 0 < value < 10_000_000:
candidates.append(value)
if not candidates:
raise HeadError("jev-response-invalid")
return min(candidates)
def load_encoder(model_dir):
"""Load the pinned snapshot and build the state `pool_pair` reads.
ONE loader for both processes. The sidecar layers its `id2label` mapping on
top (it needs the checkpoint's own classifier); the trainer freezes the
parameters. Everything that decides how a pair becomes a vector — the
device, the template, the window, the final-hidden-state hook — is decided
here, once, or the trainer and the sidecar pool differently.
"""
import torch
from transformers import AutoModelForSequenceClassification, AutoTokenizer
torch.set_num_threads(1)
tokenizer = AutoTokenizer.from_pretrained(
str(model_dir), local_files_only=True, trust_remote_code=False
)
model = AutoModelForSequenceClassification.from_pretrained(
str(model_dir),
local_files_only=True,
trust_remote_code=False,
use_safetensors=True,
)
# MPS where the host has it, CPU otherwise. Reported on /health so an
# operator can tell a slow CPU fallback from a healthy accelerated load.
device = "mps" if getattr(torch.backends, "mps", None) and torch.backends.mps.is_available() else "cpu"
model.to(device)
model.eval()
captured, _handle = capture_final_hidden_state(model)
return {
"torch": torch,
"tokenizer": tokenizer,
"model": model,
"device": device,
"template": resolve_template(model.config),
"window": resolve_window(tokenizer, model.config),
"hiddenCapture": captured,
}
def capture_final_hidden_state(model):
"""Register a forward hook that keeps ONLY the final hidden state.
Returns `(captured, handle)`, where `captured["value"]` holds the last
encoder output after each forward pass.
`output_hidden_states=True` would be one line shorter and is what this
started as. It is also the expensive way: it retains one activation tensor
per decoder layer for the whole pass, and this checkpoint has dozens — a
few hundred megabytes of extra peak allocation per pair, on a machine that
may already be holding the 9 GB sidecar copy resident, to use exactly one
of them. A hook on the base model keeps the one.
"""
captured = {}
def keep(_module, _inputs, output):
# A base `PreTrainedModel` returns either a tuple whose first element is
# the last hidden state, or a ModelOutput carrying it by name.
captured["value"] = output[0] if isinstance(output, tuple) else output.last_hidden_state
return captured, model.base_model.register_forward_hook(keep)
def pool_pair(state, premise: str, hypothesis: str):
"""Encode one pair and return `(pooled_vector, stock_logits)`.
The pooled vector is the LAST token's final hidden state. openjev's 4B NLI
checkpoint is a causal decoder, so that is the only position which has
attended to the whole pair — a mean over the sequence would dilute the
hypothesis, which is the half that varies.
The checkpoint's own classifier logits come out of the same forward pass.
The trainer needs both (the head's input, and the stock baseline it has to
beat), and paying for two passes to collect them would double the cost of
the one step that dominates a training run.
"""
torch = state["torch"]
tokenizer = state["tokenizer"]
model = state["model"]
text = state["template"].format(premise=premise, hypothesis=hypothesis)
token_ids = tokenizer.encode(text, add_special_tokens=True, truncation=False)
# Same refusal as `score_pairs`: a verdict on a truncated prefix is not a
# verdict on the pair, and nothing downstream could tell the difference.
if len(token_ids) > state["window"]:
raise HeadError("jev-premise-too-large")
device = state.get("device", "cpu")
model_inputs = {
"input_ids": torch.tensor([token_ids]).to(device),
"attention_mask": torch.tensor([[1] * len(token_ids)]).to(device),
}
# The hook is installed once per loaded model and lives in `state`;
# re-registering it per pair would stack a new one on every call.
captured = state["hiddenCapture"]
captured.pop("value", None)
with torch.inference_mode():
output = model(**model_inputs)
hidden = captured.get("value")
if hidden is None:
raise HeadError("jev-head-invalid")
pooled = hidden[0, -1, :].to("cpu").float().tolist()
logits = output.logits[0].to("cpu").float().tolist()
return pooled, logits
def validate_head(raw, *, revision: str):
"""Validate a head artifact's shape, size and encoder revision.
Mirrors `parseJevHead` + `isHeadCompatible` in server/lib/jevHead.js. The
revision check is not a nicety: the head's inputs are hidden states from one
specific checkpoint, so applying it across a different one produces
confident numbers with nothing anywhere to signal they are meaningless.
"""
if not isinstance(raw, dict):
raise HeadError("jev-head-invalid")
if raw.get("schemaVersion") != HEAD_SCHEMA_VERSION:
raise HeadError("jev-head-invalid")
if raw.get("pooling") != POOLING:
raise HeadError("jev-head-invalid")
if raw.get("architecture") not in ARCHITECTURES:
raise HeadError("jev-head-invalid")
if list(raw.get("labels") or []) != list(LABELS):
raise HeadError("jev-head-invalid")
base = raw.get("baseModel")
if not isinstance(base, dict) or base.get("revision") != revision:
raise HeadError("jev-head-revision-mismatch")
layers = raw.get("layers")
hidden_size = raw.get("hiddenSize")
if not isinstance(layers, list) or not 1 <= len(layers) <= 2:
raise HeadError("jev-head-invalid")
# The architecture's own layer count. Mirrors `parseJevHead`: a `linear`
# head declaring two layers, or an `mlp1` declaring one, is an artifact
# whose stated shape disagrees with its weights — and the two validators
# disagreeing about that is precisely the drift this mirror must not have.
if len(layers) != (2 if raw["architecture"] == "mlp1" else 1):
raise HeadError("jev-head-invalid")
if not isinstance(hidden_size, int) or hidden_size < 1:
raise HeadError("jev-head-invalid")
params = 0
expected_input = hidden_size
for position, layer in enumerate(layers):
if not isinstance(layer, dict):
raise HeadError("jev-head-invalid")
weight = layer.get("weight")
bias = layer.get("bias")
if not isinstance(weight, list) or not weight or not isinstance(bias, list):
raise HeadError("jev-head-invalid")
if len(bias) != len(weight):
raise HeadError("jev-head-invalid")
for row in weight:
if not isinstance(row, list) or len(row) != expected_input:
raise HeadError("jev-head-invalid")
if not all(isinstance(cell, (int, float)) and math.isfinite(cell) for cell in row):
raise HeadError("jev-head-invalid")
params += len(row)
if not all(isinstance(cell, (int, float)) and math.isfinite(cell) for cell in bias):
raise HeadError("jev-head-invalid")
params += len(bias)
expected_input = len(weight)
if position == 0 and len(layers) == 2 and len(weight) > MAX_HIDDEN:
raise HeadError("jev-head-invalid")
if expected_input != len(LABELS):
raise HeadError("jev-head-invalid")
if params > MAX_PARAMS:
raise HeadError("jev-head-too-large")
return raw
def head_path(heads_dir, slug: str):
"""Resolve `<heads_dir>/<slug>.json`, refusing a slug that is a path.
The charset is matched BEFORE the name touches the filesystem, and the
resolved path is required to stay inside `heads_dir` — the sidecar accepts a
head name from a local caller, and a name is not a path.
"""
if not isinstance(slug, str) or not SLUG.match(slug):
raise HeadError("jev-head-invalid")
root = Path(heads_dir).resolve()
path = (root / f"{slug}.json").resolve()
if path.parent != root:
raise HeadError("jev-head-invalid")
return path
def head_file_identity(heads_dir, slug: str):
"""A cheap `(mtime_ns, size)` fingerprint, or None when there is no file.
What makes the sidecar's head cache invalidatable from the side that can
observe an adoption: Node rewrites the file, the fingerprint moves, the
next request re-reads it.
"""
path = head_path(heads_dir, slug)
try:
info = path.stat()
except OSError:
return None
return (info.st_mtime_ns, info.st_size)
def load_head(heads_dir, slug: str, *, revision: str):
"""Read and validate `<heads_dir>/<slug>.json`."""
path = head_path(heads_dir, slug)
if not path.is_file():
raise HeadError("jev-head-not-found")
try:
raw = json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError):
raise HeadError("jev-head-unreadable")
return validate_head(raw, revision=revision)
def apply_head(head, pooled):
"""Run a validated head over one pooled vector, returning label probabilities.
Plain Python arithmetic, not a torch graph: the head is at most a couple of
thousand multiply-accumulates per pair on top of a 4B forward pass, and
keeping it here means the applied maths is the same code an operator can
read next to the weights they are being asked to adopt.
"""
activations = list(pooled)
layers = head["layers"]
for position, layer in enumerate(layers):
weight = layer["weight"]
bias = layer["bias"]
activations = [
sum(cell * value for cell, value in zip(row, activations)) + offset
for row, offset in zip(weight, bias)
]
# ReLU between layers only. The final layer emits logits, which the
# softmax below turns into the same distribution the stock classifier
# head produces — so everything downstream is unchanged.
if position < len(layers) - 1:
activations = [value if value > 0 else 0.0 for value in activations]
ceiling = max(activations)
exponentiated = [math.exp(value - ceiling) for value in activations]
total = sum(exponentiated)
return {label: value / total for label, value in zip(LABELS, exponentiated)}