FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""Per-Layer Embedding (PLE) for Qwen3.8-Flash-Next: hashed n-gram features injected at layer 1.
HF reference: ``Qwen4ExpTextNGramEmbedding`` (modeling_qwen4_exp.py:1018) and
``Qwen4ExpTextPLELayer`` (:1117). Per token::
E = table[hash(ngram)] # 16 heads (8 x 2-gram, 8 x 3-gram) x 160 -> 2560
K = norm_key(key_proj(E)).view(hc, hidden) # V = value_proj(E) [hidden]
Q = norm_query(R).view(hc, hidden)
u = <K_i, Q_i> / sqrt(hidden) # per stream
U = sigmoid(sign(u) * sqrt(max(|u|, 1e-6))) * V
D = U + silu(conv1d(norm_conv(U))) # depthwise, kernel 4, dilation ngram_size
R += D # before the attention hyper-connection mix
The table is the 47.7 GiB FP8 n-gram store: ``PinnedUVATable`` keeps it in pinned host memory and
gathers rows over UVA, optionally started early on a side stream (``PLELayer.start_prefetch``).
``GpuResidentTable`` is the small-table oracle the pinned backend is diffed against.
"""
from __future__ import annotations
import math
from dataclasses import dataclass
from typing import TYPE_CHECKING, List, Protocol, Sequence, Tuple
import torch
import torch.nn.functional as F
from freetoken.core import get_global_ctx
from freetoken.layers import BaseOP, LinearReplicated
from freetoken.mm import restore_placeholder
from .config import PLE_CONV_STATE, PLE_NGRAM_STATE
from .hc import GroupedPlusOneRMSNorm
if TYPE_CHECKING:
from freetoken.core import Batch
from freetoken.models.config import ModelConfig
from .config import Qwen4ExpArgs
_MASK64 = (1 << 64) - 1
_SPLITMIX_GAMMA = 0x9E3779B97F4A7C15
_SPLITMIX_M1 = 0xBF58476D1CE4E5B9
_SPLITMIX_M2 = 0x94D049BB133111EB
_PLE_LAYER_PRIME = 10007
class PLETableBackend(Protocol):
"""Row store for one PLE layer's n-gram embedding table (Qwen3.8: 40M rows x 160, FP8 + one scalar scale).
Frozen contract. ``GpuResidentTable`` (oracle, small tables) and ``PinnedUVATable`` (the real 47.7 GiB pinned-host table) implement it. Rows are addressed by the
GLOBAL hashed id, i.e. the per-head vocab offset is already added by ``NGramEmbedding``.
``lookup`` gets ``row_ids [T, num_ngram_heads]`` (int64, device) and returns
``[T, num_ngram_heads * head_dim]`` in ``dtype``, already dequantized (fp8 -> dtype, times the
scalar weight_scale). ``out``, when given, is the destination and is returned as-is (CUDA-graph
decode reuses a fixed buffer).
``prefetch`` may start the gather early on a side stream (the model issues it before layer 0 and
joins it in ``lookup``); a backend with no async path makes it a no-op.
"""
num_rows: int
head_dim: int
dtype: torch.dtype
def lookup(self, row_ids: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor: ...
def prefetch(self, row_ids: torch.Tensor) -> None: ...
class GpuResidentTable:
"""PLE table held whole in GPU memory; ``index_select`` oracle for the pinned-host backend."""
def __init__(
self, weight: torch.Tensor, scale: float = 1.0, dtype: torch.dtype | None = None
) -> None:
self.weight = weight
self.scale = float(scale)
self.num_rows, self.head_dim = weight.shape
self.dtype = dtype if dtype is not None else (
torch.bfloat16 if weight.dtype.itemsize < 2 else weight.dtype
)
def lookup(self, row_ids: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor:
rows = self.weight.index_select(0, row_ids.reshape(-1)).to(self.dtype)
if self.scale != 1.0:
rows = rows * self.scale
rows = rows.view(*row_ids.shape[:-1], -1)
if out is None:
return rows
out.copy_(rows)
return out
def prefetch(self, row_ids: torch.Tensor) -> None:
return None
class ZeroTable:
"""Dummy-weight stand-in: every lookup reads zeros (dummy checkpoints ship no table)."""
def __init__(self, num_rows: int, head_dim: int, dtype: torch.dtype = torch.bfloat16) -> None:
self.num_rows = int(num_rows)
self.head_dim = head_dim
self.dtype = dtype
def lookup(self, row_ids: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor:
if out is not None:
return out.zero_()
return torch.zeros(
(*row_ids.shape[:-1], row_ids.shape[-1] * self.head_dim),
dtype=self.dtype,
device=row_ids.device,
)
def prefetch(self, row_ids: torch.Tensor) -> None:
return None
class PinnedUVATable:
"""PLE table left in pinned host memory; rows are gathered over UVA by a Triton kernel.
``weight`` must be the filled and ``pin()``ed ``HostBank.tensor`` from
``weight.load_ple_table`` (``[num_rows, head_dim]``, fp8-e4m3 or bf16); an unregistered host
buffer is not device-addressable and the kernel faults on it. ``scale`` is the checkpoint's
scalar ``weight_scale``. Gathers emit bf16 into a staging buffer, one per captured decode size
and one growable buffer for everything else.
``prefetch`` runs the gather on a private stream and the next ``lookup`` joins it. ``lookup``
returns a view of that staging buffer, so the rows must be consumed before the next lookup.
"""
def __init__(
self,
weight: torch.Tensor,
scale: float = 1.0,
*,
device: torch.device | None = None,
prefetch: bool = True,
) -> None:
assert weight.device.type == "cpu" and weight.is_contiguous()
assert weight.dtype in (torch.float8_e4m3fn, torch.bfloat16), weight.dtype
from freetoken.kernel.pinned import device_ptr
self.weight = weight
self.scale = float(scale)
self.num_rows, self.head_dim = weight.shape
self.dtype = torch.bfloat16
self._is_fp8 = weight.dtype == torch.float8_e4m3fn
self._device = device or torch.device("cuda", torch.cuda.current_device())
# WDDM maps registered host memory at a different device address; on Linux/UVA this is data_ptr
self._table_ptr = device_ptr(weight)
self._stream = torch.cuda.Stream(device=self._device) if prefetch else None
self._staging: torch.Tensor | None = None
self._graph_staging: dict[int, torch.Tensor] = {}
self._pending: Tuple[torch.Tensor, torch.Tensor] | None = None
def _stage(self, rows: int) -> torch.Tensor:
# Captured graphs keep one buffer per size for good: growing the eager one would free the
# block a replay still writes to.
if torch.cuda.is_current_stream_capturing():
buf = self._graph_staging.get(rows)
if buf is None:
buf = torch.empty((rows, self.head_dim), dtype=self.dtype, device=self._device)
self._graph_staging[rows] = buf
return buf
buf = self._staging
if buf is None or buf.shape[0] < rows:
buf = torch.empty((rows, self.head_dim), dtype=self.dtype, device=self._device)
self._staging = buf
return buf[:rows]
def _gather(self, row_ids: torch.Tensor, dst: torch.Tensor) -> torch.Tensor:
from freetoken.kernel.triton.ple import ple_gather_rows
return ple_gather_rows(
self._table_ptr,
self.num_rows,
self.head_dim,
row_ids.reshape(-1),
dst,
self.scale,
self._is_fp8,
)
def prefetch(self, row_ids: torch.Tensor) -> None:
if self._stream is None or row_ids.numel() == 0:
return
dst = self._stage(row_ids.numel())
self._stream.wait_stream(torch.cuda.current_stream(self._device))
if not torch.cuda.is_current_stream_capturing():
row_ids.record_stream(self._stream)
with torch.cuda.stream(self._stream):
self._gather(row_ids, dst)
self._pending = (row_ids, dst)
def lookup(self, row_ids: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor:
pending, self._pending = self._pending, None
if pending is not None:
# join even on a miss: the stale prefetch owns the staging buffer about to be reused
torch.cuda.current_stream(self._device).wait_stream(self._stream)
if pending is not None and pending[0] is row_ids:
rows = pending[1]
else:
rows = self._gather(row_ids, self._stage(row_ids.numel()))
rows = rows.view(*row_ids.shape[:-1], -1)
if out is None:
return rows
out.copy_(rows)
return out
def _splitmix64(value: int) -> int:
value = (value + _SPLITMIX_GAMMA) & _MASK64
value = ((value ^ (value >> 30)) * _SPLITMIX_M1) & _MASK64
value = ((value ^ (value >> 27)) * _SPLITMIX_M2) & _MASK64
return (value ^ (value >> 31)) & _MASK64
def _is_prime(value: int) -> bool:
if value < 2:
return False
if value % 2 == 0:
return value == 2
for divisor in range(3, math.isqrt(value) + 1, 2):
if value % divisor == 0:
return False
return True
def _nth_prime_after(start: int, count: int) -> int:
prime = start
for _ in range(count):
prime += 1
while not _is_prime(prime):
prime += 1
return prime
def derive_ngram_hash_constants(
*,
vocab_size: int,
ngram_size: int,
num_ngram_heads: int,
ngram_vocab_size_base: int,
ple_layer_index: int,
seed: int = 1234,
) -> Tuple[List[int], List[int], List[int]]:
"""Recompute (multipliers, per-head vocab sizes, per-head offsets) the way HF derives them at init.
The checkpoint ships these as int64 tensors, so serving loads them; this is the dummy-weight
path and the oracle a loader test can check the checkpoint values against.
"""
half_bound = max(1, ((1 << 63) - 1) // max(vocab_size, 1) // 2)
base_seed = seed + _PLE_LAYER_PRIME * ple_layer_index
multipliers = [
2 * (_splitmix64((base_seed + _SPLITMIX_GAMMA * (i + 1)) & _MASK64) % half_bound) + 1
for i in range(ngram_size)
]
sizes: List[int] = []
offsets: List[int] = []
total = 0
for head in range(num_ngram_heads):
global_head = ple_layer_index * num_ngram_heads + head
size = _nth_prime_after(ngram_vocab_size_base - 1, global_head + 1)
sizes.append(size)
offsets.append(total)
total += size
return multipliers, sizes, offsets
@dataclass
class PLEMetadata:
"""Per-forward PLE inputs, built once and shared by every PLE layer (sibling of ``FLAMetadata``).
Frozen contract:
input_ids [T] int device -- this forward's tokens, ragged, concatenated in request order
cu_seqlens [B+1] int device -- query indptr; decode is ``arange(B+1)``
seq_lens host per-request token counts; avoids a device sync in the ragged conv loop
ngram_context [B, ngram_size-1] int64 device -- the tokens immediately BEFORE each request's
first token of this forward, read from the ``ple_ngram_ctx`` slot state and
forced to the boundary (eos) id for fresh rows. The hash never crosses eos,
so a fresh sequence passes all-eos.
state_slots [B] int64 device -- linear-state slot per request (``Req.linear_slot_idx`` or
``Req.table_idx``); keys every PLE slot state
fresh_slots [B] bool device or None -- request starts a new sequence, so read a zero state
is_decode one token per request (the batched 4-tap path)
"""
input_ids: torch.Tensor
cu_seqlens: torch.Tensor
seq_lens: Sequence[int]
ngram_context: torch.Tensor
state_slots: torch.Tensor
fresh_slots: torch.Tensor | None
is_decode: bool
def _state_slot(req) -> int:
slot = getattr(req, "linear_slot_idx", None)
return req.table_idx if slot is None else slot
def _ngram_context_pool() -> torch.Tensor:
pool = get_global_ctx().linear_state_pool
assert pool is not None and pool.has_slot_state(PLE_NGRAM_STATE), (
"PLE needs the ple_ngram_ctx slot state (or an explicit context_pool=)"
)
return pool.slot_state(PLE_NGRAM_STATE)
def build_ple_metadata(
batch: Batch,
args: Qwen4ExpArgs,
device: torch.device,
context_pool: torch.Tensor | None = None,
) -> PLEMetadata:
"""Build ``PLEMetadata`` from a scheduler batch.
The n-gram context is per-request device state (``ple_ngram_ctx`` [num_slots, ngram_size-1],
rolled forward once per forward by ``commit_ngram_context``), so it never lags the sampled
token under overlap scheduling and follows the slot on COW/snapshot. A decode batch takes its
slots straight off ``batch.fla_metadata.cache_indices`` (persistent under a graph), so the
build is capture-safe and sync-free. Reuses ``batch.fla_metadata`` (slots / indptr / fresh
mask) when the scheduler built it.
"""
reqs = batch.padded_reqs
ctx_len = args.ngram_size - 1
eos = args.ngram_boundary_token_id
if context_pool is None:
context_pool = _ngram_context_pool()
assert context_pool.shape[-1] == ctx_len, (
f"ple_ngram_ctx holds {context_pool.shape[-1]} ids, config wants {ctx_len}"
)
fla = getattr(batch, "fla_metadata", None)
# image rows hash as the placeholder token, never as the content pad ids
input_ids = batch.input_ids
if args.image_token_id is not None:
input_ids = restore_placeholder(input_ids, args.image_token_id)
if batch.is_decode and fla is not None:
slots = fla.cache_indices.long()
bs = slots.numel()
return PLEMetadata(
input_ids=input_ids,
cu_seqlens=torch.arange(bs + 1, dtype=torch.int32, device=device),
seq_lens=(1,) * bs,
ngram_context=context_pool.index_select(0, slots).long(),
state_slots=slots,
fresh_slots=None,
is_decode=True,
)
lens = [r.extend_len for r in reqs]
if fla is not None and fla.has_initial_state is not None:
cu = fla.cu_seqlens
slots = fla.cache_indices.long()
fresh = ~fla.has_initial_state
else: # direct-op callers (tests) with no scheduler metadata
pin = {"device": "cpu", "pin_memory": torch.cuda.is_available()}
cu = torch.tensor([0, *lens], dtype=torch.int64, **pin).cumsum_(0).to(device, non_blocking=True)
slots = torch.tensor([_state_slot(r) for r in reqs], dtype=torch.int64, **pin).to(device, non_blocking=True)
fresh = torch.tensor([r.cached_len == 0 for r in reqs], dtype=torch.bool, **pin).to(device, non_blocking=True)
context = context_pool.index_select(0, slots).long()
context = torch.where(fresh.unsqueeze(1), context.new_full((), eos), context)
return PLEMetadata(
input_ids=input_ids,
cu_seqlens=cu,
seq_lens=tuple(lens),
ngram_context=context,
state_slots=slots,
fresh_slots=fresh,
is_decode=batch.is_decode,
)
def commit_ngram_context(meta: PLEMetadata, fla, context_pool: torch.Tensor | None = None) -> None:
"""Roll each request's ``ple_ngram_ctx`` forward past this forward's tokens.
Called ONCE per forward after every PLE layer ran (the layers only read the context);
also writes the boundary-aligned window to the track slot so a donated snapshot restores
the context together with the conv state. Pure device arithmetic, capture-safe.
"""
if context_pool is None:
context_pool = _ngram_context_pool()
ids = meta.input_ids.long()
ctx_len = meta.ngram_context.shape[1]
steps = torch.arange(ctx_len, device=ids.device)
if meta.is_decode:
nxt = torch.cat([meta.ngram_context[:, 1:], ids.view(-1, 1)], dim=1)
else:
cu = meta.cu_seqlens.long()
cand = cu[1:].unsqueeze(1) - ctx_len + steps
# short extends fall back to the old context: token j of the new window sits at
# old-context column extend_len + j when it predates this forward
old = meta.ngram_context.gather(
1, ((cu[1:] - cu[:-1]).unsqueeze(1) + steps).clamp_(max=ctx_len - 1)
)
nxt = torch.where(cand >= cu[:-1].unsqueeze(1), ids[cand.clamp_min(0)], old)
context_pool.index_copy_(0, meta.state_slots, nxt.to(context_pool.dtype))
if fla is not None and fla.track_boundary_row is not None:
win = ids[fla.track_boundary_row.unsqueeze(1) - ctx_len + steps]
context_pool.index_copy_(0, fla.track_dst, win.to(context_pool.dtype))
class NGramEmbedding(BaseOP):
"""Hashed n-gram lookup: splitmix64 mix of the last n token ids -> per-head prime vocab -> table rows.
Weight keys (checkpoint names): ``layer_multipliers`` [ngram_size], ``ngram_heads_vocab_sizes``
and ``ngram_heads_offsets`` [num_ngram_heads], all int64. The table itself is NOT a state-dict
entry (128 checkpoint shards land in a ``PLETableBackend``); attach it with ``attach_table``.
"""
def __init__(self, args: Qwen4ExpArgs, table: PLETableBackend | None = None) -> None:
self.ngram_size = args.ngram_size
self.heads_per_ngram = args.heads_per_ngram
self.num_heads = args.num_ngram_heads
self.eos_token_id = args.ngram_boundary_token_id
self.layer_multipliers = torch.empty(args.ngram_size, dtype=torch.int64)
self.ngram_heads_vocab_sizes = torch.empty(self.num_heads, dtype=torch.int64)
self.ngram_heads_offsets = torch.empty(self.num_heads, dtype=torch.int64)
self._table = table
def attach_table(self, table: PLETableBackend) -> None:
self._table = table
@property
def table(self) -> PLETableBackend:
assert self._table is not None, "PLE table backend was never attached"
return self._table
def _window(self, meta: PLEMetadata):
"""The hash window as ``(packed [B, W], select)``, where ``select`` picks this forward's tokens."""
ids = meta.input_ids.long()
ctx_len = self.ngram_size - 1
if meta.is_decode:
# a window of exactly ngram_size columns holds every shift the hash can reach
return torch.cat([meta.ngram_context, ids.view(-1, 1)], dim=1), lambda t: t[:, -1]
num_reqs = len(meta.seq_lens)
width = ctx_len + max(meta.seq_lens)
cu = meta.cu_seqlens.long()
# Pack the ragged tokens into [B, ctx+max_len] so the shift/boundary logic is one gather.
flat_pos = torch.arange(ids.numel(), device=ids.device)
req = (torch.searchsorted(cu, flat_pos, right=True) - 1).clamp_(max=num_reqs - 1)
col = flat_pos - cu[req] + ctx_len
packed = ids.new_full((num_reqs, width), self.eos_token_id)
packed[:, :ctx_len] = meta.ngram_context
packed[req, col] = ids
return packed, lambda t: t[req, col]
def _shift_ignore_eos(self, packed: torch.Tensor) -> List[torch.Tensor]:
"""``out[s][b, p]`` = the token ``s`` places left of ``p``, or eos when the window crosses a boundary."""
num_reqs, width = packed.shape
pos = torch.arange(width, device=packed.device)
eos_pos = torch.where(packed == self.eos_token_id, pos, -1)
prev_eos = torch.cummax(eos_pos, dim=1).values
prev_eos = torch.cat([eos_pos.new_full((num_reqs, 1), -1), prev_eos[:, :-1]], dim=1)
in_segment = pos.unsqueeze(0) - prev_eos - 1
shifted = [packed]
for shift in range(1, self.ngram_size):
src = pos - shift
gathered = packed.gather(1, src.clamp_min(0).unsqueeze(0).expand(num_reqs, -1))
valid = (src.unsqueeze(0) >= 0) & (in_segment >= shift)
shifted.append(torch.where(valid, gathered, packed.new_full((), self.eos_token_id)))
return shifted
def row_ids(self, meta: PLEMetadata) -> torch.Tensor:
"""Global table row per (token, hash head): ``[T, num_ngram_heads]`` int64."""
packed, select = self._window(meta)
tokens = [select(s) for s in self._shift_ignore_eos(packed)]
blocks = []
for ngram in range(2, self.ngram_size + 1):
start = (ngram - 2) * self.heads_per_ngram
end = start + self.heads_per_ngram
mixed = tokens[0] * self.layer_multipliers[0]
for position in range(1, ngram):
mixed = torch.bitwise_xor(mixed, tokens[position] * self.layer_multipliers[position])
head_ids = torch.remainder(mixed.unsqueeze(-1), self.ngram_heads_vocab_sizes[start:end])
blocks.append(head_ids + self.ngram_heads_offsets[start:end])
return torch.cat(blocks, dim=-1)
def forward(self, meta: PLEMetadata, out: torch.Tensor | None = None) -> torch.Tensor:
return self.table.lookup(self.row_ids(meta), out)
class _DepthwiseConv1d(BaseOP):
"""Holds the depthwise conv weight ``[width, 1, kernel]`` (key ``conv1d.weight``)."""
def __init__(self, width: int, kernel: int) -> None:
self.weight = torch.empty(width, 1, kernel)
def short_conv_reference(
x: torch.Tensor,
meta: PLEMetadata,
states: torch.Tensor,
weight: torch.Tensor,
dilation: int,
) -> torch.Tensor:
"""Per-request ``F.conv1d`` over ``[state | chunk]``, advancing ``states`` in place.
Transcription of the HF conv; the shipping paths (one packed conv for prefill, a tap read for
decode) are diffed against it.
"""
groups = weight.shape[0]
state_len = states.shape[-1]
slots = meta.state_slots
state = states.index_select(0, slots).to(x.dtype)
if meta.fresh_slots is not None:
state = torch.where(meta.fresh_slots.view(-1, 1, 1), torch.zeros_like(state), state)
outs = []
new_state = torch.empty_like(state)
offset = 0
for i, n in enumerate(meta.seq_lens):
chunk = x[offset : offset + n].transpose(0, 1).unsqueeze(0)
history = torch.cat([state[i : i + 1], chunk], dim=-1)
out = F.conv1d(history, weight, groups=groups, dilation=dilation)
outs.append(out.squeeze(0).transpose(0, 1))
new_state[i] = history[0, :, -state_len:]
offset += n
states.index_copy_(0, slots, new_state.to(states.dtype))
return F.silu(torch.cat(outs, dim=0))
class PLELayer(BaseOP):
"""PLE block: hashed n-gram value gated by the residual streams, then a dilated depthwise conv.
``forward(R, batch) -> D [T, hc_count*hidden]``; the caller adds ``D`` to ``R`` before the
attention hyper-connection mix. ``meta`` defaults to ``build_ple_metadata(batch, ...)``;
``conv_states`` defaults to ``ctx.linear_state_pool.slot_state("ple_conv", layer_id)`` and is
``[num_slots, hc_count*hidden, (ple_conv_kernel_size-1)*ngram_size]`` in the model dtype -- the
last conv-input columns per request, oldest first. Both are arguments so the reference is
testable before the pool and the scheduler carry them.
``start_prefetch(batch)`` builds the metadata and starts the table gather on the backend's side
stream; call it at the top of the model forward so the rows land while layer 0 runs, and
``forward`` joins it.
Weight keys (checkpoint names, prefix stripped): ``key_proj.weight`` [hc*hidden, ple_embed_dim],
``value_proj.weight`` [hidden, ple_embed_dim], ``norm_key/norm_query/norm_conv.weight``
[hc*hidden] (zero-centered, loaded RAW), ``conv1d.weight`` [hc*hidden, 1, kernel], plus the
three ``ple_embedding`` int64 hash buffers.
"""
def __init__(
self, config: ModelConfig, layer_id: int, table: PLETableBackend | None = None, *, prefix: str = ""
) -> None:
args = config.qwen4_args
self.args = args
self.layer_id = layer_id
self.ple_index = args.ple_layer_ids.index(layer_id)
self.hc_count = args.hc_count
self.hidden_size = args.hidden_size
self.dilation = args.ple_conv_dilation
self.state_len = args.ple_conv_state_len
width = args.ple_state_width
self.ple_embedding = NGramEmbedding(args, table)
self.key_proj = LinearReplicated(
args.ple_embed_dim, width, has_bias=False, quant_config=config.quant, prefix=f"{prefix}.key_proj"
)
self.value_proj = LinearReplicated(
args.ple_embed_dim, args.hidden_size, has_bias=False, quant_config=config.quant, prefix=f"{prefix}.value_proj"
)
self.norm_key = GroupedPlusOneRMSNorm(width, config.rms_norm_eps, self.hc_count)
self.norm_query = GroupedPlusOneRMSNorm(width, config.rms_norm_eps, self.hc_count)
self.norm_conv = GroupedPlusOneRMSNorm(width, config.rms_norm_eps, self.hc_count)
self.conv1d = _DepthwiseConv1d(width, args.ple_conv_kernel_size)
from freetoken.kernel.fla.chunk import CHUNK_SIZE
# the track snapshot gathers the last state_len conv inputs before a xCHUNK boundary; a longer history would reach before the forward's first token
assert self.state_len <= CHUNK_SIZE, (
f"PLE conv history {self.state_len} exceeds CHUNK_SIZE {CHUNK_SIZE}"
)
self._pending: Tuple[PLEMetadata, torch.Tensor] | None = None
def start_prefetch(self, batch: Batch, meta: PLEMetadata | None = None) -> None:
"""Hash this forward's n-grams and start the table gather on the side stream."""
if meta is None:
meta = build_ple_metadata(batch, self.args, batch.input_ids.device)
row_ids = self.ple_embedding.row_ids(meta)
self._pending = (meta, row_ids)
self.ple_embedding.table.prefetch(row_ids)
def forward(
self,
R: torch.Tensor,
batch: Batch,
meta: PLEMetadata | None = None,
conv_states: torch.Tensor | None = None,
) -> torch.Tensor:
pending, self._pending = self._pending, None
row_ids = None
if meta is None:
if pending is not None:
meta, row_ids = pending
else:
meta = build_ple_metadata(batch, self.args, R.device)
elif pending is not None and pending[0] is meta:
row_ids = pending[1]
if row_ids is None:
row_ids = self.ple_embedding.row_ids(meta)
embeddings = self.ple_embedding.table.lookup(row_ids).to(R.dtype)
key = self.norm_key.forward(self.key_proj.forward(embeddings))
value = self.value_proj.forward(embeddings)
query = self.norm_query.forward(R)
shape = (-1, self.hc_count, self.hidden_size)
gate = (key.view(shape) * query.view(shape)).sum(-1, keepdim=True) / math.sqrt(self.hidden_size)
gate = torch.sigmoid(gate.sign() * gate.abs().clamp_min(1e-6).sqrt())
gated = (gate * value.unsqueeze(-2)).flatten(-2)
states = conv_states if conv_states is not None else self._conv_state_slab(R)
x = self.norm_conv.forward(gated)
fla = getattr(batch, "fla_metadata", None)
if fla is not None and fla.track_boundary_row is not None:
self._write_track_snapshot(states, x, fla)
return gated + self._short_conv(x, meta, states)
def _write_track_snapshot(self, states: torch.Tensor, x: torch.Tensor, fla) -> None:
"""Copy the conv history at the GDN track boundary into the same donatable slot, so a radix
prefix hit restores PLE and GDN state together. Track slots never alias the live slots this
forward advances, so the two writes are order-independent."""
src = fla.track_boundary_row.unsqueeze(1) + torch.arange(
-self.state_len, 0, device=x.device
)
window = x[src].transpose(-1, -2).contiguous()
states.index_copy_(0, fla.track_dst, window.to(states.dtype))
def _conv_state_slab(self, R: torch.Tensor) -> torch.Tensor:
pool = get_global_ctx().linear_state_pool
assert pool is not None, "PLE needs ctx.linear_state_pool or an explicit conv_states"
assert pool.has_slot_state(PLE_CONV_STATE), (
"ModelConfig.slot_states does not declare the PLE conv history"
)
return pool.slot_state(PLE_CONV_STATE, self.layer_id)
def _read_state(
self, meta: PLEMetadata, states: torch.Tensor, dtype: torch.dtype
) -> torch.Tensor:
state = states.index_select(0, meta.state_slots).to(dtype)
if meta.fresh_slots is not None:
state = torch.where(meta.fresh_slots.view(-1, 1, 1), torch.zeros_like(state), state)
return state
def _short_conv(
self, x: torch.Tensor, meta: PLEMetadata, states: torch.Tensor
) -> torch.Tensor:
"""silu of the dilated depthwise conv over [state | x], and roll the per-request state."""
if meta.is_decode:
return self._decode_conv(x, meta, states)
return self._prefill_conv(x, meta, states)
def _decode_conv(
self, x: torch.Tensor, meta: PLEMetadata, states: torch.Tensor
) -> torch.Tensor:
"""Batched tap read: taps t-9, t-6, t-3 come off the state slab, tap t from this token."""
state = self._read_state(meta, states, x.dtype)
column = x.unsqueeze(-1)
# fp32 products, like the conv1d the prefill path runs
window = torch.cat([state[..., :: self.dilation], column], dim=-1).float()
out = (window * self.conv1d.weight.squeeze(1).float()).sum(-1)
states.index_copy_(
0, meta.state_slots, torch.cat([state[..., 1:], column], dim=-1).to(states.dtype)
)
return F.silu(out.to(x.dtype))
def _prefill_conv(
self, x: torch.Tensor, meta: PLEMetadata, states: torch.Tensor
) -> torch.Tensor:
"""One conv over every request packed as ``[state_0 | chunk_0 | state_1 | chunk_1 | ...]``.
The blocks abut exactly, so each output window stays inside its own request: request i's
first token reads history columns base_i .. base_i+state_len, which is its own state.
"""
lens = list(meta.seq_lens)
num_reqs, width = len(lens), x.shape[1]
out_index, state_index, next_state_index = self._prefill_indices(lens, x.device)
state = self._read_state(meta, states, x.dtype)
history = x.new_empty(width, x.shape[0] + num_reqs * self.state_len)
history.index_copy_(1, state_index, state.permute(1, 0, 2).reshape(width, -1))
history.index_copy_(1, out_index + self.state_len, x.transpose(0, 1).contiguous())
out = F.conv1d(
history.unsqueeze(0), self.conv1d.weight, groups=width, dilation=self.dilation
).squeeze(0)
new_state = history.index_select(1, next_state_index).view(width, num_reqs, self.state_len)
states.index_copy_(
0, meta.state_slots, new_state.permute(1, 0, 2).to(states.dtype).contiguous()
)
return F.silu(out.index_select(1, out_index).transpose(0, 1))
def _prefill_indices(self, lens: List[int], device: torch.device):
"""Columns of the packed history: this forward's outputs, the state block, the next state block."""
state_len = self.state_len
counts = torch.tensor(lens, dtype=torch.int64)
cu = torch.cat([counts.new_zeros(1), counts.cumsum(0)])
pad = torch.arange(len(lens), dtype=torch.int64) * state_len
base = cu[:-1] + pad
out_index = torch.arange(int(cu[-1])) + torch.repeat_interleave(pad, counts)
span = torch.arange(state_len, dtype=torch.int64)
packed = torch.cat(
[
out_index,
(base.unsqueeze(1) + span).reshape(-1),
((base + counts).unsqueeze(1) + span).reshape(-1),
]
)
if torch.cuda.is_available():
packed = packed.pin_memory()
packed = packed.to(device, non_blocking=True)
n_out, n_state = out_index.numel(), len(lens) * state_len
return packed[:n_out], packed[n_out : n_out + n_state], packed[n_out + n_state :]
__all__ = [
"GpuResidentTable",
"NGramEmbedding",
"ZeroTable",
"PLELayer",
"PLEMetadata",
"PLETableBackend",
"PinnedUVATable",
"build_ple_metadata",
"derive_ngram_hash_constants",
"short_conv_reference",
]