Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

gateway: DMel audio input for Inkling over the OpenAI API

Kyle Kelley779ce88d4486
diff --git a/c/openai_server.py b/c/openai_server.py
index 907ea880..c129714b 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -457,6 +457,170 @@ def split_inkling(text):
return _INK_MARKER.sub("", text), _INK_MARKER.sub("", "".join(reasoning))
+# ---- Inkling DMel audio input ------------------------------------------------------------
+# Inkling takes audio as discretized log-mel frames ("DMel"): 80 slaney mel
+# bands per 50 ms hop, quantized to 16 levels in log10 [-7, 2]. One frame = one
+# <|audio|> placeholder token; the engine swaps in the frame's embedding at that
+# position. The DSP below matches tml-renderers 0.1.0 (via tinkernel-audio,
+# which is byte-golden against the official wheel): 100 ms periodic-Hann window
+# centered on i*hop with zero edge padding, magnitude-domain mel projection,
+# and a turn-level RMS boost for quiet audio (rms < 0.01).
+
+AUDIO_SAMPLE_RATE = 16_000
+AUDIO_HOP = 800
+AUDIO_WINDOW = 1_600
+AUDIO_MEL_BANDS = 80
+AUDIO_DMEL_LEVELS = 16
+AUDIO_DMEL_MIN, AUDIO_DMEL_MAX = -7.0, 2.0
+AUDIO_RMS_FLOOR = 0.01
+AUDIO_LOG_FLOOR = 1.0e-10
+
+_MEL_FILTERS = None
+
+
+def _np():
+ try:
+ import numpy
+ except ImportError:
+ raise APIError(400, "Audio input needs numpy on the gateway (pip install numpy).",
+ None, "unsupported_content_type")
+ return numpy
+
+
+def _mel_filters(np):
+ """Slaney mel filter bank, [80, 801], normalization 2/(upper-lower)."""
+ global _MEL_FILTERS
+ if _MEL_FILTERS is not None:
+ return _MEL_FILTERS
+ fft_freqs = np.arange(AUDIO_WINDOW // 2 + 1, dtype=np.float64) * AUDIO_SAMPLE_RATE / AUDIO_WINDOW
+
+ def hz_to_mel(hz):
+ hz = np.asarray(hz, dtype=np.float64)
+ return np.where(hz >= 1000.0, 15.0 + np.log(np.maximum(hz, 1e-30) / 1000.0) / 0.06875177742094912,
+ hz / 66.66666666666667)
+
+ def mel_to_hz(mel):
+ mel = np.asarray(mel, dtype=np.float64)
+ return np.where(mel >= 15.0, 1000.0 * np.exp(0.06875177742094912 * (mel - 15.0)),
+ 66.66666666666667 * mel)
+
+ max_mel = hz_to_mel(AUDIO_SAMPLE_RATE / 2.0)
+ mel_points = mel_to_hz(np.linspace(0.0, float(max_mel), AUDIO_MEL_BANDS + 2))
+ lower, center, upper = mel_points[:-2], mel_points[1:-1], mel_points[2:]
+ rising = (fft_freqs[None, :] - lower[:, None]) / (center - lower)[:, None]
+ falling = (upper[:, None] - fft_freqs[None, :]) / (upper - center)[:, None]
+ weights = np.maximum(np.minimum(rising, falling), 0.0) * (2.0 / (upper - lower))[:, None]
+ _MEL_FILTERS = weights.astype(np.float32)
+ return _MEL_FILTERS
+
+
+def dmel_encode(samples):
+ """Mono 16 kHz f32 PCM -> u8 DMel bytes, [ceil(n/800), 80] row-major."""
+ np = _np()
+ samples = np.asarray(samples, dtype=np.float32)
+ n = samples.shape[0]
+ if n == 0:
+ raise APIError(400, "Audio clip is empty.", None, "invalid_value")
+ frames = -(-n // AUDIO_HOP)
+ half = AUDIO_WINDOW // 2
+ padded = np.zeros(half + frames * AUDIO_HOP + half, dtype=np.float32)
+ padded[half:half + n] = samples
+ idx = (np.arange(frames)[:, None] * AUDIO_HOP) + np.arange(AUDIO_WINDOW)[None, :]
+ hann = (0.5 - 0.5 * np.cos(2.0 * np.pi * np.arange(AUDIO_WINDOW, dtype=np.float64)
+ / AUDIO_WINDOW)).astype(np.float32)
+ windows = padded[idx] * hann[None, :]
+ sqmag = np.abs(np.fft.rfft(windows, axis=1)) ** 2 # [frames, 801]
+ rms = math.sqrt(float(np.sum(samples.astype(np.float64) ** 2)) / n)
+ scale = AUDIO_RMS_FLOOR / rms if 0.0 < rms < AUDIO_RMS_FLOOR else 1.0
+ mag = np.sqrt(np.maximum(sqmag * (scale * scale), AUDIO_LOG_FLOOR)).astype(np.float32)
+ energy = mag @ _mel_filters(np).T # [frames, 80]
+ logmel = np.log10(np.maximum(energy, AUDIO_LOG_FLOOR))
+ norm = np.clip((np.clip(logmel, AUDIO_DMEL_MIN, AUDIO_DMEL_MAX) - AUDIO_DMEL_MIN)
+ / (AUDIO_DMEL_MAX - AUDIO_DMEL_MIN), 0.0, 1.0)
+ q = np.clip(np.ceil(norm * (AUDIO_DMEL_LEVELS - 1) - 0.5), 0, AUDIO_DMEL_LEVELS - 1)
+ return q.astype(np.uint8).tobytes()
+
+
+def decode_wav_mono16k(data, param):
+ """Minimal RIFF/WAVE reader: PCM16 or float32, any channel count (mixed
+ down), sample rate must already be 16 kHz — resampling belongs at the
+ capture edge, not in the gateway."""
+ import struct as _struct
+ np = _np()
+ if len(data) < 44 or data[:4] != b"RIFF" or data[8:12] != b"WAVE":
+ raise APIError(400, "Audio must be a RIFF/WAVE file.", param, "invalid_value")
+ pos, fmt, raw = 12, None, None
+ while pos + 8 <= len(data):
+ cid, size = data[pos:pos + 4], _struct.unpack_from("<I", data, pos + 4)[0]
+ body = data[pos + 8:pos + 8 + size]
+ if cid == b"fmt ":
+ fmt = _struct.unpack_from("<HHIIHH", body, 0)
+ elif cid == b"data":
+ raw = body
+ pos += 8 + size + (size & 1)
+ if fmt is None or raw is None:
+ raise APIError(400, "WAV file is missing fmt/data chunks.", param, "invalid_value")
+ audio_format, channels, rate, _, _, bits = fmt
+ if audio_format == 0xFFFE: # WAVE_FORMAT_EXTENSIBLE: trust the bit width
+ audio_format = 3 if bits == 32 else 1
+ if rate != AUDIO_SAMPLE_RATE:
+ raise APIError(400, f"Audio must be {AUDIO_SAMPLE_RATE} Hz (got {rate}). "
+ "Resample at the capture edge.", param, "invalid_value")
+ if audio_format == 1 and bits == 16:
+ samples = np.frombuffer(raw, dtype="<i2").astype(np.float32) / 32768.0
+ elif audio_format == 3 and bits == 32:
+ samples = np.frombuffer(raw, dtype="<f4").astype(np.float32)
+ else:
+ raise APIError(400, f"Unsupported WAV encoding (format {audio_format}, {bits}-bit); "
+ "use PCM16 or float32.", param, "invalid_value")
+ if channels > 1:
+ samples = samples[:len(samples) - len(samples) % channels]
+ samples = samples.reshape(-1, channels).mean(axis=1)
+ return samples
+
+
+def inkling_content_segments(content, param, audio_out):
+ """Split OpenAI message content into ordered TMLv0 segments:
+ ("text", str) for merged text runs, ("audio", n_frames) per input_audio
+ part (its DMel bytes appended to audio_out in prompt order)."""
+ if isinstance(content, str):
+ return [("text", content)]
+ if not isinstance(content, list):
+ raise APIError(400, "Message content must be a string or an array of parts.", param)
+ segments = []
+ for index, part in enumerate(content):
+ ptype = part.get("type") if isinstance(part, dict) else None
+ if ptype in ("text", "input_text"):
+ if not isinstance(part.get("text"), str):
+ raise APIError(400, "Text content parts require a string `text` field.",
+ f"{param}.{index}.text")
+ if segments and segments[-1][0] == "text":
+ segments[-1] = ("text", segments[-1][1] + part["text"])
+ else:
+ segments.append(("text", part["text"]))
+ elif ptype == "input_audio":
+ spec = part.get("input_audio")
+ if not isinstance(spec, dict) or not isinstance(spec.get("data"), str):
+ raise APIError(400, "`input_audio` parts need base64 `data`.",
+ f"{param}.{index}.input_audio")
+ if spec.get("format", "wav") != "wav":
+ raise APIError(400, "Only WAV audio is supported (mono, 16 kHz, PCM16/float32).",
+ f"{param}.{index}.input_audio.format", "unsupported_content_type")
+ import base64
+ try:
+ wav = base64.b64decode(spec["data"], validate=True)
+ except Exception:
+ raise APIError(400, "`input_audio.data` is not valid base64.",
+ f"{param}.{index}.input_audio.data")
+ dmel = dmel_encode(decode_wav_mono16k(wav, f"{param}.{index}.input_audio.data"))
+ audio_out.append(dmel)
+ segments.append(("audio", len(dmel) // AUDIO_MEL_BANDS))
+ else:
+ raise APIError(400, "Unsupported content part type for the Inkling engine.",
+ f"{param}.{index}", "unsupported_content_type")
+ return segments
+
+
def render_chat_kimi(messages, enable_thinking=False, reasoning_effort=None, tools=None,
tool_choice=None):
"""Validated multi-turn K3 payload for the C engine.
@@ -494,7 +658,7 @@ def render_chat_kimi(messages, enable_thinking=False, reasoning_effort=None, too
def render_chat_inkling(messages, enable_thinking=False, reasoning_effort=None, tools=None,
- tool_choice=None):
+ tool_choice=None, audio_out=None):
"""Text-only subset of Inkling's chat_template.jinja: role tokens with
<|content_text|> parts and <|end_message|> terminators, an assistant
<|content_model_end_sampling|> after each prior model turn, the
@@ -536,8 +700,20 @@ def render_chat_inkling(messages, enable_thinking=False, reasoning_effort=None,
prompt.append(effort_str)
effort_emitted = True
raw = message.get("content")
- text = content_text(raw, f"messages.{index}.content") if raw is not None else ""
- prompt.append(f"{rtok}<|content_text|>{text}<|end_message|>")
+ if audio_out is not None and role == "user" and isinstance(raw, list):
+ # multipart user content: text runs and audio clips become separate
+ # TMLv0 messages, in part order (a message carries ONE content type).
+ # Each DMel frame is one <|audio|> placeholder; the engine replaces
+ # those embeddings with the frames appended to audio_out.
+ for kind, val in inkling_content_segments(raw, f"messages.{index}.content", audio_out):
+ if kind == "text":
+ prompt.append(f"{rtok}<|content_text|>{val}<|end_message|>")
+ else:
+ prompt.append(f"{rtok}<|content_audio_input|>"
+ + "<|audio|>" * val + "<|audio_end|><|end_message|>")
+ else:
+ text = content_text(raw, f"messages.{index}.content") if raw is not None else ""
+ prompt.append(f"{rtok}<|content_text|>{text}<|end_message|>")
if role == "assistant":
prompt.append("<|content_model_end_sampling|>")
if not effort_emitted: # all-system edge case: fallback
@@ -1339,7 +1515,7 @@ class Engine:
self._fail_pending(error)
def generate(self, prompt, max_tokens, temperature, top_p, on_text, cache_slot=0,
- cancelled=None, grammar=None, stopped=None, on_accept=None):
+ cancelled=None, grammar=None, stopped=None, on_accept=None, audio=None):
if isinstance(cache_slot, bool) or not isinstance(cache_slot, int) or not 0 <= cache_slot < self.kv_slots:
raise APIError(400, "Invalid cache slot.", "cache_slot")
payload = prompt.encode("utf-8")
@@ -1348,6 +1524,12 @@ class Engine:
gpayload = grammar.encode("utf-8") if grammar else b""
if b"\0" in gpayload:
raise APIError(400, "NUL bytes are not supported in grammars.", "response_format")
+ # audio (inkling only): the optional 7th SUBMIT field is grammar bytes
+ # for glm and DMel bytes for inkling — the two engines never see the
+ # other's extension, and inkling rejects grammars upstream.
+ apayload = audio or b""
+ if gpayload and apayload:
+ raise APIError(400, "Grammar and audio cannot be combined.", "response_format")
decoder = codecs.getincrementaldecoder("utf-8")("replace")
def decode(data):
@@ -1366,14 +1548,15 @@ class Engine:
request_id = str(self.next_request_id)
self.next_request_id += 1
self.pending[request_id] = events
+ xpayload = gpayload or apayload
header = (f"SUBMIT {request_id} {cache_slot} {len(payload)} {max_tokens} "
f"{temperature:.8g} {top_p:.8g}"
- + (f" {len(gpayload)}" if gpayload else "") + "\n").encode()
+ + (f" {len(xpayload)}" if xpayload else "") + "\n").encode()
try:
with self.write_lock:
if self.process.poll() is not None:
raise RuntimeError("colibri engine is not running")
- self.process.stdin.write(header + payload + gpayload + b"\n")
+ self.process.stdin.write(header + payload + xpayload + b"\n")
self.process.stdin.flush()
except Exception:
with self.pending_lock:
@@ -1766,7 +1949,7 @@ class APIHandler(BaseHTTPRequestHandler):
return {"type": "error", "error": {"type": error.error_type, "message": error.message}}
def generation(self, body, prompt, request_id, chat, tools=None, tool_choice=None,
- enable_thinking=False):
+ enable_thinking=False, audio=None):
# COLI_DEBUG tees the engine transaction to stderr: 1 = decoded output stream only,
# 2 = both sides (rendered prompt + output). render_chat already folds prior turns and
# tool results into `prompt`, so level 2 is the full conversation the engine saw.
@@ -1821,7 +2004,8 @@ class APIHandler(BaseHTTPRequestHandler):
stop_filter = StopFilter(stop_sequences, output.append, ignore_leading_stop)
stats = self.server.engine.generate(
prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,
- self.client_disconnected, grammar=grammar, stopped=stop_filter.stopped)
+ self.client_disconnected, grammar=grammar, stopped=stop_filter.stopped,
+ **({"audio": audio} if audio else {}))
stop_filter.finish()
text = "".join(output)
reasoning = ""
@@ -1990,7 +2174,7 @@ class APIHandler(BaseHTTPRequestHandler):
stats = self.server.engine.generate(
prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,
lambda: not connected, grammar=grammar, stopped=stop_filter.stopped,
- on_accept=start_stream)
+ on_accept=start_stream, **({"audio": audio} if audio else {}))
stop_filter.finish()
if think:
think.finish()
@@ -2019,7 +2203,7 @@ class APIHandler(BaseHTTPRequestHandler):
stats = self.server.engine.generate(
prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,
lambda: not connected, grammar=grammar, stopped=stop_filter.stopped,
- on_accept=start_stream)
+ on_accept=start_stream, **({"audio": audio} if audio else {}))
stop_filter.finish()
if content_split:
content_split.close()
@@ -2080,10 +2264,16 @@ class APIHandler(BaseHTTPRequestHandler):
tool_choice = body.get("tool_choice")
renderer = (render_chat_inkling if ARCH == "inkling" else
render_chat_kimi if ARCH == "kimi" else render_chat)
- prompt = renderer(body.get("messages"), enable_thinking, reasoning_effort, tools,
- tool_choice)
+ audio_clips = [] if ARCH == "inkling" else None
+ if audio_clips is not None:
+ prompt = renderer(body.get("messages"), enable_thinking, reasoning_effort, tools,
+ tool_choice, audio_out=audio_clips)
+ else:
+ prompt = renderer(body.get("messages"), enable_thinking, reasoning_effort, tools,
+ tool_choice)
self.generation(body, prompt, request_id, True, tools, tool_choice,
- enable_thinking=enable_thinking)
+ enable_thinking=enable_thinking,
+ audio=b"".join(audio_clips) if audio_clips else None)
# ---- Anthropic /v1/messages (#343) ----------------------------------------------------
ANTHROPIC_STOP = {"stop": "end_turn", "length": "max_tokens", "tool_calls": "tool_use"}