Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
gateway: DMel audio input for Inkling over the OpenAI API
Kyle Kelley779ce88d4486
diff --git a/c/openai_server.py b/c/openai_server.pyindex 907ea880..c129714b 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -457,6 +457,170 @@ def split_inkling(text):return _INK_MARKER.sub("", text), _INK_MARKER.sub("", "".join(reasoning))+# ---- Inkling DMel audio input ------------------------------------------------------------+# Inkling takes audio as discretized log-mel frames ("DMel"): 80 slaney mel+# bands per 50 ms hop, quantized to 16 levels in log10 [-7, 2]. One frame = one+# <|audio|> placeholder token; the engine swaps in the frame's embedding at that+# position. The DSP below matches tml-renderers 0.1.0 (via tinkernel-audio,+# which is byte-golden against the official wheel): 100 ms periodic-Hann window+# centered on i*hop with zero edge padding, magnitude-domain mel projection,+# and a turn-level RMS boost for quiet audio (rms < 0.01).++AUDIO_SAMPLE_RATE = 16_000+AUDIO_HOP = 800+AUDIO_WINDOW = 1_600+AUDIO_MEL_BANDS = 80+AUDIO_DMEL_LEVELS = 16+AUDIO_DMEL_MIN, AUDIO_DMEL_MAX = -7.0, 2.0+AUDIO_RMS_FLOOR = 0.01+AUDIO_LOG_FLOOR = 1.0e-10++_MEL_FILTERS = None+++def _np():+ try:+ import numpy+ except ImportError:+ raise APIError(400, "Audio input needs numpy on the gateway (pip install numpy).",+ None, "unsupported_content_type")+ return numpy+++def _mel_filters(np):+ """Slaney mel filter bank, [80, 801], normalization 2/(upper-lower)."""+ global _MEL_FILTERS+ if _MEL_FILTERS is not None:+ return _MEL_FILTERS+ fft_freqs = np.arange(AUDIO_WINDOW // 2 + 1, dtype=np.float64) * AUDIO_SAMPLE_RATE / AUDIO_WINDOW++ def hz_to_mel(hz):+ hz = np.asarray(hz, dtype=np.float64)+ return np.where(hz >= 1000.0, 15.0 + np.log(np.maximum(hz, 1e-30) / 1000.0) / 0.06875177742094912,+ hz / 66.66666666666667)++ def mel_to_hz(mel):+ mel = np.asarray(mel, dtype=np.float64)+ return np.where(mel >= 15.0, 1000.0 * np.exp(0.06875177742094912 * (mel - 15.0)),+ 66.66666666666667 * mel)++ max_mel = hz_to_mel(AUDIO_SAMPLE_RATE / 2.0)+ mel_points = mel_to_hz(np.linspace(0.0, float(max_mel), AUDIO_MEL_BANDS + 2))+ lower, center, upper = mel_points[:-2], mel_points[1:-1], mel_points[2:]+ rising = (fft_freqs[None, :] - lower[:, None]) / (center - lower)[:, None]+ falling = (upper[:, None] - fft_freqs[None, :]) / (upper - center)[:, None]+ weights = np.maximum(np.minimum(rising, falling), 0.0) * (2.0 / (upper - lower))[:, None]+ _MEL_FILTERS = weights.astype(np.float32)+ return _MEL_FILTERS+++def dmel_encode(samples):+ """Mono 16 kHz f32 PCM -> u8 DMel bytes, [ceil(n/800), 80] row-major."""+ np = _np()+ samples = np.asarray(samples, dtype=np.float32)+ n = samples.shape[0]+ if n == 0:+ raise APIError(400, "Audio clip is empty.", None, "invalid_value")+ frames = -(-n // AUDIO_HOP)+ half = AUDIO_WINDOW // 2+ padded = np.zeros(half + frames * AUDIO_HOP + half, dtype=np.float32)+ padded[half:half + n] = samples+ idx = (np.arange(frames)[:, None] * AUDIO_HOP) + np.arange(AUDIO_WINDOW)[None, :]+ hann = (0.5 - 0.5 * np.cos(2.0 * np.pi * np.arange(AUDIO_WINDOW, dtype=np.float64)+ / AUDIO_WINDOW)).astype(np.float32)+ windows = padded[idx] * hann[None, :]+ sqmag = np.abs(np.fft.rfft(windows, axis=1)) ** 2 # [frames, 801]+ rms = math.sqrt(float(np.sum(samples.astype(np.float64) ** 2)) / n)+ scale = AUDIO_RMS_FLOOR / rms if 0.0 < rms < AUDIO_RMS_FLOOR else 1.0+ mag = np.sqrt(np.maximum(sqmag * (scale * scale), AUDIO_LOG_FLOOR)).astype(np.float32)+ energy = mag @ _mel_filters(np).T # [frames, 80]+ logmel = np.log10(np.maximum(energy, AUDIO_LOG_FLOOR))+ norm = np.clip((np.clip(logmel, AUDIO_DMEL_MIN, AUDIO_DMEL_MAX) - AUDIO_DMEL_MIN)+ / (AUDIO_DMEL_MAX - AUDIO_DMEL_MIN), 0.0, 1.0)+ q = np.clip(np.ceil(norm * (AUDIO_DMEL_LEVELS - 1) - 0.5), 0, AUDIO_DMEL_LEVELS - 1)+ return q.astype(np.uint8).tobytes()+++def decode_wav_mono16k(data, param):+ """Minimal RIFF/WAVE reader: PCM16 or float32, any channel count (mixed+ down), sample rate must already be 16 kHz — resampling belongs at the+ capture edge, not in the gateway."""+ import struct as _struct+ np = _np()+ if len(data) < 44 or data[:4] != b"RIFF" or data[8:12] != b"WAVE":+ raise APIError(400, "Audio must be a RIFF/WAVE file.", param, "invalid_value")+ pos, fmt, raw = 12, None, None+ while pos + 8 <= len(data):+ cid, size = data[pos:pos + 4], _struct.unpack_from("<I", data, pos + 4)[0]+ body = data[pos + 8:pos + 8 + size]+ if cid == b"fmt ":+ fmt = _struct.unpack_from("<HHIIHH", body, 0)+ elif cid == b"data":+ raw = body+ pos += 8 + size + (size & 1)+ if fmt is None or raw is None:+ raise APIError(400, "WAV file is missing fmt/data chunks.", param, "invalid_value")+ audio_format, channels, rate, _, _, bits = fmt+ if audio_format == 0xFFFE: # WAVE_FORMAT_EXTENSIBLE: trust the bit width+ audio_format = 3 if bits == 32 else 1+ if rate != AUDIO_SAMPLE_RATE:+ raise APIError(400, f"Audio must be {AUDIO_SAMPLE_RATE} Hz (got {rate}). "+ "Resample at the capture edge.", param, "invalid_value")+ if audio_format == 1 and bits == 16:+ samples = np.frombuffer(raw, dtype="<i2").astype(np.float32) / 32768.0+ elif audio_format == 3 and bits == 32:+ samples = np.frombuffer(raw, dtype="<f4").astype(np.float32)+ else:+ raise APIError(400, f"Unsupported WAV encoding (format {audio_format}, {bits}-bit); "+ "use PCM16 or float32.", param, "invalid_value")+ if channels > 1:+ samples = samples[:len(samples) - len(samples) % channels]+ samples = samples.reshape(-1, channels).mean(axis=1)+ return samples+++def inkling_content_segments(content, param, audio_out):+ """Split OpenAI message content into ordered TMLv0 segments:+ ("text", str) for merged text runs, ("audio", n_frames) per input_audio+ part (its DMel bytes appended to audio_out in prompt order)."""+ if isinstance(content, str):+ return [("text", content)]+ if not isinstance(content, list):+ raise APIError(400, "Message content must be a string or an array of parts.", param)+ segments = []+ for index, part in enumerate(content):+ ptype = part.get("type") if isinstance(part, dict) else None+ if ptype in ("text", "input_text"):+ if not isinstance(part.get("text"), str):+ raise APIError(400, "Text content parts require a string `text` field.",+ f"{param}.{index}.text")+ if segments and segments[-1][0] == "text":+ segments[-1] = ("text", segments[-1][1] + part["text"])+ else:+ segments.append(("text", part["text"]))+ elif ptype == "input_audio":+ spec = part.get("input_audio")+ if not isinstance(spec, dict) or not isinstance(spec.get("data"), str):+ raise APIError(400, "`input_audio` parts need base64 `data`.",+ f"{param}.{index}.input_audio")+ if spec.get("format", "wav") != "wav":+ raise APIError(400, "Only WAV audio is supported (mono, 16 kHz, PCM16/float32).",+ f"{param}.{index}.input_audio.format", "unsupported_content_type")+ import base64+ try:+ wav = base64.b64decode(spec["data"], validate=True)+ except Exception:+ raise APIError(400, "`input_audio.data` is not valid base64.",+ f"{param}.{index}.input_audio.data")+ dmel = dmel_encode(decode_wav_mono16k(wav, f"{param}.{index}.input_audio.data"))+ audio_out.append(dmel)+ segments.append(("audio", len(dmel) // AUDIO_MEL_BANDS))+ else:+ raise APIError(400, "Unsupported content part type for the Inkling engine.",+ f"{param}.{index}", "unsupported_content_type")+ return segments++def render_chat_kimi(messages, enable_thinking=False, reasoning_effort=None, tools=None,tool_choice=None):"""Validated multi-turn K3 payload for the C engine.@@ -494,7 +658,7 @@ def render_chat_kimi(messages, enable_thinking=False, reasoning_effort=None, toodef render_chat_inkling(messages, enable_thinking=False, reasoning_effort=None, tools=None,- tool_choice=None):+ tool_choice=None, audio_out=None):"""Text-only subset of Inkling's chat_template.jinja: role tokens with<|content_text|> parts and <|end_message|> terminators, an assistant<|content_model_end_sampling|> after each prior model turn, the@@ -536,8 +700,20 @@ def render_chat_inkling(messages, enable_thinking=False, reasoning_effort=None,prompt.append(effort_str)effort_emitted = Trueraw = message.get("content")- text = content_text(raw, f"messages.{index}.content") if raw is not None else ""- prompt.append(f"{rtok}<|content_text|>{text}<|end_message|>")+ if audio_out is not None and role == "user" and isinstance(raw, list):+ # multipart user content: text runs and audio clips become separate+ # TMLv0 messages, in part order (a message carries ONE content type).+ # Each DMel frame is one <|audio|> placeholder; the engine replaces+ # those embeddings with the frames appended to audio_out.+ for kind, val in inkling_content_segments(raw, f"messages.{index}.content", audio_out):+ if kind == "text":+ prompt.append(f"{rtok}<|content_text|>{val}<|end_message|>")+ else:+ prompt.append(f"{rtok}<|content_audio_input|>"+ + "<|audio|>" * val + "<|audio_end|><|end_message|>")+ else:+ text = content_text(raw, f"messages.{index}.content") if raw is not None else ""+ prompt.append(f"{rtok}<|content_text|>{text}<|end_message|>")if role == "assistant":prompt.append("<|content_model_end_sampling|>")if not effort_emitted: # all-system edge case: fallback@@ -1339,7 +1515,7 @@ class Engine:self._fail_pending(error)def generate(self, prompt, max_tokens, temperature, top_p, on_text, cache_slot=0,- cancelled=None, grammar=None, stopped=None, on_accept=None):+ cancelled=None, grammar=None, stopped=None, on_accept=None, audio=None):if isinstance(cache_slot, bool) or not isinstance(cache_slot, int) or not 0 <= cache_slot < self.kv_slots:raise APIError(400, "Invalid cache slot.", "cache_slot")payload = prompt.encode("utf-8")@@ -1348,6 +1524,12 @@ class Engine:gpayload = grammar.encode("utf-8") if grammar else b""if b"\0" in gpayload:raise APIError(400, "NUL bytes are not supported in grammars.", "response_format")+ # audio (inkling only): the optional 7th SUBMIT field is grammar bytes+ # for glm and DMel bytes for inkling — the two engines never see the+ # other's extension, and inkling rejects grammars upstream.+ apayload = audio or b""+ if gpayload and apayload:+ raise APIError(400, "Grammar and audio cannot be combined.", "response_format")decoder = codecs.getincrementaldecoder("utf-8")("replace")def decode(data):@@ -1366,14 +1548,15 @@ class Engine:request_id = str(self.next_request_id)self.next_request_id += 1self.pending[request_id] = events+ xpayload = gpayload or apayloadheader = (f"SUBMIT {request_id} {cache_slot} {len(payload)} {max_tokens} "f"{temperature:.8g} {top_p:.8g}"- + (f" {len(gpayload)}" if gpayload else "") + "\n").encode()+ + (f" {len(xpayload)}" if xpayload else "") + "\n").encode()try:with self.write_lock:if self.process.poll() is not None:raise RuntimeError("colibri engine is not running")- self.process.stdin.write(header + payload + gpayload + b"\n")+ self.process.stdin.write(header + payload + xpayload + b"\n")self.process.stdin.flush()except Exception:with self.pending_lock:@@ -1766,7 +1949,7 @@ class APIHandler(BaseHTTPRequestHandler):return {"type": "error", "error": {"type": error.error_type, "message": error.message}}def generation(self, body, prompt, request_id, chat, tools=None, tool_choice=None,- enable_thinking=False):+ enable_thinking=False, audio=None):# COLI_DEBUG tees the engine transaction to stderr: 1 = decoded output stream only,# 2 = both sides (rendered prompt + output). render_chat already folds prior turns and# tool results into `prompt`, so level 2 is the full conversation the engine saw.@@ -1821,7 +2004,8 @@ class APIHandler(BaseHTTPRequestHandler):stop_filter = StopFilter(stop_sequences, output.append, ignore_leading_stop)stats = self.server.engine.generate(prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,- self.client_disconnected, grammar=grammar, stopped=stop_filter.stopped)+ self.client_disconnected, grammar=grammar, stopped=stop_filter.stopped,+ **({"audio": audio} if audio else {}))stop_filter.finish()text = "".join(output)reasoning = ""@@ -1990,7 +2174,7 @@ class APIHandler(BaseHTTPRequestHandler):stats = self.server.engine.generate(prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,lambda: not connected, grammar=grammar, stopped=stop_filter.stopped,- on_accept=start_stream)+ on_accept=start_stream, **({"audio": audio} if audio else {}))stop_filter.finish()if think:think.finish()@@ -2019,7 +2203,7 @@ class APIHandler(BaseHTTPRequestHandler):stats = self.server.engine.generate(prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,lambda: not connected, grammar=grammar, stopped=stop_filter.stopped,- on_accept=start_stream)+ on_accept=start_stream, **({"audio": audio} if audio else {}))stop_filter.finish()if content_split:content_split.close()@@ -2080,10 +2264,16 @@ class APIHandler(BaseHTTPRequestHandler):tool_choice = body.get("tool_choice")renderer = (render_chat_inkling if ARCH == "inkling" elserender_chat_kimi if ARCH == "kimi" else render_chat)- prompt = renderer(body.get("messages"), enable_thinking, reasoning_effort, tools,- tool_choice)+ audio_clips = [] if ARCH == "inkling" else None+ if audio_clips is not None:+ prompt = renderer(body.get("messages"), enable_thinking, reasoning_effort, tools,+ tool_choice, audio_out=audio_clips)+ else:+ prompt = renderer(body.get("messages"), enable_thinking, reasoning_effort, tools,+ tool_choice)self.generation(body, prompt, request_id, True, tools, tool_choice,- enable_thinking=enable_thinking)+ enable_thinking=enable_thinking,+ audio=b"".join(audio_clips) if audio_clips else None)# ---- Anthropic /v1/messages (#343) ----------------------------------------------------ANTHROPIC_STOP = {"stop": "end_turn", "length": "max_tokens", "tool_calls": "tool_use"}