Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

44.9 KB
Raw Blame
<!doctype html>
<html lang="en">
  <head>
    <meta charset="utf-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1" />
    <title>Ingestion consumer · routing debug</title>
    <style>
      :root {
        --bg: #14161a;
        --panel: #1c1f26;
        --panel-2: #23272f;
        --border: #2c313b;
        --text: #e6e9ef;
        --muted: #8b93a1;
        --accent: #4f7cff;
        --good: #3fb950;
        --warn: #d29922;
        --error: #f85149;
        --mono: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace;
      }
      * { box-sizing: border-box; }
      body {
        margin: 0;
        background: var(--bg);
        color: var(--text);
        font: 13px/1.5 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
      }
      header {
        display: flex; align-items: baseline; gap: 16px;
        padding: 12px 18px; border-bottom: 1px solid var(--border);
        background: var(--panel); position: sticky; top: 0; z-index: 5; flex-wrap: wrap;
      }
      header h1 { font-size: 15px; margin: 0; font-weight: 600; }
      header .meta { color: var(--muted); font-family: var(--mono); font-size: 12px; }
      .dot { display: inline-block; width: 8px; height: 8px; border-radius: 50%; margin-right: 6px; }
      .dot.live { background: var(--good); box-shadow: 0 0 6px var(--good); }
      .dot.down { background: var(--error); }
      .layout { display: grid; grid-template-columns: minmax(0, 360px) minmax(0, 1fr); gap: 14px; padding: 14px; }
      @media (max-width: 900px) { .layout { grid-template-columns: 1fr; } }
      .panel { background: var(--panel); border: 1px solid var(--border); border-radius: 8px; overflow: hidden; }
      .panel > h2 {
        font-size: 12px; text-transform: uppercase; letter-spacing: 0.06em;
        color: var(--muted); margin: 0; padding: 10px 14px; border-bottom: 1px solid var(--border);
        display: flex; justify-content: space-between; align-items: center;
      }
      .stats { display: flex; flex-wrap: wrap; gap: 10px; padding: 12px 14px; }
      .stat { background: var(--panel-2); border: 1px solid var(--border); border-radius: 6px; padding: 8px 12px; min-width: 84px; }
      .stat .n { font-size: 20px; font-weight: 600; font-family: var(--mono); }
      .stat .l { font-size: 11px; color: var(--muted); }
      table { width: 100%; border-collapse: collapse; }
      th, td { text-align: left; padding: 7px 14px; border-bottom: 1px solid var(--border); font-size: 12px; }
      th { color: var(--muted); font-weight: 500; font-size: 11px; text-transform: uppercase; letter-spacing: 0.04em; }
      td.mono, th.mono { font-family: var(--mono); }
      td .bar { height: 6px; border-radius: 3px; background: var(--accent); min-width: 2px; }
      .pill { display: inline-block; padding: 1px 7px; border-radius: 10px; font-size: 11px; font-weight: 600; white-space: nowrap; }
      .pill.healthy, .pill.committed, .pill.completed { background: rgba(63,185,80,0.16); color: var(--good); }
      .pill.degraded, .pill.retrying, .pill.partial { background: rgba(210,153,34,0.16); color: var(--warn); }
      .pill.unhealthy, .pill.stalled, .pill.failed { background: rgba(248,81,73,0.16); color: var(--error); }
      .pill.draining, .pill.inflight { background: rgba(139,147,161,0.18); color: var(--muted); }
      .pill.accent { background: rgba(79,124,255,0.18); color: #a9c0ff; }

      /* tabs */
      .tabs { display: flex; gap: 4px; padding: 8px 14px 0; border-bottom: 1px solid var(--border); background: var(--panel); }
      .tab { padding: 7px 14px; font-size: 12px; color: var(--muted); cursor: pointer; border: 1px solid transparent; border-bottom: none; border-radius: 6px 6px 0 0; }
      .tab.active { color: var(--text); background: var(--panel-2); border-color: var(--border); }
      .tab .badge { color: var(--muted); font-family: var(--mono); font-size: 11px; margin-left: 4px; }

      /* live feed: a bounded tail — fixed height, not vh-based, so it stays
         stable when the page is embedded in an auto-sizing iframe. */
      #feed { max-height: 600px; overflow-y: auto; font-family: var(--mono); font-size: 12px; }
      .ev { padding: 5px 14px; border-bottom: 1px solid var(--border); display: flex; gap: 10px; align-items: baseline; }
      .ev .t { color: var(--muted); flex: 0 0 62px; font-size: 11px; }
      .ev .body { flex: 1 1 auto; word-break: break-word; }
      .ev.accent .body { color: #a9c0ff; }
      .ev.good .body { color: #7ee787; }
      .ev.warn .body { color: #e3b341; }
      .ev.error .body { color: #ff7b72; }
      .ev .tag { color: var(--muted); font-size: 11px; }
      code { color: #d2a8ff; }
      .empty { padding: 18px 14px; color: var(--muted); }
      .controls { display: flex; gap: 8px; align-items: center; }
      .controls label { color: var(--muted); font-size: 11px; }

      /* polls list: grows with content — the page (or embedding iframe)
         expands instead of squeezing the list into an inner scroll box. */
      #polls-view { overflow-y: visible; }
      .brow { display: grid; grid-template-columns: 96px 90px 1fr auto; gap: 10px; align-items: center; padding: 8px 14px; border-bottom: 1px solid var(--border); cursor: pointer; }
      .brow:hover { background: var(--panel-2); }
      .brow .id { font-family: var(--mono); color: #d2a8ff; }
      .brow .badges { display: flex; gap: 5px; flex-wrap: wrap; justify-content: flex-end; }
      .badge-min { font-size: 10px; padding: 1px 6px; border-radius: 8px; background: var(--panel-2); color: var(--muted); border: 1px solid var(--border); white-space: nowrap; }
      .badge-min.warn { color: var(--warn); border-color: rgba(210,153,34,0.4); }
      .badge-min.error { color: var(--error); border-color: rgba(248,81,73,0.4); }
      .brow .tm { color: var(--muted); font-family: var(--mono); font-size: 11px; }

      /* batch detail */
      .detail { padding: 14px; }
      .back { color: var(--accent); cursor: pointer; font-size: 12px; margin-bottom: 12px; display: inline-block; }
      .dsum { display: flex; flex-wrap: wrap; gap: 10px; margin-bottom: 16px; }
      .diagram { display: flex; align-items: stretch; gap: 14px; flex-wrap: wrap; margin-bottom: 18px; }
      .node { background: var(--panel-2); border: 1px solid var(--border); border-radius: 8px; padding: 10px 12px; min-width: 150px; }
      .node .title { font-size: 11px; text-transform: uppercase; letter-spacing: 0.05em; color: var(--muted); margin-bottom: 6px; }
      .node .big { font-family: var(--mono); font-size: 15px; }
      .node.worker.acked { border-color: rgba(63,185,80,0.4); }
      .node.worker.stalled { border-color: rgba(248,81,73,0.5); }
      .arrow { align-self: center; color: var(--muted); font-size: 20px; }
      .wgrid { display: flex; gap: 12px; flex-wrap: wrap; }

      /* flow graph */
      .flow { position: relative; padding: 6px 4px 10px; overflow-x: auto; margin-bottom: 18px; }
      .flow-main { display: flex; gap: 82px; align-items: center; }
      .flow-col { display: flex; flex-direction: column; gap: 16px; justify-content: center; }
      .flow .node { position: relative; z-index: 1; min-width: 122px; }
      .flow svg { position: absolute; top: 0; left: 0; z-index: 0; pointer-events: none; overflow: visible; }
      .node.src { border-color: rgba(79,124,255,0.45); }
      .node.sink-ok { border-color: rgba(63,185,80,0.5); }
      .node.sink-pending { opacity: 0.75; }
      .edge-label { font-size: 10px; font-family: var(--mono); }
      .flow-legend { display: flex; flex-wrap: wrap; gap: 14px; margin-bottom: 10px; color: var(--muted); font-size: 11px; }
      .flow-legend i { display: inline-block; width: 12px; height: 3px; border-radius: 2px; margin-right: 5px; vertical-align: middle; }
      .node .line { font-size: 12px; color: var(--text); }
      .node .sub { font-size: 11px; color: var(--muted); }
      .tl { border-left: 2px solid var(--border); margin-left: 6px; padding-left: 14px; }
      .tl .item { position: relative; padding: 4px 0; font-size: 12px; }
      .tl .item::before { content: ""; position: absolute; left: -20px; top: 9px; width: 8px; height: 8px; border-radius: 50%; background: var(--accent); }
      .tl .item.good::before { background: var(--good); }
      .tl .item.warn::before { background: var(--warn); }
      .tl .item.error::before { background: var(--error); }
      .tl .item .t { color: var(--muted); font-family: var(--mono); font-size: 11px; margin-right: 8px; }

      /* commit submissions */
      .banner { margin: 12px 14px 0; padding: 8px 12px; border-radius: 6px; font-size: 12px; }
      .banner.ok { background: rgba(63,185,80,0.12); color: var(--good); }
      .banner.bad { background: rgba(248,81,73,0.14); color: var(--error); font-weight: 600; }
      #commits { overflow-y: visible; }
      tr.ooo td { background: rgba(248,81,73,0.12); }
      .offchips { display: flex; flex-wrap: wrap; gap: 6px; }
      .offchip { display: inline-flex; align-items: baseline; gap: 6px; padding: 2px 8px; border-radius: 6px;
        background: var(--panel-2); border: 1px solid var(--border); font-family: var(--mono); font-size: 12px; }
      .offchip .pl { color: var(--muted); font-size: 11px; }
      .offchip .ov { color: var(--text); }
      .offchip.bad { background: rgba(248,81,73,0.14); border-color: rgba(248,81,73,0.5); }
      .offchip.bad .ov { color: var(--error); font-weight: 700; }
      .offchip .was { color: var(--error); font-size: 11px; }
    </style>
  </head>
  <body>
    <header>
      <h1>Ingestion consumer</h1>
      <span class="meta" id="group">group —</span>
      <span class="meta" id="routing">routing —</span>
      <span class="meta"><span class="dot down" id="conn-dot"></span><span id="conn">connecting…</span></span>
    </header>

    <div class="layout">
      <!-- left: live state -->
      <div style="display:flex; flex-direction:column; gap:14px;">
        <section class="panel">
          <h2>Batcher load</h2>
          <div class="stats" id="disp-stats"></div>
        </section>
        <section class="panel">
          <h2>Workers</h2>
          <table>
            <thead>
              <tr><th>Worker</th><th>State</th><th>Requests</th><th>Messages</th><th>Fails</th><th>Err</th></tr>
            </thead>
            <tbody id="workers"><tr><td class="empty" colspan="6">Waiting for data…</td></tr></tbody>
          </table>
        </section>
      </div>

      <!-- right: tabbed detail -->
      <section class="panel">
        <div class="tabs">
          <div class="tab active" data-tab="feed">Live feed</div>
          <div class="tab" data-tab="polls">Polls <span class="badge" id="badge-polls"></span></div>
          <div class="tab" data-tab="commits">Commit submissions <span class="badge" id="badge-commits"></span></div>
        </div>

        <div id="tab-feed">
          <h2>
            Event stream
            <span class="controls">
              <label><input type="checkbox" id="pause" /> pause</label>
            </span>
          </h2>
          <div id="feed"><div class="empty">Waiting for events…</div></div>
        </div>

        <div id="tab-polls" style="display:none;">
          <h2>
            Polls
            <span class="controls">
              <label><input type="checkbox" id="polls-pause" /> pause list</label>
              <label><input type="checkbox" id="polls-issues" /> issues only</label>
            </span>
          </h2>
          <div id="polls-view"><div class="empty">Waiting for polls…</div></div>
        </div>

        <div id="tab-commits" style="display:none;">
          <div class="empty">Async commit submissions accepted by the Kafka client, not broker confirmations. Offsets are Kafka next-to-read positions and may span multiple polls.</div>
          <div id="commit-banner"></div>
          <div id="commits"><div class="empty">Waiting for commit submissions…</div></div>
        </div>
      </section>
    </div>

    <script>
      const MAX_FEED = 600;
      const MAX_POLLS = 400;
      const MAX_REQUESTS = 2000;
      const MAX_POLL_EVENTS = 500;
      const MAX_COMMITS = 400;

      const seen = new Set();
      let feedEmpty = true;
      let activeTab = "feed";
      let selectedPoll = null;
      let openDetailId = null;

      // client-side models rebuilt from the event stream
      let polls = {};
      let pollOrder = [];
      let requests = {};
      let requestOrder = [];
      let commits = [];
      let partitionHigh = {}; // "topic/partition" -> highest submitted next-to-read offset
      let oooCount = 0;
      let renderPending = false;

      const $ = (id) => document.getElementById(id);

      // Replace a scroll container's contents without losing the user's place.
      // "prepend": newest rows arrive at the top, so keep the rows currently in
      // view fixed (unless the user is already at the top, where we follow the
      // newest). "append": content grows below the viewport, so just restore.
      function setHTML(el, html, mode) {
        if (!mode || mode === "none") { el.innerHTML = html; return; }
        const prevTop = el.scrollTop;
        const prevHeight = el.scrollHeight;
        const nearTop = prevTop <= 4;
        el.innerHTML = html;
        if (mode === "prepend") el.scrollTop = nearTop ? 0 : prevTop + (el.scrollHeight - prevHeight);
        else el.scrollTop = prevTop;
      }

      const shortId = (id) => (id ? id.slice(-8) : "—");
      const shortWorker = (w) => (w || "").replace(/^https?:\/\//, "");
      const fmtTime = (ms) => {
        const d = new Date(ms);
        return d.toLocaleTimeString("en-US", { hour12: false }) + "." + String(d.getMilliseconds()).padStart(3, "0");
      };
      const fmtBytes = (n) => (n < 1024 ? `${n} B` : n < 1048576 ? `${(n / 1024).toFixed(1)} KB` : `${(n / 1048576).toFixed(1)} MB`);
      const fmtDur = (ms) => {
        if (ms == null || !isFinite(ms) || ms < 0) return "—";
        if (ms < 1000) return `${Math.round(ms)}ms`;
        // Round before picking the unit so 59,999ms doesn't render as "60.0s"
        // or 119,999ms as "1m 60s".
        if (ms < 60000) {
          const seconds = Math.round(ms / 100) / 10;
          if (seconds < 60) return `${seconds.toFixed(1)}s`;
        }
        const totalSeconds = Math.round(ms / 1000);
        return `${Math.floor(totalSeconds / 60)}m ${String(totalSeconds % 60).padStart(2, "0")}s`;
      };

      // ---------- point-in-time load (polled fast, no event backlog) ----------
      let loadSamples = []; // rolling {t, v} of total in-flight, for a peak-hold
      const PEAK_WINDOW_MS = 4000;

      async function refreshState() {
        let s;
        try { s = await (await fetch("debug/load")).json(); } catch (e) { return; }
        $("group").textContent = "group " + s.group_id;

        // Routing summary: strategy, and under aperture this pod's peer
        // position plus its current ring slice (or the full-pool fallback).
        const r = s.routing || {};
        let routingText = "routing " + (r.strategy || "—");
        if (r.strategy === "aperture") {
          routingText += s.peer_index != null && s.peer_count
            ? ` · peer ${s.peer_index + 1}/${s.peer_count}`
            : " · peers unknown";
          routingText += r.slice
            ? ` · slice ${r.slice.length}/${r.ring.length}`
            : " · full-pool fallback";
        }
        $("routing").textContent = routingText;
        const sliceSet = new Set(r.slice || []);

        // A consumer pod from before the batcher rename still answers with
        // `dispatcher`; read whichever key the pod sends during a rollout.
        const d = s.batcher || s.dispatcher || {};
        // In-flight is transient (a request is sent→resolved in ms), so a
        // single sample often reads 0. Hold the peak over a short window so the
        // panel reflects the concurrency actually happening.
        const now = Date.now();
        loadSamples.push({ t: now, v: d.total_in_flight ?? 0 });
        loadSamples = loadSamples.filter((x) => now - x.t <= PEAK_WINDOW_MS);
        const peak = loadSamples.reduce((m, x) => Math.max(m, x.v), 0);

        $("disp-stats").innerHTML = [
          ["in-flight", d.total_in_flight ?? 0],
          ["peak 4s", peak],
          ["requests", d.in_flight_requests ?? 0],
          ["pending", d.pending_messages ?? 0],
          ["queued", fmtBytes(d.queued_bytes ?? 0)],
          ["keys", d.keys ?? 0],
          ["claimed", d.claimed_keys ?? 0],
          ["waiting", d.waiting_keys ?? 0],
        ].map(([l, n]) => `<div class="stat"><div class="n">${n}</div><div class="l">${l}</div></div>`).join("");

        const maxLoad = Math.max(1, ...s.workers.map((w) => w.in_flight_messages ?? 0));
        // Stable order — slice members first, then by IP. Sorting by load made
        // rows jump around on every poll.
        const workerSortKey = (url) => {
          const host = (url.match(/^[a-z]+:\/\/([^:/]+)/i) || [])[1] || url;
          // Zero-pad IPv4 octets so "10.2.30.1" sorts numerically, not lexically.
          return /^\d+(\.\d+){3}$/.test(host)
            ? host.split(".").map((octet) => octet.padStart(3, "0")).join(".")
            : host;
        };
        $("workers").innerHTML = s.workers.length
          ? s.workers.sort((a, b) =>
              (sliceSet.has(b.url) - sliceSet.has(a.url)) ||
              workerSortKey(a.url).localeCompare(workerSortKey(b.url))).map((w) => {
              const cls = w.draining ? "draining" : w.state;
              const label = w.draining ? "draining" : w.state;
              const rate = (w.passive_error_rate * 100).toFixed(0) + "%";
              const inFlight = w.in_flight_messages ?? 0;
              const pct = Math.round((inFlight / maxLoad) * 100);
              const slice = sliceSet.has(w.url) ? ` <span class="badge-min" title="in this consumer's aperture slice">slice</span>` : "";
              return `<tr>
                <td class="mono">${shortWorker(w.url)}${slice}</td>
                <td><span class="pill ${cls}">${label}</span></td>
                <td class="mono">${w.in_flight_requests ?? 0}</td>
                <td class="mono"><div style="display:flex;align-items:center;gap:8px;">
                  <div class="bar" style="width:${pct}%"></div><span>${inFlight}</span></div></td>
                <td class="mono">${w.consecutive_probe_failures}</td>
                <td class="mono">${rate}</td>
              </tr>`;
            }).join("")
          : `<tr><td class="empty" colspan="6">No workers</td></tr>`;
      }

      // ---------- event → model ----------
      // A poll is one Kafka batch the consumer submitted; a request is one
      // send the batcher made. A request can merge messages from consecutive
      // polls, so the two are matched by the topic-partition offset ranges
      // both events carry.
      function resetModels() {
        seen.clear();
        polls = {}; pollOrder = []; requests = {}; requestOrder = []; commits = []; partitionHigh = {}; oooCount = 0;
        selectedPoll = null;
        $("feed").innerHTML = '<div class="empty">Waiting for events…</div>';
        feedEmpty = true;
        scheduleRender();
      }

      function pollFor(id) {
        let p = polls[id];
        if (!p) {
          p = { id, ts: null, messages: 0, partitions: [], assignment_epoch: null, spans: [], requests: [],
                completed: null, failed: null, events: [] };
          polls[id] = p;
          pollOrder.push(id);
          while (pollOrder.length > MAX_POLLS) delete polls[pollOrder.shift()];
        }
        return p;
      }

      function requestFor(id) {
        let r = requests[id];
        if (!r) {
          r = { id, ts: null, worker: null, messages: 0, routing_keys: 0, bytes: 0, replay: false, budgeted: false,
                assignment_epoch: null, spans: [], polls: [], resolved: null, failed: null, events: [] };
          requests[id] = r;
          requestOrder.push(id);
          if (requestOrder.length > MAX_REQUESTS) evictRequest();
        }
        return r;
      }

      // Drop the oldest settled request, and only an unsettled one when
      // nothing settled is left: its response is still to come, and the poll
      // that carries it would otherwise lose the outcome.
      function evictRequest() {
        let index = requestOrder.findIndex((id) => { const r = requests[id]; return !r || r.resolved || r.failed; });
        if (index < 0) index = 0;
        const [id] = requestOrder.splice(index, 1);
        const r = requests[id];
        delete requests[id];
        if (!r) return;
        r.polls.forEach((pid) => {
          const p = polls[pid];
          if (!p) return;
          p.requests = p.requests.filter((rid) => rid !== id);
          p.events = p.events.filter((ev) => ev.request_id !== id);
        });
      }

      const overlaps = (a, b) => a.some((x) => b.some((y) =>
        x.topic === y.topic && x.partition === y.partition &&
        x.ranges.some(([xf, xl]) => y.ranges.some(([yf, yl]) => xf <= yl && yf <= xl))));

      // Attach a request to every poll whose offsets it carries. The epoch
      // must match too: after a partition is revoked and read again, the new
      // assignment's requests repeat offsets the old assignment's polls hold.
      function linkRequest(r) {
        for (let i = pollOrder.length - 1; i >= 0; i--) {
          const p = polls[pollOrder[i]];
          if (!p || p.assignment_epoch !== r.assignment_epoch || p.requests.includes(r.id) || !overlaps(r.spans, p.spans)) continue;
          p.requests.push(r.id);
          r.polls.push(p.id);
        }
      }

      // A request's events also go on the timeline of every poll it carries.
      // A poll that retries for long keeps its own events and the newest
      // request events only.
      function noteRequestEvent(r, ev) {
        r.events.push(ev);
        r.polls.forEach((id) => {
          const p = polls[id];
          if (!p) return;
          p.events.push(ev);
          if (p.events.length > MAX_POLL_EVENTS) {
            const oldest = p.events.findIndex((e) => e.request_id !== undefined);
            if (oldest >= 0) p.events.splice(oldest, 1);
          }
        });
      }

      function updateModel(ev) {
        switch (ev.type) {
          case "ConsumerStarted":
            polls = {}; pollOrder = []; requests = {}; requestOrder = []; commits = []; partitionHigh = {}; oooCount = 0;
            break;
          case "BatchDispatched": {
            const p = pollFor(ev.batch_id);
            p.ts = ev.ts_ms; p.messages = ev.messages; p.partitions = ev.partitions;
            p.assignment_epoch = ev.assignment_epoch; p.spans = ev.spans || [];
            p.events.push(ev);
            break;
          }
          case "PollCompleted": {
            const p = pollFor(ev.batch_id);
            if (p.ts === null) p.ts = ev.ts_ms;
            p.completed = { accepted: ev.accepted, duration_ms: ev.duration_ms, partitions: ev.partitions, ts: ev.ts_ms };
            p.events.push(ev);
            break;
          }
          case "CommitSubmitted":
            recordCommit(ev);
            break;
          case "BatchFailed": {
            if (!ev.batch_id) break;
            const p = pollFor(ev.batch_id);
            p.failed = { error: ev.error, ts: ev.ts_ms };
            p.events.push(ev);
            break;
          }
          case "RequestSent": {
            const r = requestFor(ev.request_id);
            r.ts = ev.ts_ms; r.worker = ev.worker; r.messages = ev.messages; r.routing_keys = ev.routing_keys;
            r.bytes = ev.bytes; r.replay = ev.replay; r.budgeted = ev.budgeted;
            r.assignment_epoch = ev.assignment_epoch; r.spans = ev.spans || [];
            linkRequest(r);
            noteRequestEvent(r, ev);
            break;
          }
          case "RequestResolved": {
            const r = requestFor(ev.request_id);
            r.worker = r.worker || ev.worker;
            r.resolved = { accepted: ev.accepted, unfinished: ev.unfinished, ts: ev.ts_ms };
            noteRequestEvent(r, ev);
            break;
          }
          case "RequestFailed": {
            const r = requestFor(ev.request_id);
            r.worker = r.worker || ev.worker;
            r.failed = { cause: ev.cause, messages: ev.messages, ts: ev.ts_ms };
            noteRequestEvent(r, ev);
            break;
          }
        }
      }

      function recordCommit(ev) {
        const rows = ev.offsets.map((p) => {
          const key = p.topic + "/" + p.partition;
          const prev = partitionHigh[key];
          const outOfOrder = prev !== undefined && p.offset < prev;
          if (prev === undefined || p.offset > prev) partitionHigh[key] = p.offset;
          return { topic: p.topic, partition: p.partition, offset: p.offset, prev, outOfOrder };
        });
        const bad = rows.some((r) => r.outOfOrder);
        if (bad) oooCount++;
        commits.unshift({ ts: ev.ts_ms, rows, outOfOrder: bad });
        while (commits.length > MAX_COMMITS) commits.pop();
      }

      const pollRequests = (p) => p.requests.map((id) => requests[id]).filter(Boolean);
      const isPartial = (r) => r.resolved && r.resolved.unfinished > 0;

      function pollStatus(p) {
        if (p.completed) return "completed";
        if (p.failed) return "failed";
        const rs = pollRequests(p);
        if (rs.some((r) => r.failed)) return "retrying";
        if (rs.some(isPartial)) return "partial";
        return "inflight";
      }

      // A poll whose requests failed, came back partial, or were replays: the
      // interesting ones, even if they ultimately completed cleanly.
      function hadIssue(p) {
        return p.failed || pollRequests(p).some((r) => r.failed || r.replay || isPartial(r));
      }

      // ---------- live feed rendering ----------
      function describe(ev) {
        switch (ev.type) {
          case "ConsumerStarted": return ["accent", `▶ consumer started — group <code>${ev.group_id}</code>, ${ev.workers.length} worker(s)`];
          case "BatchDispatched": return ["", `poll <code>${shortId(ev.batch_id)}</code> dispatched — ${ev.messages} msgs, ${ev.partitions.length} partition(s)`];
          case "RequestSent": {
            const tags = [];
            if (ev.replay) tags.push("replay");
            if (ev.budgeted) tags.push("budget");
            return ["accent", `→ request <code>#${ev.request_id}</code> to ${shortWorker(ev.worker)} — ${ev.messages} msgs, ${ev.routing_keys} key(s), ${fmtBytes(ev.bytes)}${tags.length ? `<span class="tag"> · ${tags.join(" · ")}</span>` : ""}`];
          }
          case "RequestResolved": return ev.unfinished
            ? ["warn", `◐ request <code>#${ev.request_id}</code> partial — ${shortWorker(ev.worker)} acked ${ev.accepted}, ${ev.unfinished} unfinished`]
            : ["", `✓ request <code>#${ev.request_id}</code> acked — ${shortWorker(ev.worker)}, ${ev.accepted} msgs`];
          case "RequestFailed": return [ev.cause === "busy" ? "warn" : "error", `✗ request <code>#${ev.request_id}</code> failed [${ev.cause}] — ${shortWorker(ev.worker)}, ${ev.messages} msgs requeued`];
          case "PollCompleted": return ["good", `✓ completed poll <code>${shortId(ev.batch_id)}</code> — ${ev.accepted} msgs in ${ev.duration_ms}ms (not a Kafka commit)`];
          case "CommitSubmitted": return ["", `async commit submitted — next-to-read offsets ${ev.offsets.map((p) => `${escapeXml(p.topic)}/${p.partition}@${p.offset}`).join(" ")} (not broker-confirmed)`];
          case "BatchFailed": return ["error", `✗ poll failed — ${ev.error}`];
          case "CommitFailed": return ["error", `✗ commit failed — ${ev.error}`];
          case "WorkerStateChanged": return [ev.to === "unhealthy" ? "error" : ev.to === "healthy" ? "good" : "warn", `worker ${shortWorker(ev.worker)} ${ev.from} → ${ev.to}`];
          case "WorkerMembership": return ["accent", `worker ${shortWorker(ev.worker)} ${ev.action}`];
          default: return ["", ev.type];
        }
      }

      function appendFeed(ev) {
        const [cls, html] = describe(ev);
        const feed = $("feed");
        if (feedEmpty) { feed.innerHTML = ""; feedEmpty = false; }
        const row = document.createElement("div");
        row.className = "ev " + cls;
        row.innerHTML = `<span class="t">${fmtTime(ev.ts_ms)}</span><span class="body">${html}</span>`;
        feed.insertBefore(row, feed.firstChild);
        while (feed.childElementCount > MAX_FEED) feed.removeChild(feed.lastChild);
      }

      function addEvent(ev) {
        if (seen.has(ev.seq)) return;
        seen.add(ev.seq);
        if (seen.size > 50000) seen.clear();
        updateModel(ev);
        if (!$("pause").checked) appendFeed(ev);
        scheduleRender();
      }

      // ---------- polls + commits rendering (throttled) ----------
      function scheduleRender() {
        if (renderPending) return;
        renderPending = true;
        setTimeout(() => {
          renderPending = false;
          $("badge-polls").textContent = pollOrder.length ? `(${pollOrder.length})` : "";
          $("badge-commits").textContent = commits.length ? (oooCount ? `(${commits.length} · ⚠${oooCount})` : `(${commits.length})`) : "";
          // While the list is paused, freeze it so a poll doesn't scroll away
          // before you can click it, but keep an open detail view live.
          if (activeTab === "polls" && (!$("polls-pause").checked || selectedPoll)) renderPolls();
          if (activeTab === "commits") renderCommits();
        }, 250);
      }

      // A poll's requests, grouped by the worker that received them.
      function workerAgg(p) {
        const m = {};
        const touch = (w) => (m[w] = m[w] || { worker: w, requests: 0, messages: 0, acked: 0, unfinished: 0, failed: 0, replays: 0, pending: 0 });
        pollRequests(p).forEach((r) => {
          const x = touch(r.worker || "?");
          x.requests += 1; x.messages += r.messages;
          if (r.replay) x.replays += 1;
          if (r.resolved) { x.acked += r.resolved.accepted; x.unfinished += r.resolved.unfinished; }
          else if (r.failed) x.failed += 1;
          else x.pending += 1;
        });
        return Object.values(m);
      }

      function renderPolls() {
        const view = $("polls-view");
        if (selectedPoll && polls[selectedPoll]) { renderPollDetail(polls[selectedPoll]); return; }
        selectedPoll = null;
        openDetailId = null;
        const issuesOnly = $("polls-issues").checked;
        let ids = pollOrder.slice().reverse();
        if (issuesOnly) ids = ids.filter((id) => hadIssue(polls[id]));
        if (!ids.length) {
          view.innerHTML = `<div class="empty">${issuesOnly ? "No polls with failed, partial or replayed requests yet…" : "Waiting for polls…"}</div>`;
          return;
        }
        const rows = ids.map((id) => {
          const p = polls[id];
          const st = pollStatus(p);
          const rs = pollRequests(p);
          const workers = new Set(rs.map((r) => r.worker)).size;
          const replays = rs.filter((r) => r.replay).length;
          const partial = rs.filter(isPartial).length;
          const failed = rs.filter((r) => r.failed).length;
          const badges = [`<span class="badge-min">${rs.length}req · ${workers}w</span>`];
          if (replays) badges.push(`<span class="badge-min warn">↻${replays}</span>`);
          if (partial) badges.push(`<span class="badge-min warn">◐${partial}</span>`);
          if (failed) badges.push(`<span class="badge-min error">✗${failed}</span>`);
          return `<div class="brow" data-id="${id}">
            <span class="id">${shortId(id)}</span>
            <span><span class="pill ${st}">${st === "inflight" ? "in-flight" : st}</span></span>
            <span class="badges">${badges.join("")}</span>
            <span class="tm">${p.messages} msg · ${p.ts ? fmtTime(p.ts) : ""}</span>
          </div>`;
        }).join("");
        setHTML(view, `<div class="blist">${rows}</div>`, "prepend");
        view.querySelectorAll(".brow").forEach((el) => el.addEventListener("click", () => { selectedPoll = el.dataset.id; renderPollDetail(polls[selectedPoll]); }));
      }

      function requestOutcome(r) {
        if (r.resolved) {
          return r.resolved.unfinished
            ? `<span class="pill partial">partial · ${r.resolved.unfinished} unfinished</span>`
            : `<span class="pill committed">acked ${r.resolved.accepted}</span>`;
        }
        if (r.failed) return `<span class="pill ${r.failed.cause === "busy" ? "retrying" : "failed"}">${r.failed.cause}</span>`;
        return `<span class="pill inflight">in-flight</span>`;
      }

      function renderPollDetail(p) {
        const view = $("polls-view");
        const st = pollStatus(p);
        const rs = pollRequests(p);
        const workers = workerAgg(p);

        const sum = [
          ["status", `<span class="pill ${st}">${st === "inflight" ? "in-flight" : st}</span>`],
          ["messages", p.messages],
          ["requests", rs.length],
          ["workers", workers.length],
          ["replays", rs.filter((r) => r.replay).length],
          ["partial", rs.filter(isPartial).length],
          ["failed", rs.filter((r) => r.failed).length],
        ].map(([l, n]) => `<div class="stat"><div class="n">${n}</div><div class="l">${l}</div></div>`).join("");

        const flow = buildFlow(p, workers);

        const requestRows = rs.map((r) => {
          const settled = r.resolved ? r.resolved.ts : r.failed ? r.failed.ts : null;
          const tags = [r.replay ? "replay" : "", r.budgeted ? "budget" : "", r.polls.length > 1 ? `${r.polls.length} polls` : ""].filter(Boolean).join(" · ");
          return `<tr>
            <td class="mono">#${r.id}</td>
            <td class="mono">${shortWorker(r.worker)}</td>
            <td class="mono">${r.messages}</td>
            <td class="mono">${r.routing_keys}</td>
            <td class="mono">${fmtBytes(r.bytes)}</td>
            <td>${requestOutcome(r)}</td>
            <td class="mono">${settled !== null ? fmtDur(settled - r.ts) : "—"}</td>
            <td class="mono">${tags}</td>
          </tr>`;
        }).join("") || `<tr><td class="empty" colspan="8">No request carries this poll's offsets yet</td></tr>`;

        // A request's events sit on every poll it carries, so order the
        // timeline by sequence to interleave them with the poll's own.
        const timeline = p.events.slice().sort((a, b) => a.seq - b.seq).map((ev) => {
          const [cls, html] = describe(ev);
          return `<div class="item ${cls}"><span class="t">${fmtTime(ev.ts_ms)}</span>${html}</div>`;
        }).join("");

        const fresh = openDetailId !== p.id;
        openDetailId = p.id;
        const detailHtml = `<div class="detail">
          <span class="back" id="back">← all polls</span>
          <div class="dsum">${sum}</div>
          <h2 style="border:none;padding:0 0 8px;">Flow · poll ${shortId(p.id)}</h2>
          <div class="flow-legend">
            <span><i style="background:#4f7cff"></i>sent</span>
            <span><i style="background:#d29922"></i>replay / partial</span>
            <span><i style="background:#f85149"></i>failed</span>
            <span><i style="background:#3fb950"></i>acked / completed</span>
          </div>
          ${flow.html}
          <h2 style="border:none;padding:0 0 8px;">Requests</h2>
          <table><thead><tr><th>Request</th><th>Worker</th><th>Msgs</th><th>Keys</th><th>Bytes</th><th>Outcome</th><th>Took</th><th></th></tr></thead><tbody>${requestRows}</tbody></table>
          <h2 style="border:none;padding:12px 0 8px;">Lifecycle</h2>
          <div class="tl">${timeline}</div>
        </div>`;
        setHTML(view, detailHtml, fresh ? "none" : "append");
        $("back").addEventListener("click", () => { selectedPoll = null; renderPolls(); });
        requestAnimationFrame(() => drawEdges(document.getElementById("flow"), flow.edges));
      }

      // Build the node/edge model for a poll's flow graph. Nodes sit in four
      // lanes (Kafka → batcher → workers → completion).
      function buildFlow(p, workers) {
        const partsShort = p.partitions.map((pp) => `${pp.partition}@${pp.offset}`).join(" ") || "—";
        const sourceNode = `<div class="node src" id="n-source"><div class="title">Kafka poll</div><div class="big">${p.messages}</div><div class="sub">msgs · p ${partsShort}</div></div>`;
        const rs = pollRequests(p);
        const keyRuns = rs.reduce((a, r) => a + r.routing_keys, 0);
        const batcherNode = `<div class="node" id="n-batcher"><div class="title">Batcher</div><div class="big">${rs.length}</div><div class="sub">request(s) · ${keyRuns} key run(s)</div></div>`;

        const workerNodes = workers.map((w, i) => {
          const settled = w.pending === 0 && w.requests > 0;
          const cls = w.failed ? "stalled" : settled ? "acked" : "";
          const badges = [];
          if (w.replays) badges.push(`<span class="badge-min warn">↻${w.replays}</span>`);
          if (w.unfinished) badges.push(`<span class="badge-min warn">◐${w.unfinished}</span>`);
          if (w.failed) badges.push(`<span class="badge-min error">✗${w.failed}</span>`);
          if (settled && !w.failed) badges.push(`<span class="badge-min">✓ acked ${w.acked}</span>`);
          return `<div class="node worker ${cls}" id="n-w-${i}"><div class="title">${shortWorker(w.worker)}</div><div class="big">${w.messages}</div><div class="sub">${w.requests} req · ${badges.join(" ") || "in flight"}</div></div>`;
        }).join("") || '<div class="node" id="n-w-none"><div class="sub">no request yet</div></div>';

        const sinkNode = p.completed
          ? `<div class="node sink-ok" id="n-sink"><div class="title">Poll completed</div><div class="big">${p.completed.accepted} msgs</div><div class="sub">${p.completed.duration_ms}ms · poll max offsets ${p.completed.partitions.map((pp) => `${pp.partition}@${pp.offset}`).join(" ")}</div><div class="sub">Not a Kafka commit</div></div>`
          : p.failed
            ? `<div class="node worker stalled" id="n-sink"><div class="title">Failed</div><div class="sub">${escapeXml(p.failed.error)}</div></div>`
            : `<div class="node sink-pending" id="n-sink"><div class="title">In-flight</div><div class="sub">not completed</div></div>`;

        const html = `<div class="flow" id="flow">
          <div class="flow-main">
            <div class="flow-col">${sourceNode}</div>
            <div class="flow-col">${batcherNode}</div>
            <div class="flow-col">${workerNodes}</div>
            <div class="flow-col">${sinkNode}</div>
          </div>
          <svg></svg>
        </div>`;

        const C = { muted: "#8b93a1", warn: "#d29922", good: "#3fb950", error: "#f85149", accent: "#4f7cff" };
        const edges = [];
        edges.push({ from: "n-source", to: "n-batcher", label: `${p.messages} msgs`, color: C.accent });
        workers.forEach((w, i) => {
          const color = w.failed ? C.error : (w.replays || w.unfinished) ? C.warn : C.muted;
          edges.push({ from: "n-batcher", to: `n-w-${i}`, label: `${w.requests} req · ${w.messages} msgs`, color });
        });

        if (p.completed) {
          if (workers.length) workers.forEach((w, i) => { if (!w.failed) edges.push({ from: `n-w-${i}`, to: "n-sink", label: "acked", color: C.good }); });
          else edges.push({ from: "n-batcher", to: "n-sink", label: "", color: C.good });
        }

        return { html, edges };
      }

      const cid = (c) => c.replace("#", "");
      const escapeXml = (s) => String(s).replace(/[<>&]/g, (m) => ({ "<": "&lt;", ">": "&gt;", "&": "&amp;" }[m]));

      // Draw SVG edges between the flow nodes, anchored on the box borders with a
      // smooth orthogonal-ish bezier and a coloured arrowhead + label.
      function drawEdges(container, edges) {
        if (!container) return;
        const svg = container.querySelector("svg");
        if (!svg) return;
        const crect = container.getBoundingClientRect();
        const rectOf = (id) => {
          const el = document.getElementById(id);
          if (!el) return null;
          const r = el.getBoundingClientRect();
          return { cx: r.left + r.width / 2 - crect.left, cy: r.top + r.height / 2 - crect.top, hw: r.width / 2, hh: r.height / 2 };
        };
        const border = (r, ox, oy) => {
          const dx = ox - r.cx, dy = oy - r.cy;
          if (!dx && !dy) return { x: r.cx, y: r.cy };
          const s = 1 / Math.max(Math.abs(dx) / r.hw, Math.abs(dy) / r.hh);
          return { x: r.cx + dx * s, y: r.cy + dy * s };
        };

        const colorSet = {};
        edges.forEach((e) => (colorSet[e.color] = true));
        let defs = "<defs>";
        Object.keys(colorSet).forEach((c) => {
          defs += `<marker id="arw-${cid(c)}" viewBox="0 0 10 10" refX="8.5" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0 0 L10 5 L0 10 z" fill="${c}"/></marker>`;
        });
        defs += "</defs>";

        let body = "";
        edges.forEach((e) => {
          const rs = rectOf(e.from), rt = rectOf(e.to);
          if (!rs || !rt) return;
          const sa = border(rs, rt.cx, rt.cy), ta = border(rt, rs.cx, rs.cy);
          const horiz = Math.abs(rt.cx - rs.cx) >= Math.abs(rt.cy - rs.cy);
          const mx = (sa.x + ta.x) / 2, my = (sa.y + ta.y) / 2;
          const c1 = horiz ? { x: mx, y: sa.y } : { x: sa.x, y: my };
          const c2 = horiz ? { x: mx, y: ta.y } : { x: ta.x, y: my };
          body += `<path d="M ${sa.x} ${sa.y} C ${c1.x} ${c1.y}, ${c2.x} ${c2.y}, ${ta.x} ${ta.y}" fill="none" stroke="${e.color}" stroke-width="1.6" opacity="0.9" marker-end="url(#arw-${cid(e.color)})"/>`;
          if (e.label) {
            const lw = e.label.length * 5.7 + 10;
            body += `<rect x="${mx - lw / 2}" y="${my - 8}" width="${lw}" height="15" rx="3" fill="#1c1f26" stroke="${e.color}" stroke-opacity="0.35"/>`;
            body += `<text x="${mx}" y="${my + 2.5}" text-anchor="middle" class="edge-label" fill="${e.color}">${escapeXml(e.label)}</text>`;
          }
        });

        svg.setAttribute("width", container.scrollWidth);
        svg.setAttribute("height", container.scrollHeight);
        svg.innerHTML = defs + body;
      }

      function renderCommits() {
        const banner = $("commit-banner");
        banner.innerHTML = oooCount
          ? `<div class="banner bad">⚠ ${oooCount} out-of-order submission(s) detected — submitted offsets went backwards for a partition</div>`
          : commits.length ? `<div class="banner ok">✓ all ${commits.length} retained submissions are in order (submitted offsets nondecreasing per partition)</div>` : "";

        const el = $("commits");
        if (!commits.length) { el.innerHTML = '<div class="empty">Waiting for commit submissions…</div>'; return; }
        const rows = commits.map((c) => {
          const chips = c.rows
            .slice()
            .sort((a, b) => a.partition - b.partition)
            .map((r) =>
              `<span class="offchip ${r.outOfOrder ? "bad" : ""}">
                 <span class="pl">${escapeXml(r.topic)}/P${r.partition}</span>
                 <span class="ov">${r.offset}</span>
                 ${r.outOfOrder ? `<span class="was">↓ was ${r.prev}</span>` : ""}
               </span>`
            ).join("");
          return `<tr class="${c.outOfOrder ? "ooo" : ""}">
            <td class="mono">${fmtTime(c.ts)}</td>
            <td><div class="offchips">${chips}</div></td>
            <td>${c.outOfOrder ? '<span class="pill failed">out of order</span>' : ""}</td>
          </tr>`;
        }).join("");
        setHTML(el, `<table><thead><tr><th>Submitted at</th><th>Submitted next-to-read offset per partition</th><th></th></tr></thead><tbody>${rows}</tbody></table>`, "prepend");
      }

      // ---------- tabs ----------
      document.querySelectorAll(".tab").forEach((t) =>
        t.addEventListener("click", () => {
          activeTab = t.dataset.tab;
          document.querySelectorAll(".tab").forEach((x) => x.classList.toggle("active", x === t));
          $("tab-feed").style.display = activeTab === "feed" ? "" : "none";
          $("tab-polls").style.display = activeTab === "polls" ? "" : "none";
          $("tab-commits").style.display = activeTab === "commits" ? "" : "none";
          if (activeTab === "polls") renderPolls();
          if (activeTab === "commits") renderCommits();
        })
      );

      // Re-render immediately when the poll filters change (and on un-pause).
      $("polls-issues").addEventListener("change", renderPolls);
      $("polls-pause").addEventListener("change", () => { if (!$("polls-pause").checked) renderPolls(); });
      // Redraw the flow graph's edges when the layout width changes.
      window.addEventListener("resize", () => {
        if (activeTab === "polls" && selectedPoll && polls[selectedPoll]) renderPollDetail(polls[selectedPoll]);
      });

      // ---------- live SSE ----------
      function connect() {
        const es = new EventSource("debug/events");
        es.onopen = () => {
          $("conn").textContent = "live"; $("conn-dot").className = "dot live";
          resetModels(); // reconnect/restart replays the backlog; rebuild cleanly
        };
        es.onmessage = (m) => { try { addEvent(JSON.parse(m.data)); } catch (e) {} };
        es.onerror = () => { $("conn").textContent = "reconnecting…"; $("conn-dot").className = "dot down"; };
      }

      refreshState();
      setInterval(refreshState, 600);
      connect();
    </script>
  </body>
</html>