Skip to content

Xbox Cloud Gaming app for PS5 Homebrew

C
38.8 KB
Raw Blame
// SPDX-License-Identifier: GPL-3.0-or-later
// Copyright (C) 2026 RafaelNGP
#include "stream/stream_session.h"

#include "stream/h264_receiver.h"
#include "stream/rtcp_reporter.h"

#include "platform/platform.h"
#include "util/json.h"
#include "util/log.h"

#include <rtc/rtc.hpp>

#include <arpa/inet.h>
#include <netinet/in.h>
#include <sys/socket.h>

#include <atomic>
#include <chrono>
#include <condition_variable>
#include <cstdio>
#include <mutex>
#include <sstream>
#include <thread>
#include <vector>

namespace xc::stream {

namespace {

// Fixed values the xbox.com web client sends.
constexpr const char* kControlAccessKey = "4BDB3609-C1F1-4195-9B37-FEFF45DA8B8E";
constexpr const char* kHandshakeId = "be0bfc6d-1e83-4c8a-90ed-fa8601c5a179";
constexpr const char* kClientAppInstallId = "c97d7ee0-73b2-4239-bf1d-9d805a338429";
// SystemUiType of the system UIs this client draws itself.
constexpr int kShowVirtualKeyboard = 10;

// libdatachannel's name for one of its states ("checking", "srflx"...), for the log.
template <class T>
std::string named(const T& value) {
    std::ostringstream out;
    out << value;
    return out.str();
}

std::string uuid4() {
    uint8_t b[16];
    platform::randomBytes(b, sizeof b);
    b[6] = static_cast<uint8_t>((b[6] & 0x0f) | 0x40);
    b[8] = static_cast<uint8_t>((b[8] & 0x3f) | 0x80);
    char s[37];
    std::snprintf(s, sizeof s, "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x", b[0], b[1],
                  b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9], b[10], b[11], b[12], b[13], b[14], b[15]);
    return s;
}

std::vector<std::string> midsInOrder(const std::string& sdp) {
    std::vector<std::string> mids;
    size_t pos = 0;
    while ((pos = sdp.find("a=mid:", pos)) != std::string::npos) {
        pos += 6;
        size_t end = sdp.find_first_of("\r\n", pos);
        mids.push_back(sdp.substr(pos, end - pos));
    }
    return mids;
}

void initRtcLogging() {
    static std::once_flag once;
    std::call_once(once, [] {
        rtc::InitLogger(rtc::LogLevel::Warning, [](rtc::LogLevel level, std::string message) {
            if (level <= rtc::LogLevel::Error)
                XC_LOGE("rtc: %s", message.c_str());
            else
                XC_LOGW("rtc: %s", message.c_str());
        });
    });
}

}  // namespace

bool decodeTeredo(const std::string& ipv6, std::string& ipv4, int& port) {
    in6_addr a{};
    if (::inet_pton(AF_INET6, ipv6.c_str(), &a) != 1) return false;
    const uint8_t* b = a.s6_addr;
    if (!(b[0] == 0x20 && b[1] == 0x01 && b[2] == 0x00 && b[3] == 0x00)) return false;
    // Bytes 10-11: obfuscated (inverted) port; 12-15: inverted client IPv4.
    port = static_cast<uint16_t>(~((b[10] << 8) | b[11]));
    char s[16];
    std::snprintf(s, sizeof s, "%u.%u.%u.%u", b[12] ^ 0xff, b[13] ^ 0xff, b[14] ^ 0xff, b[15] ^ 0xff);
    ipv4 = s;
    return true;
}

struct StreamSession::Impl {
    xcloud::GssvClient& gssv;
    StreamCallbacks cb;
    StreamOptions opt;

    std::shared_ptr<rtc::PeerConnection> pc;
    std::shared_ptr<rtc::DataChannel> chat, control, input, message;
    std::shared_ptr<rtc::Track> audio, video;

    std::mutex mutex;
    std::condition_variable cv;
    std::vector<rtc::Candidate> localCandidates;
    bool gatheringDone = false;
    bool handshakeDone = false;
    bool failed = false;
    std::string failReason;

    std::atomic<bool> open{false};
    std::atomic<bool> resolutionPending{false};  // 1440: asked at the first video frame
    std::atomic<bool> closedNotified{false};
    std::atomic<uint32_t> inputSequence{1};
    uint64_t lastKeepaliveMs = 0;
    std::atomic<bool> keepaliveInFlight{false};
    platform::Thread keepaliveThread;
    uint64_t lastKeyframeMs = 0;
    VideoReceiveStats videoStats;
    std::shared_ptr<RtcpReporter> videoReporter;
    H264Receiver* videoReceiverRaw = nullptr;  // owned by the video track
    // The video receiver (a media handler, which libdatachannel may still be
    // running after the session is gone) reaches the session through this.
    struct Guard {
        std::mutex mutex;
        Impl* self = nullptr;
    };
    std::shared_ptr<Guard> guard = std::make_shared<Guard>();
    std::atomic<int> otherInputReports{0};
    // Controllers: which slots are attached; what each sent last and when
    // (sendGamepad's thread). A pad that didn't change is sent again only
    // every kPadRepeatMs, not at the input loop's rate.
    static constexpr uint64_t kPadRepeatMs = 50;
    std::atomic<uint32_t> padsAttached{0};
    GamepadFrame lastPad[4];
    uint64_t lastPadMs[4] = {};
    // After the picture stopped for a while (a lag spike), the server may
    // keep receiving the pad and ignore it: the pads are announced again.
    std::atomic<uint64_t> lastVideoMs{0};
    std::atomic<bool> padsStale{false};
    bool loggedFirstView = false;
    bool loggedFirstMenu = false;

    Impl(xcloud::GssvClient& g, StreamCallbacks c, StreamOptions o) : gssv(g), cb(std::move(c)), opt(o) {}

    double nowMs() const { return stream::clockMs(); }

    void fail(const std::string& reason, bool recoverable = true) {
        {
            std::lock_guard<std::mutex> lock(mutex);
            if (!failed) {
                failed = true;
                failReason = reason;
            }
        }
        cv.notify_all();
        open = false;
        if (!closedNotified.exchange(true)) {
            XC_LOGW("stream closed: %s", reason.c_str());
            if (cb.closed) cb.closed(reason, recoverable);
        }
    }

    static void sendText(const std::shared_ptr<rtc::DataChannel>& ch, const std::string& text) {
        // The web client sends its JSON as binary (TextEncoder) messages.
        if (!ch || !ch->isOpen()) return;
        try {
            ch->send(reinterpret_cast<const rtc::byte*>(text.data()), text.size());
        } catch (const std::exception& e) {
            XC_LOGW("send on %s: %s", ch->label().c_str(), e.what());
        }
    }

    static void sendBinary(const std::shared_ptr<rtc::DataChannel>& ch, const uint8_t* data, size_t size) {
        if (!ch || !ch->isOpen() || !data || size == 0) return;
        try {
            ch->send(reinterpret_cast<const rtc::byte*>(data), size);
        } catch (const std::exception& e) {
            XC_LOGW("send on %s: %s", ch->label().c_str(), e.what());
        }
    }

    static void sendBinary(const std::shared_ptr<rtc::DataChannel>& ch, const std::vector<uint8_t>& data) {
        sendBinary(ch, data.data(), data.size());
    }

    static std::string asText(const rtc::message_variant& m) {
        if (auto s = std::get_if<std::string>(&m)) return *s;
        const auto& b = std::get<rtc::binary>(m);
        return std::string(reinterpret_cast<const char*>(b.data()), b.size());
    }

    // --- message channel -----------------------------------------------------

    std::string messageEnvelope(const std::string& target, const json::Value& content) {
        json::Value v = json::Value::object();
        v.set("type", "Message");
        v.set("content", content.dump());
        v.set("id", uuid4());
        v.set("target", target);
        v.set("cv", "");
        return v.dump();
    }

    void completeTransaction(const std::string& id, const json::Value& content) {
        json::Value v = json::Value::object();
        v.set("type", "TransactionComplete");
        v.set("content", content.dump());
        v.set("id", id);
        v.set("cv", "");
        sendText(message, v.dump());
    }

    void cancelTransaction(const std::string& id) {
        json::Value v = json::Value::object();
        v.set("type", "ReceiverCancel");
        v.set("id", id);
        v.set("cv", "");
        sendText(message, v.dump());
    }

    void sendResolution(const std::string& alias) {
        // The HQ tiers and 1440 need Game Pass Ultimate in a market where
        // they're enabled; elsewhere the service ignores them and keeps what
        // it had (1080 at the start). The plain tier first, then: measured,
        // "720" switches in a few seconds, "720HQ" is ignored in Brazil.
        // 1440 alone, as the xbox.com client sends it (never 1080 before it).
        std::string base = alias;
        if (base.size() > 2 && base.compare(base.size() - 2, 2, "HQ") == 0) base.resize(base.size() - 2);
        // The user's own Xbox offers every tier on any plan: just the one asked for.
        if (gssv.isHome()) base = alias;
        auto send = [this](const std::string& a) {
            json::Value res = json::Value::object();
            res.set("message", "userRequestedResolutionUpdate");
            res.set("resolutionAlias", a);
            sendText(control, res.dump());
            XC_LOGI("requested stream resolution %s", a.c_str());
        };
        send(base);
        if (alias != base) send(alias);
    }

    void sendClientConfig() {
        json::Value ver = json::Value::array();
        ver.push(0);
        ver.push(2);
        ver.push(0);
        json::Value ui = json::Value::object();
        ui.set("version", ver);
        json::Value systemUis = json::Value::array();
        if (cb.textInput) systemUis.push(kShowVirtualKeyboard);
        ui.set("systemUis", systemUis);
        sendText(message, messageEnvelope("/streaming/systemUi/configuration", ui));

        json::Value install = json::Value::object();
        install.set("clientAppInstallId", kClientAppInstallId);
        sendText(message, messageEnvelope("/streaming/properties/clientappinstallidchanged", install));

        json::Value orientation = json::Value::object();
        orientation.set("orientation", 0);
        sendText(message, messageEnvelope("/streaming/characteristics/orientationchanged", orientation));

        json::Value touch = json::Value::object();
        touch.set("touchInputEnabled", false);
        sendText(message, messageEnvelope("/streaming/characteristics/touchinputenabledchanged", touch));

        sendText(message, messageEnvelope("/streaming/characteristics/clientdevicecapabilities", json::Value::object()));

        // The picture's size, as the web client sends its video element's:
        // a 1440p tier needs a 1440p one (the display itself is 4K).
        int w = opt.resolutionAlias == "1440" ? 2560 : 1920, h = opt.resolutionAlias == "1440" ? 1440 : 1080;
        json::Value dims = json::Value::object();
        // horizontal / vertical: the screen's physical size in millimetres,
        // as the web client derives it (a 19" 16:9 panel's).
        dims.set("horizontal", 423);
        dims.set("vertical", 238);
        dims.set("preferredWidth", w);
        dims.set("preferredHeight", h);
        dims.set("safeAreaLeft", 0);
        dims.set("safeAreaTop", 0);
        dims.set("safeAreaRight", w);
        dims.set("safeAreaBottom", h);
        dims.set("supportsCustomResolution", true);
        sendText(message, messageEnvelope("/streaming/characteristics/dimensionschanged", dims));

        // Last of the web client's handshake: the title isn't constrained.
        json::Value constrain = json::Value::object();
        constrain.set("constrain", false);
        json::Value tx = json::Value::object();
        tx.set("type", "TransactionStart");
        tx.set("content", constrain.dump());
        tx.set("id", uuid4());
        tx.set("target", "/streaming/title/constrain");
        tx.set("cv", "");
        sendText(message, tx.dump());
    }

    void sendGamepadChanged(int index, bool added) {
        if (index >= 0 && index < 32) {
            if (added) padsAttached |= 1u << index;
            else padsAttached &= ~(1u << index);
        }
        json::Value v = json::Value::object();
        v.set("message", "gamepadChanged");
        v.set("gamepadIndex", index);
        v.set("wasAdded", added);
        sendText(control, v.dump());
    }

    void onHandshakeAck() {
        XC_LOGI("message channel handshake complete");
        json::Value auth = json::Value::object();
        auth.set("message", "authorizationRequest");
        auth.set("accessKey", kControlAccessKey);
        sendText(control, auth.dump());
        // Same sequence as the web client: reset slot 0, then attach the pad.
        sendGamepadChanged(0, true);
        sendGamepadChanged(0, false);
        sendGamepadChanged(0, true);
        // 1440 once the picture is running (the first video frame), as the
        // xbox.com client sends it; the other tiers right away.
        if (opt.resolutionAlias == "1440" && !gssv.isHome())
            resolutionPending = true;
        else if (!opt.resolutionAlias.empty())
            sendResolution(opt.resolutionAlias);


        // One touch point, as the web client announces (touch is then turned
        // on only while a touch screen is up: setTouchEnabled()).
        sendBinary(input, clientMetadataReport(0, nowMs(), static_cast<uint8_t>(std::getenv("XC_TOUCH_POINTS") ? std::atoi(std::getenv("XC_TOUCH_POINTS")) : 1)));
        sendClientConfig();
        {
            std::lock_guard<std::mutex> lock(mutex);
            handshakeDone = true;
        }
        open = true;
        cv.notify_all();
    }

    void onMessageChannel(const rtc::message_variant& m) {
        std::string text = asText(m);
        auto j = json::parse(text);
        if (!j) {
            XC_LOGW("message channel: unparsable %s", text.substr(0, 200).c_str());
            return;
        }
        std::string type = (*j)["type"].str();
        if (type == "HandshakeAck") {
            onHandshakeAck();
            return;
        }
        if (type == "SenderCancel") {
            XC_LOGI("message channel: the server withdrew %s", (*j)["id"].str().c_str());
            if (cb.textInputCancelled) cb.textInputCancelled((*j)["id"].str());
            return;
        }
        if (type != "Message" && type != "TransactionStart") {
            XC_LOGD("message channel: %s", text.substr(0, 300).c_str());
            return;
        }
        std::string target = (*j)["target"].str();
        std::string id = (*j)["id"].str();
        XC_LOGI("message %s %s", type.c_str(), target.c_str());
        if (target == "/streaming/properties/titleinfo") {
            auto info = json::parse((*j)["content"].str());
            if (info && cb.titleFocus) cb.titleFocus((*info)["focused"].asBool());
        }
        if (target == "/streaming/sessionLifetimeManagement/serverInitiatedDisconnect") {
            std::string body = (*j)["content"].str();
            XC_LOGW("server disconnect: %s", body.substr(0, 300).c_str());
            completeTransaction(id, json::Value(""));
            // A warning shares this message with the real kicks: the session
            // goes on (the xbox.com client only shows a notice).
            auto content = json::parse(body);
            std::string reason = content ? (*content)["reason"].str() : std::string();
            if (reason == "WarningForBeingIdle") {
                if (cb.idleWarning) cb.idleWarning(static_cast<int>((*content)["secondsUntilKick"].asInt(120)));
            } else if (reason == "KickForClosedGame") {
                // The game itself quit (or failed to start) on the server.
                char hr[16];
                std::snprintf(hr, sizeof hr, "0x%08X", static_cast<unsigned>((*content)["hr"].asInt()));
                fail(std::string("the game closed on the server (") + hr + ")", false);
            } else {
                fail(reason.empty() ? "the server ended the session" : "the server ended the session (" + reason + ")",
                     false);
            }
        } else if (target == "/streaming/systemUi/messages/ShowVirtualKeyboard" && cb.textInput) {
            auto content = json::parse((*j)["content"].str());
            TextInputRequest req;
            req.id = id;
            if (content) {
                req.title = (*content)["TitleText"].str();
                req.description = (*content)["DescriptionText"].str();
                req.defaultText = (*content)["DefaultText"].str();
                req.inputScope = static_cast<int>((*content)["InputScope"].asInt());
                req.maxLength = static_cast<int>((*content)["MaxLength"].asInt());
            }
            XC_LOGI("the game asks for text (scope %d, up to %d characters)", req.inputScope, req.maxLength);
            cb.textInput(req);
        } else if (target == "/streaming/systemUi/messages/ShowMessageDialog") {
            // No dialog UI yet: log it and pick the first (default) button.
            auto content = json::parse((*j)["content"].str());
            if (content)
                XC_LOGW("server dialog: %s / %s", (*content)["TitleText"].str().c_str(),
                        (*content)["ContentText"].str().c_str());
            json::Value result = json::Value::object();
            result.set("Result", 0);
            completeTransaction(id, result);
        }
    }

    // --- set-up ---------------------------------------------------------------

    std::shared_ptr<rtc::DataChannel> makeChannel(const char* label, const char* protocol,
                                                  bool unordered = false,
                                                  std::optional<unsigned int> maxRetransmits = std::nullopt) {
        rtc::DataChannelInit init;
        init.protocol = protocol;
        init.reliability.unordered = unordered;
        init.reliability.maxRetransmits = maxRetransmits;
        auto ch = pc->createDataChannel(label, init);
        ch->onOpen([label] { XC_LOGI("data channel %s open", label); });
        ch->onClosed([this, label] {
            XC_LOGI("data channel %s closed", label);
            if (open) fail(std::string("data channel ") + label + " closed");
        });
        return ch;
    }

    void createPeer() {
        initRtcLogging();
        XC_LOGI("webrtc: createPeer starting");
        if (std::string res = platform::diagnostics(); !res.empty()) XC_LOGI("webrtc: resources before the peer: %s", res.c_str());
        rtc::Configuration config;
        config.disableAutoNegotiation = true;
        try {
            pc = std::make_shared<rtc::PeerConnection>(config);
        } catch (const std::exception& e) {
            XC_LOGE("webrtc: PeerConnection failed to create: %s", e.what());
            if (std::string res = platform::diagnostics(); !res.empty()) XC_LOGE("webrtc: resources at the failure: %s", res.c_str());
            fail(std::string("WebRTC peer creation failed: ") + e.what());
            return;
        }
        XC_LOGI("webrtc: PeerConnection created");
        if (std::string res = platform::diagnostics(); !res.empty()) XC_LOGI("webrtc: resources after the peer: %s", res.c_str());

        pc->onLocalCandidate([this](rtc::Candidate c) {
            // Its type and transport only: the address stays out of the log.
            XC_LOGI("webrtc: local ICE candidate: %s %s", named(c.type()).c_str(),
                    c.transportType() == rtc::Candidate::TransportType::Udp ? "udp" : "tcp");
            std::lock_guard<std::mutex> lock(mutex);
            localCandidates.push_back(std::move(c));
        });
        pc->onGatheringStateChange([this](rtc::PeerConnection::GatheringState s) {
            XC_LOGI("webrtc: ICE gathering %s", named(s).c_str());
            if (s == rtc::PeerConnection::GatheringState::Complete) {
                {
                    std::lock_guard<std::mutex> lock(mutex);
                    gatheringDone = true;
                }
                cv.notify_all();
            }
        });
        pc->onIceStateChange([](rtc::PeerConnection::IceState s) { XC_LOGI("webrtc: ICE state %s", named(s).c_str()); });
        pc->onStateChange([this](rtc::PeerConnection::State s) {
            XC_LOGI("peer connection state: %d (%s)", static_cast<int>(s), named(s).c_str());
            if (s == rtc::PeerConnection::State::Failed) fail("WebRTC connection failed");
            if (s == rtc::PeerConnection::State::Disconnected) fail("WebRTC connection lost");
        });

        // Same channels, in the same order, as the web client.
        // Input channel is configured unordered and unreliable (0 retransmissions)
        // to eliminate Head-of-Line Blocking input latency under network jitter/loss.
        try {
            chat = makeChannel("chat", "chatV1");
            chat->onMessage([](rtc::message_variant m) { XC_LOGI("chat channel: %s", asText(m).c_str()); });
            control = makeChannel("control", "controlV1");
            input = makeChannel("input", "1.0", /*unordered=*/true, /*maxRetransmits=*/0);
            message = makeChannel("message", "messageV1");
        } catch (const std::exception& e) {
            XC_LOGE("webrtc: createDataChannel failed: %s", e.what());
            fail(std::string("WebRTC channel creation failed: ") + e.what());
            return;
        }

        message->onOpen([this] {
            XC_LOGI("data channel message open: sending handshake");
            json::Value hs = json::Value::object();
            hs.set("type", "Handshake");
            hs.set("version", "messageV1");
            hs.set("id", kHandshakeId);
            hs.set("cv", "0");
            sendText(message, hs.dump());
        });
        message->onMessage([this](rtc::message_variant m) { onMessageChannel(m); });
        control->onMessage([](rtc::message_variant m) { XC_LOGD("control: %s", asText(m).substr(0, 300).c_str()); });
        input->onMessage([this](rtc::message_variant m) {
            auto b = std::get_if<rtc::binary>(&m);
            if (!b) return;
            auto* d = reinterpret_cast<const uint8_t*>(b->data());
            Vibration v;
            uint32_t w, h;
            if (parseVibration(d, b->size(), v)) {
                if (cb.vibration) cb.vibration(v);
            } else if (parseServerMetadata(d, b->size(), w, h)) {
                XC_LOGI("server video size %ux%u", w, h);
            } else if (b->size() >= 2 && ++otherInputReports <= 5) {
                XC_LOGI("input channel: report type %u, %zu bytes", d[0], b->size());
            }
        });

        // Audio: sendrecv like the browser (the send side carries chat).
        rtc::Description::Audio a("0", rtc::Description::Direction::SendRecv);
        a.addOpusCodec(111, "minptime=10;useinbandfec=1;stereo=1");
        a.addSSRC(kMicSsrc, "audio");
        audio = pc->addTrack(a);

        auto audioDepacketizer = std::make_shared<rtc::OpusRtpDepacketizer>();
        audioDepacketizer->addToChain(std::make_shared<RtcpReporter>(48000, 0));

        auto rtpConfig = std::make_shared<rtc::RtpPacketizationConfig>(
            kMicSsrc, "audio", 111, rtc::OpusRtpPacketizer::DefaultClockRate);
        auto audioPacketizer = std::make_shared<rtc::OpusRtpPacketizer>(rtpConfig);
        audioDepacketizer->addToChain(audioPacketizer);

        audio->setMediaHandler(audioDepacketizer);
        audio->onFrame([this](rtc::binary data, rtc::FrameInfo info) {
            if (cb.audio) cb.audio(reinterpret_cast<const uint8_t*>(data.data()), data.size(), info.timestamp);
        });

        // Video: H.264 only, High and Main before Constrained Baseline / Baseline.
        rtc::Description::Video v("1", rtc::Description::Direction::RecvOnly);
        // High profile (0x64): enables 8x8 integer transform and improved quantization matrices.
        // Level 5.1 (0x33) allows 1440p60; level 4.2 (0x2a) is xCloud's standard 1080p60 ceiling.
        v.addH264Codec(100, "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640033");
        v.addH264Codec(101, "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64002a");
        // Main profile at level 5.1 (0x33): fallback if server doesn't negotiate High profile.
        v.addH264Codec(102, "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d0033");
        v.addH264Codec(104, "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f");
        v.addH264Codec(106, "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f");
        video = pc->addTrack(v);
        // Our jitter buffer (NACK, whole frames only) in place of
        // H264RtpDepacketizer, which hands over frames with holes in them.
        guard->self = this;
        auto videoReceiver = std::make_shared<H264Receiver>(
            [g = guard] {
                std::lock_guard<std::mutex> lock(g->mutex);
                if (!g->self || !g->self->open) return;
                try {
                    g->self->requestKeyframe();
                } catch (const std::exception& e) {
                    XC_LOGW("keyframe request: %s", e.what());
                }
            },
            videoStats);
        videoReceiver->setSimulatedLoss(opt.simulatedVideoLoss);
        videoReceiverRaw = videoReceiver.get();
        videoReporter = std::make_shared<RtcpReporter>(90000, opt.maxBitrate);
        videoReceiver->addToChain(videoReporter);
        video->setMediaHandler(videoReceiver);
        video->onFrame([this, receiver = videoReceiver.get()](rtc::binary data, rtc::FrameInfo info) {
            static thread_local bool s_netPinned = false;
            if (!s_netPinned) {
                platform::setThreadAffinity(platform::ThreadRole::Network, "xcloud_net");
                s_netPinned = true;
            }
            if (resolutionPending.exchange(false)) sendResolution(opt.resolutionAlias);
            uint64_t now = platform::nowMs();
            uint64_t last = lastVideoMs.exchange(now);
            if (last && now - last >= 1500) {
                XC_LOGI("picture back after %llu ms: announcing the controllers again",
                        static_cast<unsigned long long>(now - last));
                padsStale = true;
            }
            if (cb.video)
                cb.video(reinterpret_cast<const uint8_t*>(data.data()), data.size(), info.timestamp,
                         receiver->firstPacketUs(info.timestamp));
        });
        XC_LOGI("webrtc: createPeer completed");
    }

    bool waitFor(const std::function<bool()>& pred, int timeoutMs) {
        std::unique_lock<std::mutex> lock(mutex);
        return cv.wait_for(lock, std::chrono::milliseconds(timeoutMs), [&] { return failed || pred(); }) && !failed;
    }

    void addRemoteCandidate(const std::string& candidate, const std::string& mid) {
        std::string c = candidate;
        while (!c.empty() && (c.back() == ' ' || c.back() == '\r' || c.back() == '\n')) c.pop_back();
        try {
            pc->addRemoteCandidate(rtc::Candidate(c, mid));
            XC_LOGD("remote candidate added");  // not its address
        } catch (const std::exception& e) {
            XC_LOGW("ignoring a remote candidate: %s", e.what());
        }
    }

    bool start(std::string& err) {
        XC_LOGI("webrtc: start() calling createPeer");
        createPeer();
        if (!pc || failed) {
            std::lock_guard<std::mutex> lock(mutex);
            err = failReason.empty() ? "WebRTC PeerConnection initialization failed" : failReason;
            return false;
        }
        XC_LOGI("webrtc: calling pc->setLocalDescription");
        try {
            pc->setLocalDescription(rtc::Description::Type::Offer);
        } catch (const std::exception& e) {
            err = std::string("setLocalDescription failed: ") + e.what();
            return false;
        }
        XC_LOGI("webrtc: setLocalDescription returned; waiting for ICE gathering");
        if (!waitFor([this] { return gatheringDone; }, 10000))
            XC_LOGW("ICE gathering did not complete; continuing with %zu candidates", localCandidates.size());

        auto local = pc->localDescription();
        if (!local) {
            err = "no local SDP offer";
            return false;
        }
        std::string offer = std::string(*local);
        // No a=imageattr: the xbox.com client adds one only to *limit* a
        // device to 1080p; a 1440p-capable one sends the plain offer.
        XC_LOGD("local offer:\n%s", offer.c_str());
        if (!gssv.sendSdpOffer(offer, err)) return false;

        std::string answer;
        for (int i = 0; i < 60 && answer.empty(); ++i) {
            if (!gssv.pollSdpAnswer(answer, err)) return false;
            if (answer.empty()) platform::sleepMs(500);
        }
        if (answer.empty()) {
            err = "timed out waiting for the SDP answer";
            return false;
        }
        XC_LOGD("remote answer:\n%s", answer.c_str());
        for (size_t p = answer.find("a=rtpmap:"); p != std::string::npos; p = answer.find("a=rtpmap:", p + 1))
            XC_LOGI("answer %s", answer.substr(p, answer.find_first_of("\r\n", p) - p).c_str());
        try {
            pc->setRemoteDescription(rtc::Description(answer, rtc::Description::Type::Answer));
        } catch (const std::exception& e) {
            err = std::string("remote SDP rejected: ") + e.what();
            return false;
        }

        // Local candidates -> gssv, with their m-line index in our offer.
        std::vector<std::string> mids = midsInOrder(offer);
        std::vector<xcloud::IceCandidate> mine;
        {
            std::lock_guard<std::mutex> lock(mutex);
            for (const auto& c : localCandidates) {
                xcloud::IceCandidate ic;
                ic.candidate = std::string(c);  // "candidate:...", as a browser sends
                ic.sdpMid = c.mid();
                for (size_t i = 0; i < mids.size(); ++i)
                    if (mids[i] == ic.sdpMid) ic.sdpMLineIndex = static_cast<int>(i);
                mine.push_back(std::move(ic));
            }
        }
        XC_LOGI("sending %zu local ICE candidates", mine.size());
        if (!gssv.sendIceCandidates(mine, err)) return false;

        std::vector<xcloud::IceCandidate> theirs;
        for (int i = 0; i < 30 && theirs.empty(); ++i) {
            if (!gssv.pollIceCandidates(theirs, err)) return false;
            if (theirs.empty()) platform::sleepMs(1000);
        }
        if (theirs.empty()) {
            err = "the server sent no ICE candidates";
            return false;
        }
        std::vector<std::string> remoteMids = midsInOrder(answer);
        std::string bundleMid = remoteMids.empty() ? "0" : remoteMids.front();
        for (const auto& c : theirs) {
            if (c.candidate.find("end-of-candidates") != std::string::npos) continue;
            std::string mid = c.sdpMid.empty() ? bundleMid : c.sdpMid;
            // The address field of "candidate:<f> <c> <proto> <prio> <addr> <port> ...".
            std::vector<std::string> parts;
            size_t p = 0;
            while (p < c.candidate.size()) {
                size_t e = c.candidate.find(' ', p);
                if (e == std::string::npos) e = c.candidate.size();
                parts.push_back(c.candidate.substr(p, e - p));
                p = e + 1;
            }
            std::string ipv4;
            int port = 0;
            if (parts.size() > 4 && decodeTeredo(parts[4], ipv4, port)) {
                // The console's public address: not in the log.
                XC_LOGI("teredo candidate -> port %d (and 9002)", port);
                addRemoteCandidate("a=candidate:10 1 UDP 1 " + ipv4 + " 9002 typ host", mid);
                addRemoteCandidate("a=candidate:11 1 UDP 1 " + ipv4 + " " + std::to_string(port) + " typ host", mid);
            }
            addRemoteCandidate(c.candidate, mid);
        }

        if (!waitFor([this] { return handshakeDone; }, opt.connectTimeoutMs)) {
            std::lock_guard<std::mutex> lock(mutex);
            err = failed ? failReason : "timed out connecting the stream";
            return false;
        }
        lastKeepaliveMs = lastKeyframeMs = platform::nowMs();
        return true;
    }

    void requestKeyframe() {
        json::Value v = json::Value::object();
        v.set("message", "videoKeyframeRequested");
        v.set("ifrRequested", true);
        sendText(control, v.dump());
    }

    void tick() {
        if (!open) return;
        uint64_t now = platform::nowMs();
        if (now - lastKeepaliveMs >= 30000) {
            lastKeepaliveMs = now;
            // Only this thread starts one. While the last one is still out
            // (a slow network), none more; once it is done, its thread is
            // joined before the next is started: never a new std::thread
            // moved onto one still joinable (std::terminate), as checking
            // and then exchanging the flag allowed when the thread ended in
            // between.
            if (!keepaliveInFlight.load()) {
                if (keepaliveThread.joinable()) keepaliveThread.join();
                keepaliveInFlight = true;
                platform::startThread(keepaliveThread, [this] {
                    std::string err;
                    if (!gssv.keepalive(err)) XC_LOGW("%s", err.c_str());
                    keepaliveInFlight = false;
                }, 512u << 10);
            }
        }
        if (opt.keyframeIntervalSec > 0 && now - lastKeyframeMs >= static_cast<uint64_t>(opt.keyframeIntervalSec) * 1000u) {
            lastKeyframeMs = now;
            requestKeyframe();
        }
    }

    // Before the session goes: no callback of libdatachannel's (they hold
    // `this`) may run any more. Each reset waits for a call in progress.
    void detach() {
        if (keepaliveThread.joinable()) {  // after close(): tick() starts no other
            keepaliveThread.join();
        }
        {
            std::lock_guard<std::mutex> lock(guard->mutex);
            guard->self = nullptr;
        }
        try {
            for (const auto& ch : {chat, control, input, message})
                if (ch) ch->resetCallbacks();
            for (const auto& track : {audio, video})
                if (track) {
                    track->onFrame(nullptr);
                    track->resetCallbacks();
                }
            if (pc) pc->resetCallbacks();
        } catch (const std::exception& e) {
            XC_LOGW("detaching the session: %s", e.what());
        }
    }

    void close() {
        open = false;
        if (pc) {
            try {
                pc->close();
            } catch (const std::exception& e) {
                XC_LOGW("closing peer connection: %s", e.what());
            }
        }
    }

    ~Impl() {
        if (keepaliveThread.joinable()) {
            keepaliveThread.join();
        }
    }
};

StreamSession::StreamSession(xcloud::GssvClient& gssv, StreamCallbacks callbacks, StreamOptions options)
    : impl_(std::make_unique<Impl>(gssv, std::move(callbacks), options)) {}

StreamSession::~StreamSession() {
    impl_->closedNotified = true;  // no callback while tearing down
    impl_->close();
    impl_->detach();
}

bool StreamSession::start(std::string& err) {
    // libdatachannel reports set-up failures (e.g. no usable socket) by throwing.
    try {
        return impl_->start(err);
    } catch (const std::exception& e) {
        err = std::string("WebRTC: ") + e.what();
    } catch (...) {
        err = "WebRTC: unknown error";
    }
    impl_->close();
    return false;
}
bool StreamSession::isOpen() const { return impl_->open; }

void StreamSession::sendMouse(const MouseFrame& frame) {
    if (!impl_->open) return;
    Impl::sendBinary(impl_->input, mouseReport(impl_->inputSequence++, impl_->nowMs(), {frame}));
}

void StreamSession::sendKeys(const std::vector<KeyFrame>& keys) {
    if (!impl_->open || keys.empty()) return;
    Impl::sendBinary(impl_->input, keyboardReport(impl_->inputSequence++, impl_->nowMs(), keys));
}

void StreamSession::sendGamepad(const GamepadFrame& frame) {
    if (!impl_->open) return;
    Impl& d = *impl_;
    if (d.padsStale.exchange(false)) {
        uint32_t attached = d.padsAttached;
        for (int i = 0; i < 4; ++i)
            if (attached & (1u << i)) d.sendGamepadChanged(i, true);
        for (auto& ms : d.lastPadMs) ms = 0;  // and every pad's state right away
    }
    uint64_t now = platform::nowMs();
    if (frame.index < 4) {
        uint64_t& lastMs = d.lastPadMs[frame.index];
        if (lastMs && frame == d.lastPad[frame.index] && now - lastMs < Impl::kPadRepeatMs) return;
        d.lastPad[frame.index] = frame;
        lastMs = now;
    }
    if (!d.loggedFirstView && (frame.buttons & stream::kView)) {
        d.loggedFirstView = true;
        XC_LOGI("input: first kView sent (pad %u)", frame.index);
    }
    if (!d.loggedFirstMenu && (frame.buttons & stream::kMenu)) {
        d.loggedFirstMenu = true;
        XC_LOGI("input: first kMenu sent (pad %u)", frame.index);
    }
    uint8_t reportBuf[stream::kGamepadReportSize];
    size_t reportLen = stream::writeGamepadReport(reportBuf, sizeof(reportBuf), d.inputSequence++, d.nowMs(), frame);
    Impl::sendBinary(d.input, reportBuf, reportLen);
}

void StreamSession::sendAudio(const uint8_t* opus, size_t size, uint32_t timestamp) {
    if (!impl_ || !impl_->open || !impl_->audio || !opus || size == 0) return;
    try {
        rtc::FrameInfo info(timestamp);
        impl_->audio->sendFrame(reinterpret_cast<const rtc::byte*>(opus), size, info);
    } catch (const std::exception&) {
    }
}

void StreamSession::setTouchEnabled(bool on) {
    if (!impl_->open) return;
    json::Value touch = json::Value::object();
    touch.set("touchInputEnabled", on);
    Impl::sendText(impl_->message, impl_->messageEnvelope("/streaming/characteristics/touchinputenabledchanged", touch));
}

void StreamSession::setGamepadConnected(int index, bool connected) {
    if (!impl_->open) return;
    XC_LOGI("gamepad %d %s", index, connected ? "attached" : "detached");
    impl_->sendGamepadChanged(index, connected);
}

void StreamSession::requestKeyframe() {
    try {
        impl_->requestKeyframe();
    } catch (const std::exception& e) {
        XC_LOGW("keyframe request: %s", e.what());
    }
}
void StreamSession::reportFrame(const FrameMetadata& frame) {
    if (!impl_->open) return;
    Impl::sendBinary(impl_->input, metadataReport(impl_->inputSequence++, impl_->nowMs(), {frame}));
}

double StreamSession::clockMs() const { return stream::clockMs(); }

void StreamSession::simulateDrop() {
    XC_LOGW("test: dropping the connection");
    if (impl_->pc) impl_->pc->close();
}

double clockMs() {
    static const auto epoch = std::chrono::steady_clock::now();
    return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - epoch).count();
}

void StreamSession::completeTextInput(const std::string& id, const std::string& text) {
    json::Value v = json::Value::object();
    v.set("Text", text);
    impl_->completeTransaction(id, v);
}

void StreamSession::cancelTextInput(const std::string& id) { impl_->cancelTransaction(id); }

void StreamSession::requestResolution(const std::string& alias) { impl_->sendResolution(alias); }

void StreamSession::tick() { impl_->tick(); }
void StreamSession::close() { impl_->close(); }
int StreamSession::rttMs() const {
    try {
        if (impl_->pc)
            if (auto rtt = impl_->pc->rtt()) {
                const int ms = static_cast<int>(rtt->count());
                if (impl_->videoReceiverRaw) impl_->videoReceiverRaw->setRttMs(ms);  // its loss wait follows it
                return ms;
            }
    } catch (const std::exception&) {
    }
    return -1;
}

const VideoReceiveStats& StreamSession::videoStats() const {
    if (impl_->videoReporter) {
        impl_->videoStats.receiveRate = impl_->videoReporter->receiveRate();
        impl_->videoStats.estimate = impl_->videoReporter->estimate();
    }
    return impl_->videoStats;
}

}  // namespace xc::stream