Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

feat: add slow upgrade backoff, foreground resume triggers, and readiness checks for P2P terminal connections

0xken156f0bbc0cf2cf
diff --git a/desktop/lib/ws/ws_conn.dart b/desktop/lib/ws/ws_conn.dart
index e66c38f1a..e48e2e6ef 100644
--- a/desktop/lib/ws/ws_conn.dart
+++ b/desktop/lib/ws/ws_conn.dart
@@ -499,7 +499,11 @@ class WsConn {
var openViaPlugin = false;
final plugin = _plugin;
final requestId = payload['requestId'];
- if (plugin != null && type == 'terminal_open' && requestId is String) {
+ // Not worth the wait when the frame cannot go anyway.
+ if (plugin != null &&
+ isReady &&
+ type == 'terminal_open' &&
+ requestId is String) {
openViaPlugin = await plugin.prepareOpen(requestId);
}
return _enqueueFrame(
diff --git a/mobile/lib/p2p/terminal_p2p_plugin.dart b/mobile/lib/p2p/terminal_p2p_plugin.dart
index 83426ba88..8f99bc572 100644
--- a/mobile/lib/p2p/terminal_p2p_plugin.dart
+++ b/mobile/lib/p2p/terminal_p2p_plugin.dart
@@ -34,10 +34,15 @@ const _migrationTtl = Duration(seconds: 30);
const _migrationSweepEvery = Duration(seconds: 20);
// A connection that reached 'direct' only through a TURN pair still costs per GB —
-// worth periodically trying for a truly direct one. Never touches the live
-// connection until a replacement has already proven itself.
-const _upgradeMaxAttempts = 3;
+// and, what a person actually notices, a relay round trip on every keystroke. Worth
+// periodically trying for a truly direct one; the trial never touches the live
+// connection until a replacement has already proven itself. The CLI gives up for
+// good after three tries; a phone's relay socket can live in the foreground for
+// hours, so after those three it keeps trying at a slow pace instead — and again
+// at once when the app comes back, since that usually means the network changed.
+const _upgradeQuickAttempts = 3;
const _upgradeRetryDelay = Duration(seconds: 60);
+const _upgradeSlowRetryDelay = Duration(minutes: 15);
const _upgradeAttemptTimeout = Duration(seconds: 15);
const _upgradeDrainTimeout = Duration(seconds: 5);
const _promoteAckTimeout = Duration(seconds: 5);
@@ -125,6 +130,10 @@ class TerminalP2pPlugin implements TerminalTransportPlugin {
final _upgradeDraining = <String, DateTime>{};
void Function()? _upgradeWaitResolve;
TerminalP2pLink? _upgradeOrphan;
+ DateTime? _lastUpgradeAttemptAt;
+
+ /// Set only once a cutover succeeded — there is nothing left to upgrade to. A
+ /// demotion resets it along with the attempt count: the next link is its own.
bool _upgradeDone = false;
String get _sid =>
@@ -194,15 +203,28 @@ class TerminalP2pPlugin implements TerminalTransportPlugin {
onDisposed?.call(this);
}
- /// The app came back to the foreground: a retry waiting out its delay fires now.
+ /// The app came back to the foreground: a retry waiting out its delay fires now,
+ /// and a link sitting on TURN gets an upgrade trial at once — coming back usually
+ /// means the network changed, which is exactly when a direct pair may have
+ /// become possible.
void kickRetry() {
+ if (_disposed) return;
final timer = _retryTimer;
- if (timer == null || _disposed) return;
- timer.cancel();
- _retryTimer = null;
- _log('retry kicked · app resumed');
- _link = null;
- _startP2p();
+ if (timer != null) {
+ timer.cancel();
+ _retryTimer = null;
+ _log('retry kicked · app resumed');
+ if (_link == null) _startP2p();
+ return;
+ }
+ if (_link?.transport == TerminalP2pTransport.relay &&
+ _upgradeShadow == null &&
+ !_upgradeDone) {
+ _upgradeTimer?.cancel();
+ _upgradeTimer = null;
+ _log('upgrade kicked · app resumed');
+ _attemptUpgrade();
+ }
}
// ── Negotiation ───────────────────────────────────────────────────────────
@@ -300,9 +322,7 @@ class TerminalP2pPlugin implements TerminalTransportPlugin {
}
_retryTimer = Timer(delay, () {
_retryTimer = null;
- if (_disposed) return;
- _link = null;
- _startP2p();
+ if (!_disposed && _link == null) _startP2p();
});
}
@@ -560,14 +580,23 @@ class TerminalP2pPlugin implements TerminalTransportPlugin {
// ── Demotion, migration ───────────────────────────────────────────────────
Future<void> _demote(String reason) async {
- if (_disposed) return;
+ // Two causes can race (a refused send and relay-delivered output, say); the
+ // second finds nothing left to demote.
+ if (_disposed || (_link == null && _p2pStreams.isEmpty)) return;
// The ONE place every demotion converges, so none can fall through silently.
_log('demoted · reason=$reason streams=${_p2pStreams.length}');
_upgradeTimer?.cancel();
_upgradeTimer = null;
+ _upgradeAttempts = 0;
+ _upgradeDone = false;
final shadow = _upgradeShadow;
_upgradeShadow = null;
if (shadow != null) unawaited(shadow.stop(reason: 'primary_demoted'));
+ // An orphan from an unanswered promote was only worth keeping beside the
+ // primary it was cut over from; with that gone it is just a TURN allocation.
+ final orphan = _upgradeOrphan;
+ _upgradeOrphan = null;
+ if (orphan != null) unawaited(orphan.stop(reason: 'primary_demoted'));
_upgradeDraining.clear();
_upgradeWaitResolve = null;
final link = _link;
@@ -634,11 +663,10 @@ class TerminalP2pPlugin implements TerminalTransportPlugin {
void _scheduleUpgradeAttempt() {
if (_upgradeTimer != null || _upgradeShadow != null || _upgradeDone) return;
- if (_upgradeAttempts >= _upgradeMaxAttempts) {
- _upgradeDone = true;
- return;
- }
- _upgradeTimer = Timer(_upgradeRetryDelay, () {
+ final delay = _upgradeAttempts < _upgradeQuickAttempts
+ ? _upgradeRetryDelay
+ : _upgradeSlowRetryDelay;
+ _upgradeTimer = Timer(delay, () {
_upgradeTimer = null;
if (!_disposed) _attemptUpgrade();
});
@@ -657,6 +685,11 @@ class TerminalP2pPlugin implements TerminalTransportPlugin {
policy == null) {
return;
}
+ // A resume can ask at any moment; one full negotiation a minute is plenty.
+ final now = _now();
+ final last = _lastUpgradeAttemptAt;
+ if (last != null && now.difference(last) < _upgradeRetryDelay) return;
+ _lastUpgradeAttemptAt = now;
_upgradeAttempts++;
final attempt = _upgradeAttempts;
late final TerminalP2pLink shadow;
@@ -705,12 +738,11 @@ class TerminalP2pPlugin implements TerminalTransportPlugin {
}
void _finishUpgradeAttempt() {
- if (_upgradeAttempts >= _upgradeMaxAttempts) {
- _upgradeDone = true;
+ if (_upgradeAttempts == _upgradeQuickAttempts) {
_log(
- 'upgrade giving up · staying on turn after $_upgradeAttempts attempts',
+ 'upgrade slowing down · still on turn after $_upgradeAttempts attempts,'
+ ' retrying every ${_upgradeSlowRetryDelay.inMinutes}min',
);
- return;
}
_scheduleUpgradeAttempt();
}
@@ -774,8 +806,8 @@ class TerminalP2pPlugin implements TerminalTransportPlugin {
'protocolVersion': terminalP2pProtocolVersion,
});
}, _promoteAckTimeout);
- _upgradeDone = true; // either way, this connection is done trying
if (acked) {
+ _upgradeDone = true; // nothing left to upgrade to
unawaited(old.stop(reason: 'upgraded', notifyPeer: false));
_log('upgrade promoted · attempt=$_upgradeAttempts');
// The streams did not change wire from their own point of view, but the
@@ -787,7 +819,9 @@ class TerminalP2pPlugin implements TerminalTransportPlugin {
}
} else {
// Keep the old connection alive rather than guess it is safe to close; it is
- // closed when the plugin itself is disposed.
+ // closed when the plugin itself is disposed. The cutover itself stands, so
+ // this link is direct now and there is nothing more to try.
+ _upgradeDone = true;
_upgradeOrphan = old;
_log('upgrade promote ack timeout · keeping the old connection open');
}
diff --git a/mobile/lib/p2p/webrtc_terminal_p2p_link.dart b/mobile/lib/p2p/webrtc_terminal_p2p_link.dart
index d8e155038..f48ff66a3 100644
--- a/mobile/lib/p2p/webrtc_terminal_p2p_link.dart
+++ b/mobile/lib/p2p/webrtc_terminal_p2p_link.dart
@@ -97,9 +97,12 @@ class WebRtcTerminalP2pLink implements TerminalP2pLink {
Completer<void>? _reflexive;
Completer<void>? _drained;
- /// Candidates gathered after the offer left. Held until the answer is in: the
+ /// Candidates not carried by the offer. Held until the answer is in: the
/// responder only adds a candidate to an entry that already has a remote
/// description, and one arriving earlier is dropped or tears the session down.
+ /// Before the offer goes out everything lands here too, and what its SDP turns
+ /// out to carry is dropped again then — the alternative, ignoring candidates
+ /// until the SDP is known, loses the ones reported while it is being read.
final _lateCandidates = <RTCIceCandidate>[];
String _offeredSdp = '';
@@ -195,6 +198,7 @@ class WebRtcTerminalP2pLink implements TerminalP2pLink {
throw StateError('local_description_missing');
}
_offeredSdp = sdp;
+ _lateCandidates.removeWhere((c) => sdp.contains(c.candidate ?? ''));
sendSignal('p2p_offer', {
'sessionId': sessionId,
'protocolVersion': terminalP2pProtocolVersion,
@@ -242,8 +246,11 @@ class WebRtcTerminalP2pLink implements TerminalP2pLink {
}
case 'p2p_abort':
final reason = payload['reason'];
+ // The peer's word for it, bounded before it reaches a log line.
_fail(
- reason is String && reason.isNotEmpty ? reason : 'peer_aborted',
+ reason is String && reason.isNotEmpty
+ ? (reason.length > 64 ? reason.substring(0, 64) : reason)
+ : 'peer_aborted',
);
}
} catch (error) {
@@ -283,10 +290,12 @@ class WebRtcTerminalP2pLink implements TerminalP2pLink {
final message = data is String
? RTCDataChannelMessage(data)
: RTCDataChannelMessage.fromBinary(asBytes(data));
- // The plugin's send is a Future; a rejection is a channel that stopped being
- // usable. Reported on a later turn so the caller's relay fallback for THIS
- // frame goes out before any resync a failure triggers — otherwise the
- // higher-counter resync would arrive first and read as a replay.
+ // The plugin's send is a Future, so a rejection lands after this has already
+ // answered true and the frame is gone — unlike the CLI's synchronous werift
+ // send, which hands a refused frame back for the relay. Rare (the channel
+ // has to die between the state check and the native call); the demotion's
+ // resync restores the screen, and a lost keystroke is retyped. Reported on a
+ // later turn so nothing of the failure precedes the frames already queued.
unawaited(
channel.send(message).catchError((Object error) {
appLog.warn('p2p', 'data channel send failed', error: error);
@@ -367,6 +376,7 @@ class WebRtcTerminalP2pLink implements TerminalP2pLink {
_timeout = null;
_clearDisconnectGrace();
_settleWaiters(false);
+ _lateCandidates.clear();
_drained?.complete();
_drained = null;
if (notifyPeer) {
@@ -421,7 +431,7 @@ class WebRtcTerminalP2pLink implements TerminalP2pLink {
if (reflexive != null && !reflexive.isCompleted) reflexive.complete();
}
// Everything gathered before the offer left is already inside its SDP.
- if (_offeredSdp.isEmpty || _offeredSdp.contains(line)) return;
+ if (_offeredSdp.isNotEmpty && _offeredSdp.contains(line)) return;
if (_sawAnswer) {
_trickle(candidate);
} else {
diff --git a/mobile/test/p2p/terminal_p2p_plugin_test.dart b/mobile/test/p2p/terminal_p2p_plugin_test.dart
index 2b0384910..7ae89aa19 100644
--- a/mobile/test/p2p/terminal_p2p_plugin_test.dart
+++ b/mobile/test/p2p/terminal_p2p_plugin_test.dart
@@ -477,25 +477,64 @@ void main() {
});
});
- test(
- 'a shadow that also lands on turn is abandoned, three tries then done',
- () {
- fakeAsync((async) {
- final b = onTurn(async);
- for (var attempt = 1; attempt <= 3; attempt++) {
- async.elapse(const Duration(seconds: 60));
- expect(b.links.created, hasLength(1 + attempt));
- b.links.last.open(transport: TerminalP2pTransport.relay);
- flush(async);
- expect(b.links.last.stopReason, 'upgrade_no_gain');
- }
- async.elapse(const Duration(minutes: 5));
- expect(b.links.created, hasLength(4), reason: 'gave up');
- expect(b.links.first.stopped, isFalse);
- b.plugin.dispose();
- });
- },
- );
+ test('a shadow that also lands on turn is abandoned; three quick tries, then slow', () {
+ fakeAsync((async) {
+ final b = onTurn(async);
+ for (var attempt = 1; attempt <= 3; attempt++) {
+ async.elapse(const Duration(seconds: 60));
+ expect(b.links.created, hasLength(1 + attempt));
+ b.links.last.open(transport: TerminalP2pTransport.relay);
+ flush(async);
+ expect(b.links.last.stopReason, 'upgrade_no_gain');
+ }
+ async.elapse(const Duration(minutes: 5));
+ expect(b.links.created, hasLength(4), reason: 'slowed down, not yet');
+ async.elapse(const Duration(minutes: 10));
+ expect(b.links.created, hasLength(5), reason: 'every 15 min from now');
+ expect(b.links.last.upgrade, isTrue);
+ expect(b.links.first.stopped, isFalse);
+ b.plugin.dispose();
+ });
+ });
+
+ test('a demotion gives the next link a fresh upgrade budget', () {
+ fakeAsync((async) {
+ final b = onTurn(async);
+ for (var attempt = 1; attempt <= 3; attempt++) {
+ async.elapse(const Duration(seconds: 60));
+ b.links.last.open(transport: TerminalP2pTransport.relay);
+ flush(async);
+ }
+ // The primary dies; the retry lands on TURN again.
+ b.links.first.drop('peer_failed');
+ flush(async);
+ async.elapse(const Duration(seconds: 60));
+ final second = b.links.last;
+ expect(second.upgrade, isFalse);
+ second.open(transport: TerminalP2pTransport.relay);
+ flush(async);
+ // Quick attempts again, not the slow pace.
+ async.elapse(const Duration(seconds: 60));
+ expect(b.links.last.upgrade, isTrue);
+ expect(b.links.last, isNot(second));
+ b.plugin.dispose();
+ });
+ });
+
+ test('coming back to the foreground on turn tries an upgrade at once', () {
+ fakeAsync((async) {
+ final b = onTurn(async);
+ async.elapse(const Duration(seconds: 5));
+ expect(b.links.created, hasLength(1));
+ b.plugin.kickRetry();
+ expect(b.links.created, hasLength(2));
+ expect(b.links.last.upgrade, isTrue);
+ // Not twice while that trial is still running.
+ b.plugin.kickRetry();
+ expect(b.links.created, hasLength(2));
+ b.plugin.dispose();
+ });
+ });
test(
'an unanswered promote keeps the old connection open as an orphan',
@@ -511,8 +550,11 @@ void main() {
flush(async);
async.elapse(const Duration(seconds: 5));
expect(primary.stopped, isFalse);
+ // The promoted shadow dying takes the orphan with it.
+ shadow.drop('peer_failed');
+ flush(async);
+ expect(primary.stopReason, 'primary_demoted');
b.plugin.dispose();
- expect(primary.stopped, isTrue);
expect(shadow.stopped, isTrue);
});
},