Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Batch N: Lead poll reads only live review groups (#2073), Desktop chat cache cap (#2074) (#2076)

0xken15679d647f09a3
diff --git a/backend/integration/superTabGroupPlans.test.ts b/backend/integration/superTabGroupPlans.test.ts
index 0f9fdf9a9..debfe9954 100644
--- a/backend/integration/superTabGroupPlans.test.ts
+++ b/backend/integration/superTabGroupPlans.test.ts
@@ -16,6 +16,7 @@ import { recentDormantGroups } from '../src/superTabs/leaderboard/read.js'
import { decisionsWaiting, listDecisions } from '../src/superTabs/decisions.js'
import { GROUP_INDEX, groupsBy, listGroupIndexes, retryStaleIndex } from '../src/superTabs/parts/indexed.js'
import { logger } from '../src/utils/logger.js'
+import { IDLE_STOP_MS } from '../src/superTabs/tidy.js'
import type { SuperTabState } from '../src/superTabs/model.js'
import { writeLegacySwarm } from '../scripts/swarm-parts/legacy.js'
import { NOW, owner as ownerActor, swarm } from '../src/superTabs/read/fixture.test-support.js'
@@ -95,9 +96,9 @@ async function scene() {
}
/** One read as the profiler saw it: its command and every getMore (they share a query hash). */
-interface Read { shape: string; plans: string[]; multiPlanned: boolean; planningMicros: number; docsExamined: number; returned: number }
+interface Read { shape: string; plans: string[]; multiPlanned: boolean; planningMicros: number; docsExamined: number; returned: number; bytes: number }
interface Row { ns: string; op: string; match?: unknown; planSummary?: string; docsExamined?: number; nreturned?: number; queryHash?: string
- fromMultiPlanner?: boolean; planningTimeMicros?: number; command?: { filter?: unknown; q?: unknown } }
+ fromMultiPlanner?: boolean; planningTimeMicros?: number; responseLength?: number; command?: { filter?: unknown; q?: unknown } }
/** The SwarmGroup reads `run` sends, with the plan cache emptied first: a read the planner would serve from its cache in a
* test is planned again in prod whenever the cache misses. */
@@ -113,20 +114,21 @@ async function profiled(run: () => Promise<unknown>): Promise<Read[]> {
await db.$runCommandRaw({ profile: 0 })
}
const stages = [{ $match: { ns: { $regex: '\\.SwarmGroup$' }, op: { $in: ['command', 'getmore', 'query'] } } },
- { $project: { ...PROFILE_STAGES[0]!.$project, op: 1, queryHash: 1, fromMultiPlanner: 1, planningTimeMicros: 1 } }]
+ { $project: { ...PROFILE_STAGES[0]!.$project, op: 1, queryHash: 1, fromMultiPlanner: 1, planningTimeMicros: 1, responseLength: 1 } }]
const rows = (await db.$runCommandRaw({ aggregate: 'system.profile', pipeline: stages, cursor: { batchSize: 100000 } }) as unknown as
{ cursor: { firstBatch: Row[] } }).cursor.firstBatch
const reads = new Map<string, Read>()
for (const row of rows) {
const id = row.queryHash ?? `${reads.size}`
const filter = row.match ?? row.command?.filter ?? row.command?.q
- const read = reads.get(id) ?? { shape: '', plans: [], multiPlanned: false, planningMicros: 0, docsExamined: 0, returned: 0 }
+ const read = reads.get(id) ?? { shape: '', plans: [], multiPlanned: false, planningMicros: 0, docsExamined: 0, returned: 0, bytes: 0 }
if (filter !== undefined && !read.shape) read.shape = queryShape(filter)
if (row.planSummary && !read.plans.includes(row.planSummary)) read.plans.push(row.planSummary)
read.multiPlanned ||= !!row.fromMultiPlanner
read.planningMicros += Number(row.planningTimeMicros ?? 0)
read.docsExamined += Number(row.docsExamined ?? 0)
read.returned += Number(row.nreturned ?? 0)
+ read.bytes += Number(row.responseLength ?? 0)
reads.set(id, read)
}
return [...reads.values()]
@@ -261,21 +263,31 @@ describe('hot SwarmGroup reads name their index and read only what they return (
clock = MIGRATED + 60_000
const expected = s.quietLead.filter(r => r.lastActivityAt.getTime() >= clock - DAY).length
const all = await profiled(() => store.deliveries(owner, s.tabId, s.lead))
- const reads = where(all, ['tabId', 'dormant', 'lastActivityAt'])
+ const reads = where(all, ['tabId', 'dormant', 'lastActivityAt', 'projectId'])
expect(reads).toHaveLength(1)
expectIndexed(reads)
expect(reads[0]!.returned).toBe(expected)
expect(reads[0]!.docsExamined, JSON.stringify(reads)).toBeLessThanOrEqual(expected)
- // The same poll reads the quiet conversations its project's kept Reviewer is in (every quiet one here), through the
- // Reviewer's conversations: no index has both.
+ // The same poll reads the quiet conversations its project's kept Reviewer is in, through the Reviewer's conversations: no
+ // index has both. Only those active in the last 15 minutes can still keep it (tidy.ts); none reviews open work here.
const asleep = where(all, ['tabId', 'dormant', 'agentIds'])
expect(asleep).toHaveLength(1)
expectIndexed(asleep)
- expect(asleep[0]!.returned).toBe(s.quietLead.length + s.quietOther.length)
+ const recent = [...s.quietLead, ...s.quietOther].filter(r => r.lastActivityAt.getTime() > clock - IDLE_STOP_MS).length
+ expect(asleep[0]!.returned).toBe(recent)
const its = await db.swarmGroup.count({ where: { tabId: s.tabId, agentIds: { has: s.reviewer } } })
expect(asleep[0]!.docsExamined, JSON.stringify(asleep)).toBeLessThanOrEqual(its)
})
+ it('a Lead\'s poll reads none of the briefs or checks of the quiet conversations its Reviewers are in (Diego, 10-29)', async () => {
+ clock = MIGRATED + 60_000
+ const reads = where(await profiled(() => store.deliveries(owner, s.tabId, s.lead)), ['tabId', 'dormant', 'agentIds'])
+ expect(reads).toHaveLength(1)
+ // Each conversation here has a 2000-byte brief: a row that carried one, or its checks, would pass this alone.
+ expect(reads[0]!.returned).toBeGreaterThan(0)
+ expect(reads[0]!.bytes / reads[0]!.returned, JSON.stringify(reads)).toBeLessThan(500)
+ })
+
it('the host\'s wake check finds a change that concerns its machine, reading only the conversations changed since the cursor', async () => {
const hosts = new HostReads(store, { now: () => clock })
clock = Date.now()
diff --git a/backend/integration/superTabTidy.test.ts b/backend/integration/superTabTidy.test.ts
index 50040c968..69bdcd9f4 100644
--- a/backend/integration/superTabTidy.test.ts
+++ b/backend/integration/superTabTidy.test.ts
@@ -166,6 +166,45 @@ describe('a worker left idle', () => {
expect((await reviewerOf()).workDecision).toBe('stop')
})
+ it('is kept, as a Reviewer, while it was active in the last 15 minutes in a review group now asleep: a message, its progress or its loss', async () => {
+ // The Lead's poll reads asleep review groups only when they can still keep a Reviewer (store.ts fullDeliveries): each
+ // way of being active there must still count, and stop counting 15 minutes later.
+ const cases: Array<[string, (data: Record<string, unknown>, at: number) => Record<string, unknown>]> = [
+ ['a message', (data, at) => ({ ...data, lastActivityAt: new Date(at).toISOString() })],
+ ['its progress', (data, at) => ({ ...data, assignment: { ...data.assignment as object, progressAt: at } })],
+ ['its loss', (data, at) => ({ ...data, assignment: { ...data.assignment as object, lost: { reason: 'no progress', at } } })],
+ ]
+ for (const [name, active] of cases) {
+ const f = await swarm()
+ // It reviews the parser, whose work is already cancelled: no re-check can come.
+ const review = String((await f.execute({ type: 'conversation.create', title: 'Review: Parser', brief: 'Review it', reviewOf: f.group,
+ checks: [{ description: 'Verdict', independent: false }] })).result.conversationId)
+ const reviewerHarness = randomUUID()
+ const reviewer = await f.add(review, 'test-host', reviewerHarness, 'Reviewer')
+ await f.started(reviewer, { role: 'reviewer' })
+ const verdict = String((await f.execute({ type: 'work.assign', conversationId: review, agentId: reviewer })).result.assignmentId)
+ await f.execute({ type: 'work.cancel', conversationId: review, assignmentId: verdict, reason: 'Verdict given' })
+ await f.execute({ type: 'conversation.archive', conversationId: review, archived: true })
+ await f.delivered()
+ // Quiet for an hour by its messages, then active 10 minutes ago the way this case says, and asleep.
+ const part = await db.swarmGroup.findFirstOrThrow({ where: { tabId: f.tab, key: review } })
+ const quiet = { ...part.data as Record<string, unknown>, dormant: true, lastActivityAt: new Date(f.clock.now - 60 * MINUTE).toISOString() }
+ f.clock.now += 20 * MINUTE
+ const data = active(quiet, f.clock.now - 10 * MINUTE)
+ await db.swarmGroup.update({ where: { id: part.id }, data: { dormant: true, lastActivityAt: new Date(String(data.lastActivityAt)), data: data as object } })
+ const reviewerOf = async () => (await f.timed.read(owner, f.tab)).state.agents.find(a => a.id === reviewer)!
+ const poll = async () => {
+ await f.observe('test-host', [f.harness.worker, reviewerHarness], { harnessUncommitted: { [f.harness.worker]: false, [reviewerHarness]: false } })
+ await f.timed.deliveries(host, f.tab, f.lead)
+ }
+ await poll()
+ expect((await reviewerOf()).workDecision, name).not.toBe('stop')
+ f.clock.now += 5 * MINUTE
+ await poll()
+ expect((await reviewerOf()).workDecision, name).toBe('stop')
+ }
+ })
+
it('is kept once the owner turned tidying off', async () => {
const f = await swarm()
await f.execute({ type: 'tab.tidy', enabled: false })
diff --git a/backend/src/superTabs/store.ts b/backend/src/superTabs/store.ts
index f156fc249..d106f2121 100644
--- a/backend/src/superTabs/store.ts
+++ b/backend/src/superTabs/store.ts
@@ -24,7 +24,7 @@ import { confidentialSessions, otherRegistrations, recordConfidential } from './
import { answersRequest, dirtyGroups, noteProjectReports, projectUpdateRequest, UNANSWERED_REQUEST_MS } from './projectUpdates.js'
import { digestDue, digestMessageId, digestRequest, digestSince } from './digest.js'
import { judgedMachines, loseTasks, sweepLiveness, SWEEP_EVERY_MS } from './liveness.js'
-import { archivable, archiveLine, stopIdleWorkers, stoppedMachines } from './tidy.js'
+import { archivable, archiveLine, ASLEEP_FIELDS, asleepMatch, stopIdleWorkers, stoppedMachines, type GroupActivity } from './tidy.js'
import { clock, insertParts, loadActive, reassemble, split, type HeadData, type SwarmParts } from './parts/index.js'
import { groupsBy, listingDrops, quietSince, withoutHints } from './parts/indexed.js'
import { commitParts, indexesOf, numberMessages, type Touch, type Written } from './parts/commit.js'
@@ -2185,10 +2185,13 @@ export class SuperTabStore {
// A Lead's poll runs the time checks (liveness.ts): lost work wakes it in this same poll.
const written = ledProject(state, agentId) ? sweepLiveness(state, this.now()) : []
// The Lead's poll stops its project's idle workers too (tidy.ts): it is where what waits for each agent is known.
+ // Of their asleep groups, it reads only those that can still keep one, and only what the stop reads (tidy.ts asleepMatch),
+ // judged at the same time as the stop.
const reviewers = led ? state.agents.filter(a => agentRole(a) === 'reviewer' && a.workDecision !== 'stop' && agentProjectId(state, a) === led.id).map(a => a.id) : []
- const asleep = reviewers.length ? (await groupsBy(tx, 'agents', [{ $match: { tabId, deleted: false, dormant: true, agentIds: { $in: reviewers } } },
- { $project: { _id: 0, data: 1 } }])).map(row => row.data as unknown as Conversation) : []
- const idle = led ? stopIdleWorkers(state, led.id, this.now(), new Set((await tx.superTabDelivery.findMany({
+ const now = this.now()
+ const asleep = reviewers.length ? (await groupsBy(tx, 'agents', [{ $match: asleepMatch(state, tabId, reviewers, now) },
+ { $project: { _id: 0, ...ASLEEP_FIELDS } }])).map(row => row.data as unknown as GroupActivity) : []
+ const idle = led ? stopIdleWorkers(state, led.id, now, new Set((await tx.superTabDelivery.findMany({
where: { tabId, status: { in: [...OPEN_DELIVERY] } }, select: { agentId: true }, distinct: ['agentId'] })).map(d => d.agentId)), asleep) : []
written.push(...idle)
const update = projectUpdateRequest(state, target, this.now())
diff --git a/backend/src/superTabs/tidy.ts b/backend/src/superTabs/tidy.ts
index 97cbb122e..02f9876bf 100644
--- a/backend/src/superTabs/tidy.ts
+++ b/backend/src/superTabs/tidy.ts
@@ -1,7 +1,8 @@
+import type { Prisma } from '@prisma/client'
import { isBuddy } from './buddies.js'
import { agentDisplayName } from './agentNames.js'
import { verdictPending } from './autoReview.js'
-import { activeWork, agentRole, stableId, type Agent, type Conversation, type MessageDraft, type SuperTabState } from './model.js'
+import { activeWork, agentRole, stableId, type Agent, type Assignment, type Conversation, type MessageDraft, type SuperTabState } from './model.js'
import { agentProjectId, groupProject, projectsOf } from './projects.js'
// Nobody has to tidy a Swarm by hand (roadmap, December). A Builder left with nothing to do for 15 minutes is stopped, as
@@ -16,20 +17,45 @@ const REPORT_FRESH_MS = 2 * 60_000
const FINISHED = ['done', 'cancelled', 'stopped', 'lost']
const time = (value: string | undefined) => value === undefined ? undefined : Date.parse(value)
+/** All stopIdleWorkers reads of a group: the Lead's poll reads only this of an asleep one (ASLEEP_FIELDS). */
+export type GroupActivity = Pick<Conversation, 'id' | 'lastActivityAt' | 'archived' | 'agentIds' | 'reviewOf'> &
+ { assignment?: (Pick<Assignment, 'agentId' | 'status' | 'progressAt'> & { lost?: Pick<NonNullable<Assignment['lost']>, 'at'> }) | null }
+
+/** The fields of an asleep group's part the Lead's poll reads, as GroupActivity: never its brief, checks or task. */
+export const ASLEEP_FIELDS = Object.fromEntries(['id', 'lastActivityAt', 'archived', 'agentIds', 'reviewOf', 'assignment.agentId', 'assignment.status',
+ 'assignment.progressAt', 'assignment.lost.at'].map(field => [`data.${field}`, 1]))
+
+/**
+ * The asleep groups of `reviewers` that can still change what stopIdleWorkers decides at `now` (Diego, 10-29: the Lead's poll
+ * read every one, whole, ~4 MB a poll): those active after `now - IDLE_STOP_MS` (a message, or the progress or loss of its
+ * work, idleSince) and those reviewing work still open (recheckPending). Any other is older than every stop's threshold, so
+ * it never keeps a worker. The Reviewers' index serves it (parts/indexed.ts `agents`); the rest is checked on each of their rows.
+ */
+export function asleepMatch(state: SuperTabState, tabId: string, reviewers: string[], now: number): Record<string, Prisma.InputJsonValue> {
+ const since = now - IDLE_STOP_MS
+ const open = state.conversations.filter(c => c.assignment && !FINISHED.includes(c.assignment.status)).map(c => c.id)
+ return { tabId, deleted: false, dormant: true, agentIds: { $in: reviewers }, $or: [
+ { lastActivityAt: { $gt: { $date: new Date(since).toISOString() } } },
+ { 'data.assignment.progressAt': { $gt: since } },
+ { 'data.assignment.lost.at': { $gt: since } },
+ ...open.length ? [{ 'data.reviewOf': { $in: open } }] : [],
+ ] }
+}
+
/** The groups a worker belongs to or worked in, with those asleep that the caller read for it (`asleep`). */
-const groupsOf = (state: SuperTabState, worker: Agent, asleep: readonly Conversation[]) =>
+const groupsOf = (state: SuperTabState, worker: Agent, asleep: readonly GroupActivity[]): GroupActivity[] =>
[...state.conversations, ...asleep.filter(g => !state.conversations.some(c => c.id === g.id))]
.filter(c => c.agentIds.includes(worker.id) || c.assignment?.agentId === worker.id)
/** When the worker last had anything to do: its start, its Lead's keep, its own progress, or a message in one of its groups. */
-function idleSince(state: SuperTabState, worker: Agent, groups: Conversation[]): number {
+function idleSince(state: SuperTabState, worker: Agent, groups: GroupActivity[]): number {
const seen = [time(worker.startedSession?.createdAt), worker.keptAt, ...groups.flatMap(g => [time(g.lastActivityAt),
...g.assignment?.agentId === worker.id ? [g.assignment.progressAt, g.assignment.lost?.at] : []])]
return Math.max(...seen.filter((t): t is number => t !== undefined && Number.isFinite(t)))
}
/** A Reviewer stays while work it reviews may come back to it for a re-check: the reviewed group's task is not finished. */
-function recheckPending(state: SuperTabState, reviewer: Agent, groups: Conversation[]): boolean {
+function recheckPending(state: SuperTabState, reviewer: Agent, groups: GroupActivity[]): boolean {
if (agentRole(reviewer) !== 'reviewer') return false
return groups.some(g => {
const reviewed = g.reviewOf ? state.conversations.find(c => c.id === g.reviewOf) : undefined
@@ -62,7 +88,7 @@ function reportedClean(state: SuperTabState, worker: Agent, now: number): boolea
* #1807).
*/
export function stopIdleWorkers(state: SuperTabState, projectId: string, now: number, busy: ReadonlySet<string>,
- asleep: readonly Conversation[] = []): MessageDraft[] {
+ asleep: readonly GroupActivity[] = []): MessageDraft[] {
if (state.tidy === false) return []
const project = projectsOf(state).find(p => p.id === projectId)
const leadId = project && state.conversations.find(c => c.id === project.leadConversationId)?.leadId
diff --git a/desktop/lib/super_tabs/chat_scroll.dart b/desktop/lib/super_tabs/chat_scroll.dart
index 05285b62b..6a296a192 100644
--- a/desktop/lib/super_tabs/chat_scroll.dart
+++ b/desktop/lib/super_tabs/chat_scroll.dart
@@ -29,6 +29,14 @@ class SuperTabChatScroll extends ScrollController {
oldPosition: oldPosition,
debugLabel: debugLabel,
);
+
+ /// The list's dividing row moved [by] down the rows: the view and the
+ /// place it keeps move with them, so nothing on screen moves.
+ void moveContent(double by) {
+ for (final position in positions) {
+ if (position is _AnchoredPosition) position._moveContent(by);
+ }
+ }
}
class _AnchoredPosition extends ScrollPositionWithSingleContext {
@@ -63,6 +71,15 @@ class _AnchoredPosition extends ScrollPositionWithSingleContext {
if (_settledPixels != null) _settledPixels = _settledPixels! + correction;
}
+ void _moveContent(double by) {
+ correctPixels(pixels + by);
+ if (_anchor case final anchor?) {
+ _anchor = (messageId: anchor.messageId, top: anchor.top + by);
+ }
+ if (_settledPixels != null) _settledPixels = _settledPixels! + by;
+ if (_settledMax != null) _settledMax = _settledMax! + by;
+ }
+
@override
bool applyContentDimensions(double minScrollExtent, double maxScrollExtent) {
final target = _keptPixels(maxScrollExtent);
diff --git a/desktop/lib/super_tabs/controller.dart b/desktop/lib/super_tabs/controller.dart
index 99f5de38f..4946144d2 100644
--- a/desktop/lib/super_tabs/controller.dart
+++ b/desktop/lib/super_tabs/controller.dart
@@ -212,7 +212,10 @@ class SuperTabController extends ChangeNotifier {
void _moveView({String? to, bool devices = false}) {
_selectRequest++;
devicesSelected = devices;
- if (to != null) selectedId = to;
+ if (to != null && to != selectedId) {
+ _leftAt[selectedId] = clock.now();
+ selectedId = to;
+ }
}
/// The newest change of the Swarm this tab has read, in epoch milliseconds: what tells the app and the tag list
@@ -351,6 +354,29 @@ class SuperTabController extends ChangeNotifier {
/// start of the conversation.
final _before = <String, SuperTabPage?>{};
+ /// At most this many messages are held of a conversation whose reader is at
+ /// its end: a busy one left open all day reads, sorts and folds no more
+ /// each poll. The rows cut are read again by scrolling up.
+ static const _heldMessages = 500;
+
+ /// Per conversation, the message its view lays its list out from (its
+ /// dividing row): from it on, no row is cut, since cutting them moves the
+ /// rows on screen. Rows before it may be cut.
+ final _cutBefore = <String, SuperTabMessage>{};
+
+ /// The view of [id] lays its list out from [message] on ([_cutBefore]).
+ void keepFrom(String id, SuperTabMessage message) => _cutBefore[id] = message;
+
+ /// When the reader left each conversation they opened, until it is open
+ /// again or its messages are dropped ([_dropLeft]).
+ final _leftAt = <String, DateTime>{};
+
+ /// A conversation left this long holds no messages: opening it again reads
+ /// its latest page, as the first open did. Longer than going back and forth
+ /// between a few groups; short enough that a day of browsing holds only the
+ /// ones read lately.
+ static const _keepLeft = Duration(minutes: 15);
+
/// Each conversation's reads under way, newest and older apart: an older
/// page asked for during a poll's read is a read of its own.
final _reading = <(String, bool), Future<void>>{};
@@ -797,6 +823,31 @@ class SuperTabController extends ChangeNotifier {
void _acted() => _lastActivity = clock.now();
+ /// Drops the messages held of each conversation left [_keepLeft] ago.
+ /// Kept for now: one being read, which would land on nothing and lose where
+ /// earlier history continues, and one whose unread count its held messages
+ /// give, which would read "1" until read again.
+ void _dropLeft() {
+ final now = clock.now();
+ _leftAt.removeWhere((id, left) {
+ if (id == selectedId) return true;
+ final group = snapshot?.conversation(id);
+ if (now.difference(left) < _keepLeft ||
+ _reading.keys.any((read) => read.$1 == id) ||
+ (group != null && unreadCount(group) > 1)) {
+ return false;
+ }
+ _messages.remove(id);
+ _rows.remove(id);
+ _before.remove(id);
+ _quoted.remove(id);
+ _quoting.remove(id);
+ _reactionSeq.remove(id);
+ _forgetNewer(id);
+ return true;
+ });
+ }
+
void _keepRows(String id, List rows) {
if (!_policy.drawsSaved) return;
final kept = _rows[id] ??= {};
@@ -1554,6 +1605,7 @@ class SuperTabController extends ChangeNotifier {
lastChangedAt = parts.lastChangedAt;
// Every state read, a quiet one too: a count a failed read left open is
// finished; one that is known reads nothing.
+ _dropLeft();
_readUnreadPmChats();
_partsReadAt = clock.now();
error = null;
@@ -2079,7 +2131,24 @@ class SuperTabController extends ChangeNotifier {
combined.putIfAbsent(m.id, () => m);
}
}
- final next = combined.values.toList()..sort(superTabMessageOrder);
+ final sorted = combined.values.toList()..sort(superTabMessageOrder);
+ // Past the cap the oldest rows go, where earlier history continues;
+ // never under a reader scrolled up, nor from an older page they asked
+ // for, nor from where the view lays its list out.
+ final floor = _cutBefore[id];
+ final below = floor == null
+ ? -1
+ : sorted.indexWhere((m) => superTabMessageOrder(m, floor) >= 0);
+ final cut = older || _readingBack.contains(id)
+ ? 0
+ : max(
+ 0,
+ min(
+ sorted.length - _heldMessages,
+ below < 0 ? sorted.length : below,
+ ),
+ );
+ final next = cut > 0 ? sorted.sublist(cut) : sorted;
// An adapting read that found nothing new changes nothing on screen.
final same = _policy.adapts && _sameMessages(_messages[id], next);
_messages[id] = next;
@@ -2095,6 +2164,11 @@ class SuperTabController extends ChangeNotifier {
if (older || replacing || !_before.containsKey(id)) {
_before[id] = SuperTabPage.read(answer['next']);
}
+ if (cut > 0) {
+ _before[id] = SuperTabPage.of(next.first);
+ final kept = {for (final m in next) m.id};
+ _rows[id]?.removeWhere((messageId, _) => !kept.contains(messageId));
+ }
// Something new in the conversation on screen keeps the full poll.
if (arrived && id == selectedId) _acted();
if (!same) {
diff --git a/desktop/lib/super_tabs/view.dart b/desktop/lib/super_tabs/view.dart
index c1a43bf7e..3cd191686 100644
--- a/desktop/lib/super_tabs/view.dart
+++ b/desktop/lib/super_tabs/view.dart
@@ -2,6 +2,8 @@ import 'dart:async';
import 'package:flutter/foundation.dart' show kIsWeb;
import 'package:flutter/material.dart';
+import 'package:flutter/rendering.dart'
+ show RenderSliverMultiBoxAdaptor, SliverMultiBoxAdaptorParentData;
import 'package:flutter/services.dart';
import '../shared/theme/app_icons.dart';
@@ -254,6 +256,9 @@ class _SuperTabViewState extends State<SuperTabView>
final grew = latest > previous && !_loadingOlder.contains(id);
_latest[id] = latest;
if (grew && !atEnd) _newAfter.putIfAbsent(id, () => previous);
+ if (atEnd && _selected == id && scroll.hasClients && !_jumpingToEnd) {
+ _moveSplitToView(id, scroll);
+ }
final draft = c.accessRevoked ? '' : c.draft(id);
if (_selected != id) {
_scrollRequest++;
@@ -2057,9 +2062,70 @@ class _SuperTabViewState extends State<SuperTabView>
);
if (at >= 0) return at;
_split[id] = entries.first.message;
+ c.keepFrom(id, entries.first.message);
return 0;
}
+ /// Rows below the dividing row before a reader at the end has it moved to
+ /// the rows on screen ([_moveSplitToView]).
+ static const _rowsBelowSplit = 200;
+
+ /// For a reader at the end of [id]: moves the dividing row down to the
+ /// first row of the group at the top edge of the view, its place
+ /// unchanged, so the controller may cut the rows above (it never cuts below
+ /// it). From a group's first row the groups below stay as they were; only
+ /// rows above the view can regroup.
+ void _moveSplitToView(String id, SuperTabChatScroll scroll) {
+ final split = _split[id];
+ if (split == null) return;
+ final messages = _shownMessages(id);
+ final at = messages.indexWhere((m) => superTabMessageOrder(m, split) >= 0);
+ final top = _rowAtTop();
+ if (at < 0 || messages.length - at <= _rowsBelowSplit || top == null) {
+ return;
+ }
+ final joined = _joined(messages, split);
+ var index = messages.indexWhere((m) => m.id == top);
+ while (index > at && joined.contains(messages[index].id)) {
+ index--;
+ }
+ if (index <= at) return;
+ final first = messages[index];
+ final offset = _offsetInList(_built[first.id]);
+ if (offset == null) return;
+ // Its row starts the newer rows now: the rows are measured from it.
+ scroll.moveContent(-offset);
+ _split[id] = first;
+ c.keepFrom(id, first);
+ }
+
+ /// The built row at the top edge of the view: the one it cuts through,
+ /// else the highest one below it.
+ String? _rowAtTop() {
+ ({String messageId, double top})? at;
+ for (final MapEntry(key: messageId, value: row) in _built.entries) {
+ final top = row.mounted ? _topInList(row) : null;
+ if (top == null) continue;
+ final better =
+ at == null ||
+ (top <= 0 ? at.top > 0 || top > at.top : at.top > 0 && top < at.top);
+ if (better) at = (messageId: messageId, top: top);
+ }
+ return at?.messageId;
+ }
+
+ /// Where [row] is laid out in the newer rows' list, from its start; null
+ /// when it is not laid out there.
+ static double? _offsetInList(BuildContext? row) {
+ if (row == null || !row.mounted) return null;
+ RenderObject? child = row.findRenderObject();
+ while (child != null && child.parent is! RenderSliverMultiBoxAdaptor) {
+ child = child.parent;
+ }
+ final data = child?.parentData;
+ return data is SliverMultiBoxAdaptorParentData ? data.layoutOffset : null;
+ }
+
/// The rows that join the one above them, without their name: runs of
/// one author's short updates, three to a group. Groups are counted from
/// the list's dividing row both ways, so a page read in above regroups
diff --git a/desktop/test/super_tab_chat_scroll_test.dart b/desktop/test/super_tab_chat_scroll_test.dart
index f8d3b2878..838c3903f 100644
--- a/desktop/test/super_tab_chat_scroll_test.dart
+++ b/desktop/test/super_tab_chat_scroll_test.dart
@@ -3,6 +3,7 @@ import 'package:flutter/services.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:harness/shared/theme/app_theme.dart' as grid;
import 'package:harness/super_tabs/controller.dart';
+import 'package:harness/super_tabs/reading.dart';
import 'package:harness/super_tabs/view.dart';
import 'support/super_tab_fixture.dart';
@@ -13,9 +14,10 @@ import 'support/super_tab_fixture.dart';
void main() {
Future<SuperTabController> show(
WidgetTester tester,
- ConversationServer server,
- ) async {
- final c = server.controller();
+ ConversationServer server, {
+ SuperTabReading reading = const SuperTabReading(),
+ }) async {
+ final c = server.controller(reading: reading);
await c.refresh();
await tester.pumpWidget(
MaterialApp(
@@ -333,4 +335,124 @@ void main() {
expect(tester.takeException(), isNull);
await close(tester, c);
});
+
+ double bottomOf(WidgetTester tester, String id) =>
+ tester.getBottomLeft(row(id)).dy;
+
+ /// Each row in view, where it is drawn: a row regrouped (its name shown or
+ /// hidden) changes height, one moved changes place.
+ Map<String, Rect> onScreen(WidgetTester tester) {
+ final list = tester.getRect(
+ find.byKey(const PageStorageKey('super-messages-tab-lead')),
+ );
+ final rows = find.byWidgetPredicate(
+ (w) => switch (w.key) {
+ ValueKey<String>(:final value) => value.startsWith(
+ 'super-message-message-',
+ ),
+ _ => false,
+ },
+ );
+ return {
+ for (final element in rows.evaluate())
+ if (tester.getRect(find.byWidget(element.widget)) case final rect
+ when rect.overlaps(list))
+ (element.widget.key! as ValueKey<String>).value: rect,
+ };
+ }
+
+ /// Polls [c] while [count] agent updates arrive, 50 a poll (once with
+ /// none), checking [each] at every frame, or once settled.
+ Future<void> poll(
+ WidgetTester tester,
+ ConversationServer server,
+ SuperTabController c,
+ int count,
+ void Function() each, {
+ bool everyFrame = true,
+ }) async {
+ for (var polls = count == 0 ? 1 : count ~/ 50; polls > 0; polls--) {
+ if (count > 0) agentUpdates(server, 50);
+ await c.refresh();
+ for (var frame = 0; everyFrame && frame < 3; frame++) {
+ await tester.pump();
+ each();
+ }
+ await tester.pumpAndSettle();
+ each();
+ }
+ }
+
+ testWidgets('a reader at the end stays at the end as polls cut the oldest '
+ 'rows', (tester) async {
+ final server = ConversationServer();
+ agentUpdates(server, 30);
+ final c = await show(tester, server);
+ final end = bottomOf(tester, 'message-30');
+
+ // 1,000 updates, 50 a poll: past 500 each poll cuts the oldest. New rows
+ // reach the end a frame after they arrive, cut or not.
+ await poll(tester, server, c, 1000, everyFrame: false, () {
+ final newest = c.messages('lead').last.id;
+ expect(bottomOf(tester, newest), moreOrLessEquals(end, epsilon: .5));
+ });
+
+ expect(position(tester).extentAfter, lessThan(1));
+ expect(c.messages('lead'), hasLength(500));
+ expect(c.hasOlder('lead'), isTrue);
+ expect(tester.takeException(), isNull);
+ await close(tester, c);
+ });
+
+ testWidgets('a reader scrolled up keeps every row, in place, as polls pass '
+ 'the cap', (tester) async {
+ final server = ConversationServer();
+ agentUpdates(server, 30);
+ final c = await show(tester, server);
+ position(tester).jumpTo(position(tester).maxScrollExtent / 2);
+ await tester.pumpAndSettle();
+ final shown = onScreen(tester);
+ expect(shown, isNotEmpty);
+
+ await poll(tester, server, c, 1000, () {
+ expect(onScreen(tester), shown);
+ });
+
+ expect(c.messages('lead'), hasLength(1030));
+ expect(tester.takeException(), isNull);
+ await close(tester, c);
+ });
+
+ testWidgets('back at the end after reading up, the cut with nothing new '
+ 'moves and regroups no row', (tester) async {
+ final server = ConversationServer();
+ agentUpdates(server, 30);
+ // Desktop's: the messages read lands beside the state read, before the
+ // view hears of either.
+ final c = await show(tester, server, reading: SuperTabReading.desktop);
+ position(tester).jumpTo(position(tester).maxScrollExtent / 2);
+ await tester.pumpAndSettle();
+ await poll(tester, server, c, 600, () {});
+ expect(c.messages('lead'), hasLength(630));
+ // Back to the end: a lazy list learns its full extent as it goes.
+ for (var i = 0; i < 12 && position(tester).extentAfter > 0; i++) {
+ position(tester).jumpTo(position(tester).maxScrollExtent);
+ await tester.pumpAndSettle();
+ }
+ final shown = onScreen(tester);
+ expect(shown.keys, contains('super-message-message-630'));
+
+ // The first poll moves the list's dividing row to the rows on screen;
+ // the cut above it waits for the next.
+ for (var i = 0; i < 2; i++) {
+ await poll(tester, server, c, 0, () {
+ expect(onScreen(tester), shown);
+ });
+ }
+
+ expect(c.messages('lead'), hasLength(500));
+ expect(position(tester).extentAfter, lessThan(1));
+ expect(tester.takeException(), isNull);
+ await close(tester, c);
+ });
}
diff --git a/desktop/test/super_tab_message_cap_test.dart b/desktop/test/super_tab_message_cap_test.dart
new file mode 100644
index 000000000..44132f131
--- /dev/null
+++ b/desktop/test/super_tab_message_cap_test.dart
@@ -0,0 +1,243 @@
+import 'dart:async';
+
+import 'package:clock/clock.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:harness/super_tabs/controller.dart';
+
+import 'support/super_tab_fixture.dart';
+
+/// A conversation left open all day holds a capped tail of its messages, and
+/// one left for a while holds none: neither grows with the day.
+void main() {
+ /// Polls [c] while [count] messages arrive in [server]'s lead chat, a
+ /// batch of 50 (under a page) between polls.
+ Future<void> pollThrough(
+ ConversationServer server,
+ SuperTabController c,
+ int count,
+ ) async {
+ for (var sent = 0; sent < count; sent += 50) {
+ for (var i = 0; i < 50; i++) {
+ server.append('message');
+ }
+ await c.loadMessages('lead');
+ }
+ }
+
+ test('2,000 messages through polls hold the latest 500, with older pages '
+ 'still offered', () async {
+ final server = ConversationServer();
+ final c = server.controller();
+ await c.refresh();
+
+ await pollThrough(server, c, 2000);
+
+ final held = c.messages('lead').map((m) => m.sequence).toList();
+ expect(held, List.generate(500, (i) => i + 1501));
+ expect(c.hasOlder('lead'), isTrue);
+ });
+
+ test('scrolling up after the cut reads the rows cut, oldest kept first, and '
+ 'keeps them', () async {
+ final server = ConversationServer();
+ final c = server.controller();
+ await c.refresh();
+ await pollThrough(server, c, 2000);
+ final reads = <String>[];
+ server.onRead = reads.add;
+
+ // The Earlier row, pressed with the list at its end.
+ await c.loadMessages('lead', older: true);
+
+ expect(
+ reads.single,
+ endsWith('messages?before=1501&beforeId=message-1501'),
+ );
+ final held = c.messages('lead').map((m) => m.sequence).toList();
+ expect(held, List.generate(600, (i) => i + 1401));
+ expect(c.hasOlder('lead'), isTrue);
+ });
+
+ test('a reader scrolled up keeps every row through the polls, and back at '
+ 'the end the next poll cuts to the latest 500', () async {
+ final server = ConversationServer();
+ final c = server.controller();
+ await c.refresh();
+ c.readingBack('lead', true);
+
+ await pollThrough(server, c, 2000);
+
+ expect(
+ c.messages('lead').map((m) => m.sequence),
+ List.generate(2000, (i) => i + 1),
+ );
+ expect(c.hasOlder('lead'), isFalse);
+
+ c.readingBack('lead', false);
+ await c.loadMessages('lead');
+
+ expect(
+ c.messages('lead').map((m) => m.sequence),
+ List.generate(500, (i) => i + 1501),
+ );
+ expect(c.hasOlder('lead'), isTrue);
+ });
+
+ test('rows from where the view lays its list out are never cut', () async {
+ final server = ConversationServer();
+ final c = server.controller();
+ await c.refresh();
+ await pollThrough(server, c, 400);
+ c.keepFrom('lead', c.messages('lead').firstWhere((m) => m.sequence == 301));
+
+ await pollThrough(server, c, 600);
+
+ // Only the 300 before it go, though 1,000 are held.
+ final held = c.messages('lead').map((m) => m.sequence).toList();
+ expect(held, List.generate(700, (i) => i + 301));
+ expect(c.hasOlder('lead'), isTrue);
+
+ // Moved on to the rows on screen, the next poll cuts to the cap.
+ c.keepFrom('lead', c.messages('lead').firstWhere((m) => m.sequence == 990));
+ await pollThrough(server, c, 50);
+ expect(
+ c.messages('lead').map((m) => m.sequence),
+ List.generate(500, (i) => i + 551),
+ );
+ });
+
+ group('a conversation left', () {
+ late DateTime now;
+ late ConversationServer server;
+ setUp(() {
+ now = DateTime.utc(2026, 10, 29, 9);
+ server = ConversationServer();
+ for (var i = 1; i <= 150; i++) {
+ server.append('lead $i');
+ }
+ server.append('result', conversationId: 'offline');
+ });
+
+ Future<T> at<T>(Future<T> Function() body) =>
+ withClock(Clock(() => now), body);
+
+ test(
+ '15 minutes is dropped at the next read and reads its latest page '
+ 'when opened again',
+ () => at(() async {
+ final c = server.controller();
+ await c.refresh();
+ await c.loadMessages('lead');
+ await c.loadMessages('lead', older: true);
+ expect(c.messages('lead'), hasLength(150));
+ // Read to its end, as the view marks it there.
+ c.markRead('lead');
+ await c.select('offline');
+
+ now = now.add(const Duration(minutes: 14));
+ await c.refresh();
+ expect(c.messages('lead'), hasLength(150), reason: 'not yet 15');
+
+ now = now.add(const Duration(minutes: 1));
+ await c.refresh();
+ expect(c.messages('lead'), isEmpty);
+ expect(c.hasOlder('lead'), isFalse);
+
+ final reads = <String>[];
+ server.onRead = reads.add;
+ await c.select('lead');
+ await c.loadMessages('lead');
+
+ expect(reads.first, endsWith('/conversations/lead/messages'));
+ expect(
+ c.messages('lead').map((m) => m.sequence),
+ List.generate(100, (i) => i + 51),
+ );
+ expect(c.hasOlder('lead'), isTrue);
+ }),
+ );
+
+ test(
+ 'with unread messages held keeps them, so its count stays',
+ () => at(() async {
+ // The two newest are the Lead's, past the reader's read mark.
+ for (final m in server.messages.where((m) => m['sequence'] >= 149)) {
+ m['author'] = {'kind': 'agent', 'id': 'lead-agent', 'name': 'Lead'};
+ }
+ final c = server.controller();
+ await c.refresh();
+ await c.loadMessages('lead');
+ c.markRead('lead', through: 148);
+ await c.select('offline');
+
+ now = now.add(const Duration(hours: 1));
+ await c.refresh();
+
+ expect(c.unreadCount(c.snapshot!.conversation('lead')!), 2);
+ expect(c.messages('lead'), hasLength(100));
+ }),
+ );
+
+ test(
+ 'being read when its time is up keeps where earlier history continues',
+ () => at(() async {
+ final c = server.controller();
+ await c.refresh();
+ await c.loadMessages('lead');
+ await c.loadMessages('lead', older: true);
+ c.markRead('lead');
+ await c.select('offline');
+ server
+ ..append('lead 151')
+ ..append('lead 152');
+ final hold = server.holdReads = Completer<void>();
+ final reading = c.loadMessages('lead');
+ server.holdReads = null;
+
+ now = now.add(const Duration(minutes: 16));
+ await c.refresh();
+ hold.complete();
+ await reading;
+
+ expect(c.messages('lead'), hasLength(152));
+ }),
+ );
+
+ test(
+ 'open is never dropped, however long it is open',
+ () => at(() async {
+ final c = server.controller();
+ await c.refresh();
+ await c.loadMessages('lead');
+ await c.loadMessages('lead', older: true);
+ c.markRead('lead');
+ await c.select('offline');
+ await c.select('lead');
+
+ now = now.add(const Duration(hours: 3));
+ await c.refresh();
+
+ // All 150 still: dropped, the read would bring the latest 100.
+ expect(c.messages('lead'), hasLength(150));
+ }),
+ );
+
+ test(
+ 'opened again within 15 minutes starts its time over',
+ () => at(() async {
+ final c = server.controller();
+ await c.refresh();
+ await c.loadMessages('lead');
+ await c.select('offline');
+ now = now.add(const Duration(minutes: 10));
+ await c.select('lead');
+ await c.select('offline');
+
+ now = now.add(const Duration(minutes: 10));
+ await c.refresh();
+
+ expect(c.messages('lead'), hasLength(100));
+ }),
+ );
+ });
+}
diff --git a/desktop/test/support/super_tab_fixture.dart b/desktop/test/support/super_tab_fixture.dart
index c86a37d32..38fce8a60 100644
--- a/desktop/test/support/super_tab_fixture.dart
+++ b/desktop/test/support/super_tab_fixture.dart
@@ -6,6 +6,7 @@ import 'dart:typed_data';
import 'package:dio/dio.dart' show CancelToken;
import 'package:harness/super_tabs/attachment_upload.dart';
import 'package:harness/super_tabs/controller.dart';
+import 'package:harness/super_tabs/reading.dart';
/// A signed link as the backend's private avatar bucket hands it out (avatarLinks.ts).
String superTabSignedAvatar(String id) =>
@@ -1117,8 +1118,10 @@ class ConversationServer {
Map<String, dynamic>? initial,
String account = 'owner',
SuperTabUpload? upload,
+ SuperTabReading reading = const SuperTabReading(),
}) => SuperTabController(
avatarUpload: uploadAvatar,
+ reading: reading,
tabId: 'tab',
accountId: account,
request: (method, path, {body}) => runZoned(