Multiplayer agent harness for work.
Skip legacy tape payloads too large to parse and isolate failing sessions during step preparation (#2299)
Regan Bell605351b77c29
diff --git a/src/sessions/legacy-tape-steps.ts b/src/sessions/legacy-tape-steps.tsindex f5a3d93dd..3bafde009 100644--- a/src/sessions/legacy-tape-steps.ts+++ b/src/sessions/legacy-tape-steps.ts@@ -1,89 +1,20 @@-export const LEGACY_TAPE_STEPS_SQL = [- `CREATE TABLE IF NOT EXISTS legacy_tape_steps(- session_id TEXT NOT NULL,- seq INT NOT NULL,- step JSONB NOT NULL,- wire JSONB,- PRIMARY KEY (session_id, seq)- )`,- `CREATE TABLE IF NOT EXISTS legacy_turn_runs(- session_id TEXT NOT NULL,- seq INT NOT NULL,- run_id TEXT NOT NULL,- inferred BOOLEAN NOT NULL,- PRIMARY KEY (session_id, seq)- )`,- `CREATE TABLE IF NOT EXISTS legacy_tape_steps_progress(- singleton BOOLEAN PRIMARY KEY DEFAULT true CHECK (singleton),- started_at BIGINT,- last_session TEXT,- done BOOLEAN NOT NULL DEFAULT false,- invalid_messages BIGINT NOT NULL DEFAULT 0,- keyed_blocks BIGINT NOT NULL DEFAULT 0,- copied_steps BIGINT NOT NULL DEFAULT 0,- copied_turns BIGINT NOT NULL DEFAULT 0- )`,- `INSERT INTO legacy_tape_steps_progress DEFAULT VALUES ON CONFLICT (singleton) DO NOTHING`,- `CREATE TABLE IF NOT EXISTS legacy_tape_writes(at BIGINT NOT NULL)`,- `CREATE OR REPLACE FUNCTION legacy_tape_written() RETURNS trigger LANGUAGE plpgsql AS $written$-BEGIN- INSERT INTO legacy_tape_writes VALUES ((EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint);- RETURN NULL;-END $written$`,- `CREATE OR REPLACE PROCEDURE prepare_legacy_tape_steps(batch INT DEFAULT 50)-LANGUAGE plpgsql AS $prepare$-DECLARE- ids TEXT[];- frozen_at BIGINT;- n_invalid BIGINT;- n_keyed BIGINT;- n_steps BIGINT;- n_turns BIGINT;-BEGIN- IF NOT pg_try_advisory_lock(hashtext('prepare_legacy_tape_steps')) THEN- RAISE EXCEPTION 'prepare_legacy_tape_steps() is already running in another session';- END IF;- IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname = 'legacy_tape_frozen' AND tgrelid = 'session_tape'::regclass) THEN- IF EXISTS (SELECT 1 FROM legacy_tape_steps_progress WHERE started_at IS NOT NULL OR last_session IS NOT NULL OR done) THEN- RAISE EXCEPTION 'legacy_tape_frozen is gone but a preparation already started: reset legacy_tape_steps_progress, legacy_tape_steps, legacy_turn_runs and legacy_tape_writes before starting again';- END IF;- PERFORM set_config('lock_timeout', '5s', true);- CREATE TRIGGER legacy_tape_frozen AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON session_tape- FOR EACH STATEMENT EXECUTE FUNCTION legacy_tape_written();- ALTER TABLE session_tape ENABLE ALWAYS TRIGGER legacy_tape_frozen;- UPDATE legacy_tape_steps_progress SET started_at = (EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint;- COMMIT;- END IF;- SELECT started_at INTO frozen_at FROM legacy_tape_steps_progress;- IF to_regclass('legacy_prep_runs') IS NULL THEN- CREATE TABLE legacy_prep_runs(- id TEXT, session_id TEXT, turn_user_seq BIGINT, started_at BIGINT, finished_at BIGINT, prev_max_finished BIGINT- );- IF to_regclass('runs') IS NOT NULL THEN- INSERT INTO legacy_prep_runs- SELECT r.id, s.id, r.turn_user_seq, r.started_at, r.finished_at,- max(COALESCE(r.finished_at, 9223372036854775807)) OVER (- PARTITION BY s.id ORDER BY r.started_at, r.id ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING)- FROM runs r JOIN sessions s ON s.thread_ref = r.session_id- WHERE r.started_at IS NOT NULL;- END IF;- CREATE INDEX legacy_prep_runs_by_seq ON legacy_prep_runs(session_id, turn_user_seq);- CREATE INDEX legacy_prep_runs_by_start ON legacy_prep_runs(session_id, started_at DESC, id DESC);- COMMIT;- END IF;- LOOP- SELECT array_agg(id ORDER BY id) INTO ids- FROM (SELECT id FROM sessions- WHERE id > COALESCE((SELECT last_session FROM legacy_tape_steps_progress), '')- ORDER BY id LIMIT batch) next_sessions;- EXIT WHEN ids IS NULL;- PERFORM set_config('enable_nestloop', 'off', true);+const MAX_PAYLOAD_BYTES = 16 * 1024 * 1024;++function batchSql(sizeGuard: boolean): string {+ const tapeValid = sizeGuard+ ? `CASE WHEN octet_length(t.payload) > ${MAX_PAYLOAD_BYTES} THEN false+ ELSE pg_input_is_valid(t.payload, 'jsonb') AND left(btrim(t.payload), 1) = '{' END`+ : `pg_input_is_valid(t.payload, 'jsonb') AND left(btrim(t.payload), 1) = '{'`;+ const entryJson = sizeGuard+ ? `WHEN octet_length(e.payload) > ${MAX_PAYLOAD_BYTES} THEN NULL WHEN pg_input_is_valid(e.payload, 'jsonb') THEN e.payload::jsonb`+ : `WHEN pg_input_is_valid(e.payload, 'jsonb') THEN e.payload::jsonb`;+ return ` PERFORM set_config('enable_nestloop', 'off', true);CREATE TEMP TABLE legacy_batch_blocks ON COMMIT DROP ASWITH messages AS MATERIALIZED (SELECT t.session_id, t.seq, t.created_at, v.valid, m.*FROM session_tape tCROSS JOIN LATERAL (- SELECT pg_input_is_valid(t.payload, 'jsonb') AND left(btrim(t.payload), 1) = '{' AS valid OFFSET 0+ SELECT ${tapeValid} AS valid OFFSET 0) vCROSS JOIN LATERAL (SELECT r.* FROM json_to_record(CASE WHEN v.valid THEN t.payload::json END)@@ -117,7 +48,7 @@ BEGINWHEN e.type = 'thinking' THEN 'sig:' || (p.j ->> 'thinkingSignature')ELSE 'text:' || (p.j ->> 'text') END AS keyFROM session_entries e- CROSS JOIN LATERAL (SELECT CASE WHEN pg_input_is_valid(e.payload, 'jsonb') THEN e.payload::jsonb END AS j) p+ CROSS JOIN LATERAL (SELECT CASE ${entryJson} END AS j) pWHERE e.session_id = ANY(ids)AND e.type IN ('tool_call', 'thinking', 'text', 'assistant') AND e.created_at < frozen_atAND jsonb_typeof(p.j) = 'object' AND NOT p.j ? 'step'@@ -151,7 +82,7 @@ BEGININSERT INTO legacy_turn_runs(session_id, seq, run_id, inferred)SELECT e.session_id, e.seq, COALESCE(by_seq.id, by_window.id), by_seq.id IS NULLFROM session_entries e- CROSS JOIN LATERAL (SELECT CASE WHEN pg_input_is_valid(e.payload, 'jsonb') THEN e.payload::jsonb END AS j) p+ CROSS JOIN LATERAL (SELECT CASE ${entryJson} END AS j) pLEFT JOIN LATERAL (SELECT CASE WHEN count(*) = 1 THEN min(r.id) END AS id FROM legacy_prep_runs rWHERE r.session_id = e.session_id AND r.turn_user_seq = e.seq@@ -171,7 +102,59 @@ BEGINAND COALESCE(by_seq.id, by_window.id) IS NOT NULLON CONFLICT (session_id, seq) DO NOTHING;GET DIAGNOSTICS n_turns = ROW_COUNT;- UPDATE legacy_tape_steps_progress+`;+}++const PREPARE_HEAD = `CREATE OR REPLACE PROCEDURE prepare_legacy_tape_steps(batch INT DEFAULT 50)+LANGUAGE plpgsql AS $prepare$+DECLARE+ ids TEXT[];+ frozen_at BIGINT;+ n_invalid BIGINT;+ n_keyed BIGINT;+ n_steps BIGINT;+ n_turns BIGINT;+BEGIN+ IF NOT pg_try_advisory_lock(hashtext('prepare_legacy_tape_steps')) THEN+ RAISE EXCEPTION 'prepare_legacy_tape_steps() is already running in another session';+ END IF;+ IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname = 'legacy_tape_frozen' AND tgrelid = 'session_tape'::regclass) THEN+ IF EXISTS (SELECT 1 FROM legacy_tape_steps_progress WHERE started_at IS NOT NULL OR last_session IS NOT NULL OR done) THEN+ RAISE EXCEPTION 'legacy_tape_frozen is gone but a preparation already started: reset legacy_tape_steps_progress, legacy_tape_steps, legacy_turn_runs and legacy_tape_writes before starting again';+ END IF;+ PERFORM set_config('lock_timeout', '5s', true);+ CREATE TRIGGER legacy_tape_frozen AFTER INSERT OR UPDATE OR DELETE OR TRUNCATE ON session_tape+ FOR EACH STATEMENT EXECUTE FUNCTION legacy_tape_written();+ ALTER TABLE session_tape ENABLE ALWAYS TRIGGER legacy_tape_frozen;+ UPDATE legacy_tape_steps_progress SET started_at = (EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint;+ COMMIT;+ END IF;+ SELECT started_at INTO frozen_at FROM legacy_tape_steps_progress;+ IF to_regclass('legacy_prep_runs') IS NULL THEN+ CREATE TABLE legacy_prep_runs(+ id TEXT, session_id TEXT, turn_user_seq BIGINT, started_at BIGINT, finished_at BIGINT, prev_max_finished BIGINT+ );+ IF to_regclass('runs') IS NOT NULL THEN+ INSERT INTO legacy_prep_runs+ SELECT r.id, s.id, r.turn_user_seq, r.started_at, r.finished_at,+ max(COALESCE(r.finished_at, 9223372036854775807)) OVER (+ PARTITION BY s.id ORDER BY r.started_at, r.id ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING)+ FROM runs r JOIN sessions s ON s.thread_ref = r.session_id+ WHERE r.started_at IS NOT NULL;+ END IF;+ CREATE INDEX legacy_prep_runs_by_seq ON legacy_prep_runs(session_id, turn_user_seq);+ CREATE INDEX legacy_prep_runs_by_start ON legacy_prep_runs(session_id, started_at DESC, id DESC);+ COMMIT;+ END IF;+ LOOP+ SELECT array_agg(id ORDER BY id) INTO ids+ FROM (SELECT id FROM sessions+ WHERE id > COALESCE((SELECT last_session FROM legacy_tape_steps_progress), '')+ ORDER BY id LIMIT batch) next_sessions;+ EXIT WHEN ids IS NULL;+`;++const PREPARE_TAIL = ` UPDATE legacy_tape_steps_progressSET last_session = ids[array_length(ids, 1)],invalid_messages = invalid_messages + n_invalid, keyed_blocks = keyed_blocks + n_keyed,copied_steps = copied_steps + n_steps, copied_turns = copied_turns + n_turns;@@ -181,5 +164,111 @@ BEGINUPDATE legacy_tape_steps_progress SET done = true;COMMIT;PERFORM pg_advisory_unlock(hashtext('prepare_legacy_tape_steps'));-END $prepare$`,+END $prepare$`;++const SYSTEM_ERROR_CLASSES = "'40', '42', '53', '55', '57', '58'";++const MAX_SKIPPED_SESSIONS = 100;++const SKIPPING_LOOP = ` BEGIN+ SELECT * INTO n_invalid, n_keyed, n_steps, n_turns FROM legacy_tape_steps_batch(ids, frozen_at);+ n_skipped := 0;+ EXCEPTION WHEN OTHERS THEN+ IF left(SQLSTATE, 2) IN (${SYSTEM_ERROR_CLASSES}) THEN+ RAISE;+ END IF;+ n_invalid := 0; n_keyed := 0; n_steps := 0; n_turns := 0; n_skipped := 0;+ FOREACH one IN ARRAY ids LOOP+ BEGIN+ SELECT * INTO counts FROM legacy_tape_steps_batch(ARRAY[one], frozen_at);+ n_invalid := n_invalid + counts.n_invalid; n_keyed := n_keyed + counts.n_keyed;+ n_steps := n_steps + counts.n_steps; n_turns := n_turns + counts.n_turns;+ EXCEPTION WHEN OTHERS THEN+ IF left(SQLSTATE, 2) IN (${SYSTEM_ERROR_CLASSES}) THEN+ RAISE;+ END IF;+ last_error := SQLSTATE || ': ' || SQLERRM;+ INSERT INTO legacy_tape_skipped VALUES (one, last_error, (EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint)+ ON CONFLICT (session_id) DO UPDATE SET error = EXCLUDED.error, at = EXCLUDED.at;+ n_skipped := n_skipped + 1;+ END;+ END LOOP;+ IF n_skipped > 0 AND (SELECT skipped_sessions FROM legacy_tape_steps_progress) + n_skipped > ${MAX_SKIPPED_SESSIONS} THEN+ RAISE EXCEPTION 'more than ${MAX_SKIPPED_SESSIONS} sessions could not be prepared, so the failure is unlikely to be in the data: %', last_error;+ END IF;+ IF array_length(ids, 1) > 1 AND n_skipped = array_length(ids, 1) THEN+ RAISE EXCEPTION 'every session in the batch after % failed, so the failure is not in the data: %',+ COALESCE((SELECT last_session FROM legacy_tape_steps_progress), '(start)'), last_error;+ END IF;+ END;+`;++function replaceOnce(text: string, from: string, to: string): string {+ if (text.split(from).length !== 2) throw new Error(`legacy tape steps: expected one ${JSON.stringify(from)}`);+ return text.replace(from, to);+}++const skippingProcedure = () =>+ replaceOnce(+ replaceOnce(+ replaceOnce(+ PREPARE_HEAD + SKIPPING_LOOP + PREPARE_TAIL,+ " n_turns BIGINT;\n",+ " n_turns BIGINT;\n n_skipped BIGINT;\n one TEXT;\n counts RECORD;\n last_error TEXT;\n",+ ),+ "copied_turns = copied_turns + n_turns;",+ "copied_turns = copied_turns + n_turns,\n skipped_sessions = skipped_sessions + n_skipped;",+ ),+ "legacy_turn_runs and legacy_tape_writes before starting again",+ "legacy_turn_runs, legacy_tape_writes and legacy_tape_skipped before starting again",+ );++export const LEGACY_TAPE_STEPS_SQL = [+ `CREATE TABLE IF NOT EXISTS legacy_tape_steps(+ session_id TEXT NOT NULL,+ seq INT NOT NULL,+ step JSONB NOT NULL,+ wire JSONB,+ PRIMARY KEY (session_id, seq)+ )`,+ `CREATE TABLE IF NOT EXISTS legacy_turn_runs(+ session_id TEXT NOT NULL,+ seq INT NOT NULL,+ run_id TEXT NOT NULL,+ inferred BOOLEAN NOT NULL,+ PRIMARY KEY (session_id, seq)+ )`,+ `CREATE TABLE IF NOT EXISTS legacy_tape_steps_progress(+ singleton BOOLEAN PRIMARY KEY DEFAULT true CHECK (singleton),+ started_at BIGINT,+ last_session TEXT,+ done BOOLEAN NOT NULL DEFAULT false,+ invalid_messages BIGINT NOT NULL DEFAULT 0,+ keyed_blocks BIGINT NOT NULL DEFAULT 0,+ copied_steps BIGINT NOT NULL DEFAULT 0,+ copied_turns BIGINT NOT NULL DEFAULT 0+ )`,+ `INSERT INTO legacy_tape_steps_progress DEFAULT VALUES ON CONFLICT (singleton) DO NOTHING`,+ `CREATE TABLE IF NOT EXISTS legacy_tape_writes(at BIGINT NOT NULL)`,+ `CREATE OR REPLACE FUNCTION legacy_tape_written() RETURNS trigger LANGUAGE plpgsql AS $written$+BEGIN+ INSERT INTO legacy_tape_writes VALUES ((EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint);+ RETURN NULL;+END $written$`,+ PREPARE_HEAD + batchSql(false) + PREPARE_TAIL,+];++export const LEGACY_TAPE_STEPS_SIZE_GUARD_SQL = [+ `ALTER TABLE legacy_tape_steps_progress ADD COLUMN IF NOT EXISTS skipped_sessions BIGINT NOT NULL DEFAULT 0`,+ `CREATE TABLE IF NOT EXISTS legacy_tape_skipped(session_id TEXT PRIMARY KEY, error TEXT NOT NULL, at BIGINT NOT NULL)`,+ `CREATE OR REPLACE FUNCTION legacy_tape_steps_batch(+ ids TEXT[], frozen_at BIGINT, OUT n_invalid BIGINT, OUT n_keyed BIGINT, OUT n_steps BIGINT, OUT n_turns BIGINT+) LANGUAGE plpgsql AS $batch$+BEGIN+ IF to_regclass('pg_temp.legacy_batch_blocks') IS NOT NULL THEN+ DROP TABLE pg_temp.legacy_batch_blocks;+ END IF;+${batchSql(true)} DROP TABLE pg_temp.legacy_batch_blocks;+END $batch$`,+ skippingProcedure(),];diff --git a/src/sessions/postgres-session-store.ts b/src/sessions/postgres-session-store.tsindex 43c9105ee..765ece405 100644--- a/src/sessions/postgres-session-store.ts+++ b/src/sessions/postgres-session-store.ts@@ -39,7 +39,7 @@ import type {StoreOptions,} from "./session-store.ts";import { tsPrefixQuery } from "./entry-search.ts";-import { LEGACY_TAPE_STEPS_SQL } from "./legacy-tape-steps.ts";+import { LEGACY_TAPE_STEPS_SIZE_GUARD_SQL, LEGACY_TAPE_STEPS_SQL } from "./legacy-tape-steps.ts";import {cronIdOf,legacyOriginPattern,@@ -808,6 +808,10 @@ export function createPostgresSessionStore(connectionString: string, opts: Storeid: "sessions/store/0029-legacy-tape-steps",statements: LEGACY_TAPE_STEPS_SQL,},+ {+ id: "sessions/store/0030-legacy-tape-steps-size-guard",+ statements: LEGACY_TAPE_STEPS_SIZE_GUARD_SQL,+ },],[{diff --git a/test/postgres-legacy-tape-steps.test.ts b/test/postgres-legacy-tape-steps.test.tsindex 0e5282de2..2a03e0789 100644--- a/test/postgres-legacy-tape-steps.test.ts+++ b/test/postgres-legacy-tape-steps.test.ts@@ -203,3 +203,130 @@ test(});},);++test("pg legacy tape step preparation skips oversized payloads and sessions it cannot parse", { skip }, async () => {+ await withSchema(async (raw) => {+ await seedSession(+ raw,+ "huge",+ [message([{ type: "toolCall", id: "call-huge", name: "execute", arguments: {} }])],+ [["tool_call", { tool: "execute", callId: "call-huge" }]],+ );+ await raw.query(+ `INSERT INTO session_tape(session_id, seq, kind, harness, payload, scope_label, created_at)+ VALUES ('huge', 1, 'message', 'pi', '{"role":"assistant","content":[{"type":"text","text":"' || repeat('x', 16 * 1024 * 1024) || '"}]}', $1, 1)`,+ [scope],+ );+ await seedSession(+ raw,+ "nested",+ [message([{ type: "toolCall", id: "call-nested", name: "execute", arguments: {} }])],+ [["tool_call", { tool: "execute", callId: "call-nested" }]],+ );+ await raw.query(+ `INSERT INTO session_tape(session_id, seq, kind, harness, payload, scope_label, created_at)+ VALUES ('nested', 1, 'message', 'pi', '{"role":"assistant","content":' || repeat('[', 200000) || repeat(']', 200000) || '}', $1, 1)`,+ [scope],+ );+ await seedSession(+ raw,+ "plain",+ [message([{ type: "toolCall", id: "call-plain", name: "execute", arguments: {} }])],+ [["tool_call", { tool: "execute", callId: "call-plain" }]],+ );++ await raw.query("CALL prepare_legacy_tape_steps(50)");++ assert.deepEqual(+ (await raw.query("SELECT session_id FROM legacy_tape_steps ORDER BY session_id")).rows.map((r) => r.session_id),+ ["huge", "plain"],+ );+ const skipped = (await raw.query("SELECT session_id, error FROM legacy_tape_skipped")).rows;+ assert.deepEqual(+ skipped.map((r) => r.session_id),+ ["nested"],+ );+ assert.match(skipped[0].error, /^54001: stack depth limit exceeded/);+ assert.deepEqual(+ {+ ...(await progress(raw)),+ skipped: (await raw.query("SELECT skipped_sessions::int AS n FROM legacy_tape_steps_progress")).rows[0].n,+ },+ {+ last_session: "plain",+ done: true,+ invalid_messages: 1,+ keyed_blocks: 2,+ copied_steps: 2,+ copied_turns: 0,+ skipped: 1,+ },+ );+ });+});++test(+ "pg legacy tape step preparation stops instead of skipping when every session in a batch fails",+ { skip },+ async () => {+ await withSchema(async (raw) => {+ for (const id of ["a", "b"])+ await seedSession(+ raw,+ id,+ [message([{ type: "toolCall", id: `call-${id}`, name: "execute", arguments: {} }])],+ [["tool_call", { tool: "execute", callId: `call-${id}` }]],+ );+ await raw.query("ALTER TABLE legacy_tape_steps ADD CONSTRAINT refuse_all CHECK (false) NOT VALID");++ await assert.rejects(+ raw.query("CALL prepare_legacy_tape_steps(50)"),+ /every session in the batch after \(start\) failed, so the failure is not in the data: 23514/,+ );++ assert.deepEqual(await progress(raw), {+ last_session: null,+ done: false,+ invalid_messages: 0,+ keyed_blocks: 0,+ copied_steps: 0,+ copied_turns: 0,+ });+ assert.equal((await raw.query("SELECT count(*)::int AS n FROM legacy_tape_skipped")).rows[0].n, 0);+ });+ },+);++test("pg legacy tape step preparation stops once more than 100 sessions have been skipped", { skip }, async () => {+ await withSchema(async (raw) => {+ await raw.query(+ `INSERT INTO sessions(id, type, scope_id, thread_ref, created_at)+ SELECT 's' || lpad(g::text, 3, '0'), 'dm', $1, 'web:s' || g, 1 FROM generate_series(1, 202) g`,+ [scope],+ );+ await raw.query(+ `INSERT INTO session_tape(session_id, seq, kind, harness, payload, scope_label, created_at)+ SELECT 's' || lpad(g::text, 3, '0'), 0, 'message', 'pi',+ '{"role":"assistant","content":[{"type":"toolCall","id":"call-' || g || '","name":"execute","arguments":{}}]}', $1, 1+ FROM generate_series(1, 202, 2) g`,+ [scope],+ );+ await raw.query(+ `INSERT INTO session_entries(session_id, seq, parent_seq, type, payload, scope_label, created_at)+ SELECT 's' || lpad(g::text, 3, '0'), 0, NULL, 'tool_call', '{"tool":"execute","callId":"call-' || g || '"}', $1, 0+ FROM generate_series(1, 202, 2) g`,+ [scope],+ );+ await raw.query("ALTER TABLE legacy_tape_steps ADD CONSTRAINT refuse_all CHECK (false) NOT VALID");++ await assert.rejects(+ raw.query("CALL prepare_legacy_tape_steps(50)"),+ /more than 100 sessions could not be prepared, so the failure is unlikely to be in the data: 23514/,+ );++ const state = (await raw.query("SELECT done, skipped_sessions::int AS skipped FROM legacy_tape_steps_progress"))+ .rows[0];+ assert.equal(state.done, false);+ assert.ok(state.skipped <= 100, `skipped ${state.skipped}`);+ });+});