Bento, the office suite that fits in a file
TypeScript mit-licenseofficeoffice-365office-suiteoffice-toolsoffice365open-sourcepowerpointself-hostedsingle-page-appslidesslideshareslideshowslideshow-galleryslideshow-makerslideshows
#!/usr/bin/env node
// SPDX-License-Identifier: MIT
// Copyright (c) 2026 The Bento authors
/**
* Does the iOS text extractor agree with the one the extension ships?
*
* `docs/DECISIONS.md` (2026-08-16) settles that the three hosts each carry their
* own copy of the indexer and are held together by a shared fixture corpus
* rather than by shared code. The guarantee that buys is not "cannot diverge"
* but "cannot diverge SILENTLY" — which is only true if something actually
* looks. This is the thing that looks, for `home/ios`.
*
* It drives the REAL `library.js`: `describe()` is exported and takes its cache
* through injectable deps, so a fake file handle is enough to run the shipping
* code path — the title regex, the format regex, the encrypted test, the
* preview slice and `extractText`, exactly as the extension runs them. That
* matters more than it might look. A rig that re-implemented the reference
* would only ever prove the re-implementation agrees with the port; when
* `library.js` changed, both would be wrong together and the rig would stay
* green. Importing it means the reference cannot move without this failing.
*
* The Swift side is compiled straight from `home/ios/BentoIndex.swift` with
* `swiftc`, which is why that file is Foundation-only and has no UIKit in it.
*
* Corpus: real documents found in the repo, plus generated edge cases covering
* the places the two languages are most likely to part company — UTF-16 versus
* grapheme counting, JavaScript's `\s` versus ICU's, ASCII `\d` versus Unicode
* Nd, and every budget boundary at N-1/N/N+1. When the shared corpus lands,
* point this at it: `node scripts/test-tray-index.mjs --corpus <dir>`. It is
* additive, not a replacement — the generated cases are cheap and the real
* documents are the ones that catch what nobody thought to generate.
*
* Usage:
* node scripts/test-tray-index.mjs [--corpus <dir>] [--verbose]
*/
import { execFileSync } from 'node:child_process'
import { createHash } from 'node:crypto'
import { mkdtempSync, readFileSync, writeFileSync, readdirSync, statSync, mkdirSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join, resolve, dirname, basename } from 'node:path'
import { fileURLToPath, pathToFileURL } from 'node:url'
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..')
const args = process.argv.slice(2)
const VERBOSE = args.includes('--verbose')
const corpusIdx = args.indexOf('--corpus')
const EXTRA_CORPUS = corpusIdx !== -1 ? resolve(args[corpusIdx + 1]) : null
const decoder = new TextDecoder('utf-8')
/* ── the generated edge cases ─────────────────────────────────────────────── */
const MARKER = 'id="bento-doc"'
const doc = (body) => `<!doctype html><html><head><title>x</title></head><body>` +
`<script type="application/json" ${MARKER}>${body}</script></body></html>`
function cases() {
const out = []
const add = (name, html) => out.push({ name, html })
// Nothing to find.
add('empty', '')
add('no-marker', '<!doctype html><html><body>hello there friend</body></html>')
// The marker with no closing script tag — the reference returns null rather
// than running to the end of the file.
add('marker-unterminated', `<!doctype html><body><script ${MARKER}>{"title":"Orphan text here"}`)
// Values are taken from `:"…"`, never from keys.
add('keys-are-not-values', doc('{"heading":"alpha bravo","charlie":"delta echo"}'))
// Fewer than three consecutive ASCII letters = an id, a colour, a number.
add('short-values-skipped', doc('{"a":"#ff0088","b":"12345","c":"ab","d":"real words here"}'))
// Data URIs: 199 chars stays, 200 goes. The boundary is the whole point.
add('data-uri-199', doc(`{"src":"data:${'A'.repeat(199)}","t":"keep this prose"}`))
add('data-uri-200', doc(`{"src":"data:${'A'.repeat(200)}","t":"keep this prose"}`))
// A `data:` inside a run that was too short to strip — the reference resumes
// one character later, so the inner one is still found.
add('data-uri-nested', doc(`{"src":"data:xx data:${'B'.repeat(400)}","t":"prose here"}`))
// Value length: 400 units matches, 401 is skipped entirely.
add('value-400', doc(`{"v":"${'word '.repeat(80).slice(0, 400)}"}`))
add('value-401', doc(`{"v":"${('word '.repeat(81)).slice(0, 401)}","after":"tail words"}`))
// Escaped quotes keep a value together.
add('escaped-quote', doc('{"v":"she said \\"hello there\\" loudly"}'))
add('escaped-backslash', doc('{"v":"path\\\\to\\\\somewhere useful"}'))
// A trailing backslash cannot start a repetition, so the value never closes.
add('trailing-backslash', doc('{"v":"dangling\\\\"}'))
// Tags: 200 non-`>` chars strips, 201 does not.
add('tag-200', doc(`{"v":"alpha <${'x'.repeat(198)}> bravo"}`))
add('tag-201', doc(`{"v":"alpha <${'x'.repeat(199)}> bravo"}`))
add('tag-nested-lt', doc('{"v":"alpha <<b> bravo charlie"}'))
add('tag-empty', doc('{"v":"alpha <> bravo charlie"}'))
// Entities. The Arabic-Indic digits are the ICU trap: JavaScript's `\\d` is
// ASCII, so `&#١٢٣;` is NOT an entity and must survive.
add('entities-named', doc('{"v":"alpha & bravo &NBSP; charlie"}'))
add('entities-numeric', doc('{"v":"alpha & bravo Ӓ charlie"}'))
add('entities-arabic-indic', doc('{"v":"alpha &#١٢٣; bravo charlie"}'))
add('entities-malformed', doc('{"v":"alpha ¬anentity bravo & charlie"}'))
// Whitespace. NBSP and U+FEFF collapse (JavaScript `\\s`); U+200B does not.
add('ws-nbsp', doc('{"v":"alpha bravo charlie"}'))
add('ws-feff', doc('{"v":"alphabravo charlie"}'))
add('ws-zwsp-kept', doc('{"v":"alphabravo charlie"}'))
add('ws-exotic', doc('{"v":"alpha bravo charlie"}'))
add('ws-vertical-tab', doc('{"v":"alphabravo charlie"}'))
// UTF-16 counting. Emoji are surrogate pairs: a grapheme-counting port
// disagrees with the reference on every one of these.
add('astral-emoji', doc('{"v":"alpha \u{1F600}\u{1F601}\u{1F602} bravo charlie"}'))
add('cjk', doc('{"v":"alpha 你好世界 bravo charlie"}'))
add('combining', doc('{"v":"alpha ééé bravo charlie"}'))
// A 400-unit value made of astral characters — 200 characters, 400 units.
add('astral-value-400', doc(`{"v":"${'\u{1F600}'.repeat(200)}","after":"tail words here"}`))
// Title.
add('title-plain', doc('{"title":"Quarterly Review","v":"body words here"}'))
add('title-escaped', doc('{"title":"She said \\"go\\"","v":"body words here"}'))
add('title-empty', doc('{"title":"","v":"body words here"}'))
add('title-spaced-colon', doc('{"title" : "Spaced Out","v":"body words here"}'))
add('title-too-long', doc(`{"title":"${'t'.repeat(201)}","v":"body words here"}`))
add('title-201-boundary', doc(`{"title":"${'t'.repeat(200)}","v":"body words here"}`))
// Format, and the reason it must scan past a non-matching key.
add('format-slides', doc('{"format":"bento/slides","v":"body words here"}'))
add('format-unrelated-first', doc('{"format":"text/html","format":"bento/spaces","v":"body words"}'))
add('format-uppercase', doc('{"format":"bento/Slides","v":"body words here"}'))
// Encrypted: no text, no preview. Both spellings.
add('encrypted-format', doc('{"format":"bento/enc","ct":"opaque ciphertext here"}'))
add('encrypted-attr', `<!doctype html><body data-bento-enc="1">` +
`<script ${MARKER}>{"title":"Secret","v":"should not be extracted"}</script>` +
`<div data-bento-preview="1">leak</div><script data-bento-preview="1"></script></body>`)
// Preview slice, present and absent.
add('preview-present', doc('{"v":"body words here"}') +
`<div data-bento-preview="1"><p>page one</p></div><script data-bento-preview="1">x</script>`)
add('preview-no-remover', doc('{"v":"body words here"}') +
`<div data-bento-preview="1"><p>page one</p></div>`)
// The budget, and the deliberate overshoot: the value is kept, THEN the size
// is checked, so the last value crosses 40KB before the final slice cuts back.
const filler = Array.from({ length: 200 }, (_, i) => `"k${i}":"${`lorem ipsum dolor sit amet ${i} `.repeat(8)}"`).join(',')
add('budget-overshoot', doc(`{${filler}}`))
// Astral characters straddling the 40KB cut — the one documented deviation.
const bigAstral = Array.from({ length: 300 }, (_, i) => `"k${i}":"word ${'\u{1F600}'.repeat(80)} ${i}"`).join(',')
add('budget-astral-cut', doc(`{${bigAstral}}`))
return out
}
/* ── corpus ───────────────────────────────────────────────────────────────── */
function realDocuments() {
const found = []
const skip = new Set(['node_modules', '.git', '.worktrees', 'dist', 'dist-single'])
const walk = (dir, depth) => {
if (depth > 3) return
let entries
try { entries = readdirSync(dir, { withFileTypes: true }) } catch { return }
for (const e of entries) {
if (e.name.startsWith('.') || skip.has(e.name)) continue
const p = join(dir, e.name)
if (e.isDirectory()) walk(p, depth + 1)
else if (/\.html?$/i.test(e.name)) {
try { if (statSync(p).size > 0) found.push(p) } catch { /* raced */ }
}
}
}
walk(ROOT, 0)
// Deterministic order, and a cap so the rig stays quick on a big working tree.
return found.sort().slice(0, 60)
}
function buildCorpus() {
const dir = mkdtempSync(join(tmpdir(), 'bento-index-'))
const files = []
for (const c of cases()) {
const p = join(dir, `${c.name}.bento.html`)
writeFileSync(p, c.html, 'utf8')
files.push(p)
}
const generated = files.length
for (const p of realDocuments()) files.push(p)
if (EXTRA_CORPUS) {
let entries = []
try { entries = readdirSync(EXTRA_CORPUS, { withFileTypes: true }) } catch {
console.error(`corpus directory not readable: ${EXTRA_CORPUS}`)
process.exit(2)
}
for (const e of entries) if (e.isFile() && /\.html?$/i.test(e.name)) files.push(join(EXTRA_CORPUS, e.name))
}
return { dir, files, generated }
}
/* ── the reference, run as the extension runs it ──────────────────────────── */
async function referenceFor(files) {
const lib = await import(pathToFileURL(join(ROOT, 'home/webext/src/library.js')).href)
const deps = { get: async () => null, put: async () => {} }
const out = new Map()
for (const path of files) {
const bytes = readFileSync(path)
// What `listDocuments` computes and hands to describe() as the title
// fallback — extensions stripped, both of them.
const base = basename(path).replace(/\.bento\.html$/i, '').replace(/\.html?$/i, '')
const file = {
size: bytes.length,
lastModified: 0,
slice: (a, b) => ({ text: async () => decoder.decode(bytes.subarray(a, b)) }),
text: async () => decoder.decode(bytes),
}
const d = { handle: { getFile: async () => file, name: base }, base, folder: 'corpus', rel: [] }
const meta = await lib.describe(d, deps)
// `sniff` is not exported; it is one `includes` on the first 64KB and the
// rig checks the same thing rather than reaching into the module.
const isDoc = decoder.decode(bytes.subarray(0, 64 * 1024)).includes(MARKER)
out.set(path, {
isDoc,
title: meta.title,
app: meta.app ?? null,
encrypted: meta.encrypted,
text: meta.text ?? null,
preview: meta.preview ?? null,
})
}
return out
}
/* ── the port, compiled ───────────────────────────────────────────────────── */
const MAIN_SWIFT = `
import Foundation
struct Row: Codable {
let path: String
let isDoc: Bool
let title: String?
let app: String?
let encrypted: Bool
let text: String?
let preview: String?
}
let encoder = JSONEncoder()
for path in CommandLine.arguments.dropFirst() {
guard let data = FileManager.default.contents(atPath: path) else { continue }
let head = String(decoding: data.prefix(BentoIndex.headBytes), as: UTF8.self)
let sniff = String(decoding: data.prefix(BentoIndex.sniffBytes), as: UTF8.self)
let whole = String(decoding: data, as: UTF8.self)
let meta = BentoIndex.describe(head: head, sniffHead: sniff, whole: whole)
let row = Row(path: path,
isDoc: meta.isDocument,
title: meta.title,
app: meta.app,
encrypted: meta.encrypted,
text: meta.text,
preview: meta.preview)
if let out = try? encoder.encode(row), let s = String(data: out, encoding: .utf8) {
print(s)
}
}
`
/**
* `BentoIndex.swift` is Foundation-only, which is what lets this rig REALLY RUN
* on the Linux CI runner — Swift is installed there and the file compiles
* unchanged, so the whole diff happens on every push rather than only on a Mac.
* That is the payoff for keeping UIKit out of it.
*
* The skip below is for a machine with no Swift at all. Deliberately LOUD: a
* skip that reads like a pass is how a rig stops being run.
*/
function haveSwift() {
try {
execFileSync('swiftc', ['--version'], { stdio: 'pipe' })
return true
} catch {
return false
}
}
function buildSwift(dir) {
const mainPath = join(dir, 'main.swift')
writeFileSync(mainPath, MAIN_SWIFT, 'utf8')
const bin = join(dir, 'indexer')
const src = join(ROOT, 'home/ios/BentoIndex.swift')
try {
execFileSync('swiftc', ['-O', '-o', bin, src, mainPath], { stdio: 'pipe' })
} catch (e) {
console.error('swiftc failed:\n' + (e.stderr?.toString() ?? e.message))
process.exit(2)
}
return bin
}
function runSwift(bin, files) {
const out = new Map()
// Chunked so the argument list cannot overflow on a large corpus.
for (let i = 0; i < files.length; i += 40) {
const chunk = files.slice(i, i + 40)
const stdout = execFileSync(bin, chunk, { maxBuffer: 512 * 1024 * 1024 }).toString()
for (const line of stdout.split('\n')) {
if (!line.trim()) continue
const row = JSON.parse(line)
out.set(row.path, {
isDoc: row.isDoc,
// `library.js`'s describe() ALWAYS returns a title, falling back to the
// file's base name, because its caller has already sniffed and it never
// faces a non-document. The Swift port answers all the questions in one
// call (the shared contract in home/fixtures) so it returns nil, and the
// listing supplies the file name. Same behaviour, different seam — the
// fallback is applied here so the two are compared like for like.
title: row.title ?? basename(row.path)
.replace(/\.bento\.html$/i, '').replace(/\.html?$/i, ''),
rawTitle: row.title ?? null,
app: row.app ?? null,
encrypted: row.encrypted,
text: row.text ?? null,
preview: row.preview ?? null,
})
}
}
return out
}
/* ── compare ──────────────────────────────────────────────────────────────── */
const show = (v) => v === null ? 'null' : JSON.stringify(v.length > 90 ? v.slice(0, 90) + '…' : v)
function firstDifference(a, b) {
if (a === null || b === null) return null
const n = Math.min(a.length, b.length)
for (let i = 0; i < n; i++) if (a[i] !== b[i]) return i
return a.length === b.length ? null : n
}
/**
* The one difference that is allowed, stated narrowly enough to be a check.
*
* `extractText` ends with `.slice(0, 40KB)` on UTF-16 units, which can land
* between a surrogate pair. JavaScript keeps the lone half; a Swift `String`
* cannot hold one at all, so the port drops it — the deviation is forced by the
* language rather than chosen, and dropping beats decoding to U+FFFD in a
* search index either way. Recognised here as EXACTLY "js has one more unit,
* that unit is an unpaired high surrogate, and everything before it is
* identical", so any other disagreement about text still fails.
*/
function isSurrogateCut(a, b) {
if (typeof a !== 'string' || typeof b !== 'string') return false
if (a.length !== b.length + 1) return false
const last = a.charCodeAt(a.length - 1)
if (last < 0xd800 || last > 0xdbff) return false
return a.slice(0, -1) === b
}
function compare(path, js, sw, failures, known) {
const bad = []
for (const field of ['isDoc', 'title', 'app', 'encrypted', 'text', 'preview']) {
if (js[field] === sw[field]) continue
if (field === 'text' && isSurrogateCut(js[field], sw[field])) { known.push(basename(path)); continue }
let detail = ` js: ${show(js[field])}\n sw: ${show(sw[field])}`
if (typeof js[field] === 'string' && typeof sw[field] === 'string') {
const at = firstDifference(js[field], sw[field])
const ctx = (s) => JSON.stringify(s.slice(Math.max(0, at - 25), at + 25))
detail = ` lengths js=${js[field].length} sw=${sw[field].length}, first differs at ${at}\n` +
` js …${ctx(js[field])}…\n sw …${ctx(sw[field])}…`
}
bad.push(` ${field}:\n${detail}`)
}
if (bad.length) failures.push(` ${basename(path)}\n${bad.join('\n')}`)
return bad.length === 0
}
/* ── main ─────────────────────────────────────────────────────────────────── */
if (!haveSwift()) {
console.log('tray index: SKIPPED — no swiftc on this machine.')
console.log(' This rig is the only thing holding home/ios/BentoIndex.swift to')
console.log(' home/webext/src/library.js. Run it on a Mac before changing either.')
process.exit(0)
}
const { dir, files, generated } = buildCorpus()
const bin = buildSwift(dir)
const js = await referenceFor(files)
const sw = runSwift(bin, files)
const failures = []
const known = []
let checked = 0
let withText = 0
for (const path of files) {
const a = js.get(path), b = sw.get(path)
if (!a || !b) { failures.push(` ${basename(path)}\n missing result (js=${!!a} swift=${!!b})`); continue }
checked++
if (a.text) withText++
const ok = compare(path, a, b, failures, known)
if (VERBOSE) console.log(`${ok ? 'ok ' : 'FAIL'} ${basename(path)}`)
}
const real = files.length - generated - (EXTRA_CORPUS ? 0 : 0)
/* ── the shared corpus ────────────────────────────────────────────────────── */
/**
* `home/fixtures/` is the corpus all three hosts answer, with `expected.json`
* as the answer key and `home/doc-index.mjs` as the reference. It is a DIFFERENT
* guarantee from the diff above and worth having both: that one proves this port
* tracks the extension as it moves, this one proves all three hosts agree on one
* frozen set of answers — including Kotlin, which nothing on this machine can run.
*
* It used to SKIP when the directory was absent, because it arrived with the
* Android work and a rig that fails until an unrelated branch lands is a rig
* people learn to ignore. It has landed, so the skip now protects nothing and
* costs the whole check: the tray→home rename moved the corpus, this kept
* reading `tray/fixtures`, and for weeks the rig printed "absent, skipped" in
* green while eleven cases went untested. A path that is supposed to exist is
* a FAILURE when it does not.
*/
/**
* Any `expected.json` sitting in a fixtures directory elsewhere in the tree, so a
* MOVED corpus is told apart from a genuinely absent one. Cheap: two shallow
* levels, no full walk. (Written without the glob it describes: a literal
* star-slash in this comment would close it early.)
*
* This DECIDES nothing. The policy above is absolute — absent is a failure
* either way — and this only enriches the message, because "MISSING" sends a
* reader looking for a deleted corpus while "MISSING, and one is sitting at X"
* sends them to fix a path. Its ancestor in PR #405 threw instead, which aborted
* the run before the Swift-diff summary printed and left a raw stack trace as
* the last thing on screen — the rig read as crashed rather than stale-pathed.
* Diagnosis belongs in the message, not in the control flow.
*
* From bento-team-home-ios (PR #405); `tray` dropped from the probe roots, the
* rename having landed.
*/
function strayCorpus() {
for (const top of ['home', 'scripts', '.']) {
const base = join(ROOT, top)
let entries
try { entries = readdirSync(base, { withFileTypes: true }) } catch { continue }
for (const e of entries) {
if (!e.isDirectory()) continue
for (const probe of [join(base, e.name, 'expected.json'),
join(base, e.name, 'fixtures', 'expected.json')]) {
try { readFileSync(probe); return probe.replace(ROOT + '/', '') } catch { /* keep looking */ }
}
}
}
return null
}
function sharedCorpus() {
const dir = join(ROOT, 'home/fixtures')
let expected
try { expected = JSON.parse(readFileSync(join(dir, 'expected.json'), 'utf8')) } catch { return null }
const names = Object.keys(expected.cases).sort()
const paths = names.map((n) => join(dir, 'cases', n))
const rows = runSwift(bin, paths)
const sha = (s) => createHash('sha256').update(s, 'utf8').digest('hex').slice(0, 16)
const bad = []
for (const name of names) {
const want = expected.cases[name]
const r = rows.get(join(dir, 'cases', name))
if (!r) { bad.push(` ${name}: the port produced no answer`); continue }
const mine = {
isDocument: r.isDoc,
title: r.rawTitle,
app: r.app,
encrypted: r.encrypted,
hasPreview: r.preview !== null,
textLength: r.text === null ? null : r.text.length,
textSha256_16: r.text === null ? null : sha(r.text),
text: r.text,
}
for (const key of Object.keys(want)) {
if (JSON.stringify(want[key]) === JSON.stringify(mine[key])) continue
bad.push(` ${name} · ${key}: corpus says ${show(want[key])}, port says ${show(mine[key])}`)
}
}
// The budgets are part of the contract too — a port that agreed on every case
// while carrying a different TEXT_BUDGET agrees only by luck.
const budgets = [['textBudget', 40960], ['sniffBytes', 65536], ['headBytes', 307200]]
for (const [key, mineValue] of budgets) {
if (expected[key] !== undefined && expected[key] !== mineValue) {
bad.push(` ${key}: corpus says ${expected[key]}, port uses ${mineValue}`)
}
}
return { count: names.length, bad }
}
const shared = sharedCorpus()
console.log(`\ntray index: ${checked} documents compared ` +
`(${generated} generated edge cases, ${real} from the tree${EXTRA_CORPUS ? `, corpus ${EXTRA_CORPUS}` : ''})`)
console.log(` ${withText} yielded searchable text`)
if (known.length) {
console.log(` ${known.length} matched only after the documented surrogate-cut deviation ` +
`(${known.join(', ')})`)
}
if (shared) {
console.log(` shared corpus (home/fixtures): ${shared.count} cases` +
(shared.bad.length ? ` — ${shared.bad.length} DISAGREE` : ' — all agree'))
} else {
console.error(' shared corpus (home/fixtures): MISSING — expected.json did not load.')
console.error(' This corpus has landed; absent means a moved or broken path, not "not yet".')
const stray = strayCorpus()
if (stray) {
console.error(` It IS at ${stray} — this rig's path is stale. Update the path;`)
console.error(' do not restore the skip that hid this for weeks.')
}
// exitCode, not exit(): the Swift-diff summary above has already printed and
// the disagreement report below still needs to. A stale path must not withhold
// the other half of the rig from the reader.
process.exitCode = 1
}
if (failures.length || shared?.bad.length) {
if (failures.length) {
console.error(`\n${failures.length} disagreement(s) between library.js and BentoIndex.swift:\n`)
console.error(failures.join('\n\n'))
}
if (shared?.bad.length) {
console.error('\nDisagreements with the shared corpus (home/fixtures/expected.json):\n')
console.error(shared.bad.join('\n'))
}
process.exit(1)
}
console.log(' swift port agrees with library.js on every field')