Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
import test from 'node:test';
import assert from 'node:assert/strict';
import * as Chrome from '../src/chrome/src/agent/apocalypse-mode.js';
import * as Firefox from '../src/firefox/src/agent/apocalypse-mode.js';
// Exact existing synthetic ZIM builder; no archive files, browser, or network.
function minimalWikipediaZimFixture(options = {}) {
const encoder = new TextEncoder();
const language = options.language || 'eng';
const sourceLanguage = ({ ben: 'bn', tgl: 'tl', tur: 'tr' })[language] || language.slice(0, 2);
const articlePath = options.articlePath || 'Alan_Turing';
const articleTitle = options.articleTitle || articlePath.replace(/_/g, ' ');
const imageAssets = Array.isArray(options.imageAssets) ? options.imageAssets : [];
const mimeTypes = ['text/html', 'text/plain'];
for (const asset of imageAssets) {
const mimeType = String(asset.mimeType || '').trim().toLowerCase();
if (mimeType && !mimeTypes.includes(mimeType)) mimeTypes.push(mimeType);
}
const metadata = options.wikipedia === false ? {
Language: 'eng', Name: 'project_gutenberg_en', Source: 'www.gutenberg.org', Tags: '_category:books',
} : {
Language: language, Name: `wikipedia_${sourceLanguage}_test`, Source: `https://${sourceLanguage}.wikipedia.org/`,
Tags: options.tags ?? 'wikipedia;_category:wikipedia',
};
const articleNamespace = options.articleNamespace
|| (options.fullTextIndex === 'legacy' ? 'A' : 'C');
const entries = [
...(options.fullTextIndex
? [{
namespace: options.fullTextIndex === 'legacy' ? 'Z' : 'X',
url: options.fullTextIndex === 'legacy' ? '/fulltextIndex/xapian' : 'fulltext/xapian',
title: 'xapian', mimeType: 1, contents: 'not-a-real-index',
}]
: []),
{
namespace: articleNamespace, url: articlePath, title: articleTitle, mimeType: 0,
contents: options.articleHtml || '<!doctype html><html><body><p>Alan Turing was an English mathematician, computer scientist, logician, and cryptanalyst.</p></body></html>',
},
...Object.entries(metadata).map(([url, contents]) => ({ namespace: 'M', url, title: url, mimeType: 1, contents })),
...imageAssets.map(asset => ({
namespace: 'C',
url: String(asset.url || ''),
title: String(asset.title || asset.url || ''),
mimeType: mimeTypes.indexOf(String(asset.mimeType || '').trim().toLowerCase()),
contents: asset.contents,
})),
];
if (options.redirectTrap) {
entries.push(
{ namespace: 'C', url: 'Science', title: 'Science', redirectUrl: 'The_New_York_Times' },
{ namespace: 'C', url: 'Science_article', title: 'Science article', mimeType: 0, contents: '<p>Science is the systematic study of the natural world.</p>' },
{ namespace: 'C', url: 'The_New_York_Times', title: 'The New York Times', mimeType: 0, contents: '<p>A newspaper based in New York City.</p>' },
);
}
if (options.caseSensitiveTitle) {
for (let index = 0; index < 30; index += 1) {
entries.push({
namespace: 'C', url: `New_Example_${String(index).padStart(2, '0')}`, title: `New Example ${index}`, mimeType: 0,
contents: `<p>Prefix decoy ${index}.</p>`,
});
}
entries.push({
namespace: 'C', url: 'New_York_City', title: 'New York City', mimeType: 0,
contents: '<p>New York City is the most populous city in the United States.</p>',
});
entries.push({ namespace: 'C', url: 'NYC', title: 'NYC', redirectUrl: 'New_York_City' });
entries.push({
namespace: 'C', url: 'iPhone', title: 'iPhone', mimeType: 0,
contents: '<p>The iPhone is a line of smartphones designed by Apple.</p>',
});
entries.push({
namespace: 'C', url: 'YouTube', title: 'YouTube', mimeType: 0,
contents: '<p>YouTube is an online video sharing platform.</p>',
});
entries.push({
namespace: 'C', url: 'YouTube_TV', title: 'YouTube TV', mimeType: 0,
contents: '<p>YouTube TV is a streaming television service.</p>',
});
entries.push({
namespace: 'C', url: 'OpenAI', title: 'OpenAI', mimeType: 0,
contents: '<p>OpenAI is an artificial intelligence research organization.</p>',
});
entries.push({
namespace: 'C', url: 'United_States', title: 'United States', mimeType: 0,
contents: '<p>The United States is a country in North America.</p>',
});
entries.push({ namespace: 'C', url: 'USA', title: 'USA', redirectUrl: 'United_States' });
entries.push({ namespace: 'C', url: 'Big_Apple', title: 'Big Apple', redirectUrl: 'New_York_City' });
}
entries.sort((left, right) => {
const leftKey = `${left.namespace}/${left.url}`;
const rightKey = `${right.namespace}/${right.url}`;
return leftKey < rightKey ? -1 : leftKey > rightKey ? 1 : 0;
});
const blobs = entries.filter(entry => entry.contents != null);
blobs.forEach((entry, index) => { entry.blobIndex = index; });
const mime = encoder.encode(`${mimeTypes.join('\0')}\0\0`);
const offsetsBytes = (blobs.length + 1) * 4;
const encodedBlobs = blobs.map((entry) => {
if (entry.contents instanceof Uint8Array) return entry.contents;
if (ArrayBuffer.isView(entry.contents)) {
return new Uint8Array(entry.contents.buffer, entry.contents.byteOffset, entry.contents.byteLength);
}
if (entry.contents instanceof ArrayBuffer) return new Uint8Array(entry.contents);
return encoder.encode(String(entry.contents));
});
const cluster = new Uint8Array(1 + offsetsBytes + encodedBlobs.reduce((sum, value) => sum + value.length, 0));
cluster[0] = 1;
const clusterView = new DataView(cluster.buffer);
let blobOffset = offsetsBytes;
encodedBlobs.forEach((value, index) => {
clusterView.setUint32(1 + index * 4, blobOffset, true);
cluster.set(value, 1 + blobOffset);
blobOffset += value.length;
});
clusterView.setUint32(1 + blobs.length * 4, blobOffset, true);
const directories = entries.map((entry) => {
const url = encoder.encode(entry.url);
const title = encoder.encode(entry.title);
const redirect = Boolean(entry.redirectUrl);
const directory = new Uint8Array((redirect ? 12 : 16) + url.length + 1 + title.length + 1);
const directoryView = new DataView(directory.buffer);
directoryView.setUint16(0, redirect ? 0xffff : entry.mimeType, true);
directory[3] = entry.namespace.charCodeAt(0);
directoryView.setUint32(8, redirect ? entries.findIndex(candidate => candidate.url === entry.redirectUrl && candidate.namespace === 'C') : 0, true);
if (!redirect) directoryView.setUint32(12, entry.blobIndex, true);
directory.set(url, redirect ? 12 : 16);
directory.set(title, (redirect ? 13 : 17) + url.length);
return directory;
});
// Use the common ZIM layout where directory entries precede cluster data.
// The final cluster therefore ends at checksumPos, not at the first URL
// pointer's directory-entry offset.
const urlPointerPosition = Math.max(128, Math.ceil((80 + mime.length) / 8) * 8);
const clusterPointerPosition = urlPointerPosition + entries.length * 8;
const directoryStart = clusterPointerPosition + 8;
const directoryPositions = [];
let directoryOffset = directoryStart;
for (const directory of directories) {
directoryPositions.push(directoryOffset);
directoryOffset += directory.length;
}
const clusterStart = directoryOffset;
const checksumPosition = clusterStart + cluster.length;
const bytes = new Uint8Array(checksumPosition + 16);
const view = new DataView(bytes.buffer);
view.setUint32(0, 0x044d495a, true);
view.setUint16(4, 6, true);
view.setUint16(6, 3, true);
view.setUint32(24, entries.length, true);
view.setUint32(28, 1, true);
view.setBigUint64(32, BigInt(urlPointerPosition), true);
view.setBigUint64(40, 0xffffffffffffffffn, true);
view.setBigUint64(48, BigInt(clusterPointerPosition), true);
view.setBigUint64(56, 80n, true);
view.setUint32(64, 0, true);
view.setUint32(68, 0xffffffff, true);
view.setBigUint64(72, BigInt(checksumPosition), true);
bytes.set(mime, 80);
bytes.set(cluster, clusterStart);
directories.forEach((directory, index) => {
bytes.set(directory, directoryPositions[index]);
view.setBigUint64(urlPointerPosition + index * 8, BigInt(directoryPositions[index]), true);
});
view.setBigUint64(clusterPointerPosition, BigInt(clusterStart), true);
return new Blob([bytes], { type: 'application/x-zim' });
}
const replacement = '\uFFFD';
const invalid = [
['decimal zero', '�'], ['hex zero', '�'],
['first surrogate', '�'], ['last surrogate', '�'],
['decimal beyond Unicode', '�'], ['hex beyond Unicode', '�'],
['oversized decimal', '&#' + '9'.repeat(400) + ';'],
['oversized hexadecimal', '&#x' + 'F'.repeat(400) + ';'],
];
for (const [engine, runtime] of [['chrome', Chrome], ['firefox', Firefox]]) {
for (const [label, entity] of invalid) {
test(engine + ': malformed scalar ' + label + ' is replaceable article text', async () => {
const archive = await runtime.openKiwixZim(minimalWikipediaZimFixture({ articleHtml: '<p>Alan Turing before ' + entity + ' after.</p>' }));
const article = await archive.readArticle('Alan_Turing');
assert.equal(article.text, 'Alan Turing before ' + replacement + ' after.');
assert.equal(article.unsafeHtml, '<p>Alan Turing before ' + entity + ' after.</p>');
const results = await archive.search('Alan Turing');
assert.equal(results.length, 1);
assert.equal(results[0].excerpt, article.text);
});
}
test(engine + ': valid scalars and preexisting stripping/entity order remain', async () => {
const html='<script>discard �</script><style>discard</style><!--discard--><p>Alan Turing A 🧠  ퟿  € A A &#1114112; &lt;tag&gt;</p><p>Second line.</p>';
const archive=await runtime.openKiwixZim(minimalWikipediaZimFixture({articleHtml:html}));
const expected='Alan Turing A 🧠'+String.fromCodePoint(0x10ffff)+' '+String.fromCodePoint(0xd7ff)+' '+String.fromCodePoint(0xe000)+' '+String.fromCodePoint(0x80)+' A A � <tag>';
const article=await archive.readArticle('Alan_Turing');
assert.equal(article.text,expected+'\nSecond line.');
assert.equal((await archive.search('Alan Turing'))[0].excerpt,expected+' Second line.');
assert.equal(article.unsafeHtml,html);
});
test(engine + ': actual search/read callers preserve ready archive after one malformed article', async () => {
const blob=minimalWikipediaZimFixture({articleHtml:'<p>Alan Turing before � after.</p>', imageAssets:[{url:'Second_Article',title:'Second Article',mimeType:'text/html',contents:'<p>Second Article remains available.</p>'}]});
let current={id:engine+'-archive',archiveKind:'wikipedia',filename:'synthetic.zim',status:'ready',generation:1,updatedAt:1,size:blob.size,target:{kind:'opfs',key:'synthetic'}};
const writes=[];
const store={async getConfig(){return {enabled:true};},async listArchives(){return [current];},async getArchive(id){assert.equal(id,current.id);return current;},async putArchive(next){writes.push(next);current=next;}};
const provider=runtime.createKiwixZimProvider({storage:{async open(target){assert.equal(target,current.target);return blob;}},archiveCache:new Map()});
const options={store,providers:[provider]};
let results, failure;
try { results=await runtime.searchApocalypseArchives('Alan Turing',options); } catch(error) { failure=error; }
assert.equal(current.status,'ready',failure?.message); assert.equal(writes.length,0);
if(failure) throw failure;
assert.equal(results.length,1);
assert.equal(results[0].excerpt,'Alan Turing before '+replacement+' after.');
const article=await runtime.readApocalypseArticle(current.id,'Alan_Turing',options);
assert.equal(article.text,results[0].excerpt);
const second=await runtime.readApocalypseArticle(current.id,'Second_Article',options);
assert.equal(second.text,'Second Article remains available.');
assert.equal((await runtime.searchApocalypseArchives('Second Article',options))[0].path,'Second_Article');
assert.equal(current.status,'ready'); assert.equal(writes.length,0);
});
test(engine + ': real corrupt archive still becomes unreadable and abort does not mutate state', async () => {
const corrupt=new Blob([new Uint8Array(96)]);
let current={id:engine+'-corrupt',archiveKind:'wikipedia',filename:'corrupt.zim',status:'ready',generation:1,updatedAt:1,size:corrupt.size,target:{kind:'opfs',key:'corrupt'}};
const writes=[];
const store={async getConfig(){return {enabled:true};},async listArchives(){return [current];},async getArchive(){return current;},async putArchive(next){writes.push(next);current=next;}};
const provider=runtime.createKiwixZimProvider({storage:{async open(){return corrupt;}},archiveCache:new Map()});
const controller=new AbortController();controller.abort();
await assert.rejects(runtime.searchApocalypseArchives('Alan Turing',{store,providers:[provider],signal:controller.signal}),error=>error.name==='AbortError');
assert.equal(current.status,'ready');assert.equal(writes.length,0);
await assert.rejects(runtime.searchApocalypseArchives('Alan Turing',{store,providers:[provider]}),/Installed archive could not be read/);
assert.equal(current.status,'error');assert.equal(current.errorKind,'archive-unreadable');assert.equal(writes.length,1);
});
}