Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

fix(content): read the Library export's pages in bounded calls, and wrap Excel's page text (#1365)

Cihanbde9b0e34ddf
diff --git a/web/src/app/[locale]/(dashboard)/dashboard/content/library/_library-export.tsx b/web/src/app/[locale]/(dashboard)/dashboard/content/library/_library-export.tsx
index a63d98ed5..d40882a7f 100644
--- a/web/src/app/[locale]/(dashboard)/dashboard/content/library/_library-export.tsx
+++ b/web/src/app/[locale]/(dashboard)/dashboard/content/library/_library-export.tsx
@@ -26,7 +26,11 @@ import {
type LibraryExportFormat,
type LibraryExportItem,
} from '@/lib/content/draft-export';
-import type { ContentDraftListItem, LibraryTab } from '@/lib/content/drafts';
+import {
+ DRAFT_PAGES_PER_CALL,
+ type ContentDraftListItem,
+ type LibraryTab,
+} from '@/lib/content/drafts';
import { saveBlob } from '@/lib/download';
/**
@@ -69,14 +73,15 @@ export function useLibraryExport(brandId: string | null) {
if (!brandId || drafts.length === 0) return;
setExporting(true);
try {
- const pages = new Map(
- (
- await getContentDraftPages(
- brandId,
- drafts.map((d) => d.id),
- )
- ).map((p) => [p.id, p]),
- );
+ // A call's pages come back whole, so the list is read a slice at a
+ // time; ids a call left for its response's size go first in the next.
+ const pages = new Map<string, DraftPage>();
+ let queue = drafts.map((d) => d.id);
+ while (queue.length > 0) {
+ const read = await getContentDraftPages(brandId, queue.slice(0, DRAFT_PAGES_PER_CALL));
+ for (const page of read.pages) pages.set(page.id, page);
+ queue = [...read.rest, ...queue.slice(DRAFT_PAGES_PER_CALL)];
+ }
const items = drafts.flatMap((draft) => {
const page = pages.get(draft.id);
return page ? [item(draft, page)] : [];
@@ -105,7 +110,9 @@ export function useLibraryExport(brandId: string | null) {
const exportDraft = async (draft: ContentDraftListItem, format: DraftExportFormat) => {
if (!brandId) return;
try {
- const [page] = await getContentDraftPages(brandId, [draft.id]);
+ const {
+ pages: [page],
+ } = await getContentDraftPages(brandId, [draft.id]);
if (!page) throw new Error('not_found');
const title = titleOf(draft);
saveBlob(
diff --git a/web/src/app/[locale]/(dashboard)/dashboard/content/library/page.test.tsx b/web/src/app/[locale]/(dashboard)/dashboard/content/library/page.test.tsx
index 43ab36f9f..0e331f52c 100644
--- a/web/src/app/[locale]/(dashboard)/dashboard/content/library/page.test.tsx
+++ b/web/src/app/[locale]/(dashboard)/dashboard/content/library/page.test.tsx
@@ -122,14 +122,15 @@ beforeEach(() => {
setContentDraftStatus.mockReset().mockResolvedValue(undefined);
archiveContentDraft.mockReset().mockResolvedValue(undefined);
deleteContentDraft.mockReset().mockResolvedValue(undefined);
- getContentDraftPages.mockReset().mockImplementation(async (_brand: string, ids: string[]) =>
- ids.map((id) => ({
+ getContentDraftPages.mockReset().mockImplementation(async (_brand: string, ids: string[]) => ({
+ pages: ids.map((id) => ({
id,
title: 'Pipeline forecasting',
bodyHtml: '<h1>Pipeline forecasting</h1><p>Teams trust it.</p>',
sourceUrl: 'https://acme.example/pricing',
})),
- );
+ rest: [],
+ }));
saveBlob.mockReset();
role.canManage = true;
});
@@ -461,6 +462,52 @@ describe('the Library’s Export and Created filter (CE-002)', () => {
return [...document.body.querySelectorAll<HTMLElement>('[role="menuitem"]')];
};
+ it('reads a long list’s pages fifty at a time, in order', async () => {
+ getContentDrafts.mockResolvedValue(
+ Array.from({ length: 120 }, (_, i) => row({ id: `d${i}`, title: `Draft ${i}` })),
+ );
+ await renderPage();
+ const items = await openExport();
+ await act(async () => items[0].click());
+ await act(async () => {});
+
+ const calls = getContentDraftPages.mock.calls as [string, string[]][];
+ expect(calls.map(([, ids]) => ids.length)).toEqual([50, 50, 20]);
+ expect(calls.flatMap(([, ids]) => ids)).toEqual(Array.from({ length: 120 }, (_, i) => `d${i}`));
+ const csv = await readText((saveBlob.mock.calls[0] as [Blob, string])[0]);
+ // Every draft once: a row starts with its title (a page's text spans lines).
+ expect(csv.match(/^Draft \d+,/gm)).toHaveLength(120);
+ });
+
+ it('goes on from the ids a call left for its size, before the next slice', async () => {
+ getContentDrafts.mockResolvedValue(
+ Array.from({ length: 60 }, (_, i) => row({ id: `d${i}`, title: `Draft ${i}` })),
+ );
+ const page = (id: string) => ({
+ id,
+ title: 'Pipeline forecasting',
+ bodyHtml: '<p>x</p>',
+ sourceUrl: null,
+ });
+ // The first call reads 30 of its 50 and leaves 20.
+ getContentDraftPages.mockImplementationOnce(async (_brand: string, ids: string[]) => ({
+ pages: ids.slice(0, 30).map(page),
+ rest: ids.slice(30),
+ }));
+ await renderPage();
+ const items = await openExport();
+ await act(async () => items[0].click());
+ await act(async () => {});
+
+ const calls = getContentDraftPages.mock.calls as [string, string[]][];
+ expect(calls.map(([, ids]) => [ids[0], ids.length])).toEqual([
+ ['d0', 50],
+ ['d30', 30],
+ ]);
+ const csv = await readText((saveBlob.mock.calls[0] as [Blob, string])[0]);
+ expect(csv.match(/^Draft \d+,/gm)).toHaveLength(60);
+ });
+
it('exports the drafts listed, with their pages, as CSV', async () => {
getContentDrafts.mockResolvedValue([
row({}),
diff --git a/web/src/lib/actions/content-drafts-pages.test.ts b/web/src/lib/actions/content-drafts-pages.test.ts
new file mode 100644
index 000000000..3144c9c4b
--- /dev/null
+++ b/web/src/lib/actions/content-drafts-pages.test.ts
@@ -0,0 +1,87 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+/**
+ * The Library's Export (CE-002): each draft's page, of the brand, in the
+ * order given, at most fifty a call since every page comes back whole.
+ */
+
+const BRAND = '33333333-3333-3333-3333-333333333333';
+const id = (n: number) => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}`;
+
+let stored: Record<string, unknown>[];
+let reads: { brand: unknown; ids: string[] }[];
+
+vi.mock('@/lib/action-center/assignees', () => ({ readAssignees: vi.fn(async () => new Map()) }));
+vi.mock('@/lib/supabase/admin', () => ({ supabaseAdmin: {} }));
+vi.mock('@/lib/supabase/server', () => ({
+ createClient: vi.fn(async () => ({
+ from: () => {
+ const read = { brand: undefined as unknown, ids: [] as string[] };
+ const chain: Record<string, unknown> = {
+ select: () => chain,
+ eq: (col: string, v: unknown) => (col === 'brand_id' && (read.brand = v), chain),
+ in: (_col: string, ids: string[]) => {
+ read.ids = ids;
+ reads.push(read);
+ return chain;
+ },
+ then: (resolve: (r: unknown) => void) =>
+ resolve({
+ data: stored.filter((row) => read.ids.includes(row.id as string)),
+ error: null,
+ }),
+ };
+ return chain;
+ },
+ })),
+}));
+
+const { getContentDraftPages } = await import('./content-drafts');
+
+const page = (n: number) => ({
+ id: id(n),
+ title: `Draft ${n}`,
+ body_html: `<h1>Draft ${n}</h1>`,
+ source_url: null,
+});
+
+beforeEach(() => {
+ stored = [page(1), page(2), page(3)];
+ reads = [];
+});
+
+describe('getContentDraftPages', () => {
+ it("reads the brand's pages in one call, in the order given, leaving out one gone", async () => {
+ const { pages, rest } = await getContentDraftPages(BRAND, [id(3), id(9), id(1)]);
+ expect(rest).toEqual([]);
+ expect(pages.map((p) => p.title)).toEqual(['Draft 3', 'Draft 1']);
+ expect(pages[0]).toEqual({
+ id: id(3),
+ title: 'Draft 3',
+ bodyHtml: '<h1>Draft 3</h1>',
+ sourceUrl: null,
+ });
+ expect(reads).toEqual([{ brand: BRAND, ids: [id(3), id(9), id(1)] }]);
+ });
+
+ it('takes at most fifty ids a call, and only ids', async () => {
+ const fifty = Array.from({ length: 50 }, (_, i) => id(i + 1));
+ expect((await getContentDraftPages(BRAND, fifty)).pages).toHaveLength(3);
+ await expect(getContentDraftPages(BRAND, [...fifty, id(51)])).rejects.toThrow('invalid');
+ await expect(getContentDraftPages(BRAND, ['d1'])).rejects.toThrow('invalid');
+ await expect(getContentDraftPages('brand-1', [id(1)])).rejects.toThrow('invalid');
+ expect(reads).toHaveLength(1);
+ });
+
+ it('stops at its response budget and hands back the rest, always one page at least', async () => {
+ const big = (n: number) => ({ ...page(n), body_html: 'x'.repeat(1_200_000) });
+ stored = [big(1), big(2), big(3), page(4)];
+ // 2.4 MB fits; a third 1.2 MB page would pass the 3 MB budget.
+ const first = await getContentDraftPages(BRAND, [id(1), id(2), id(3), id(4)]);
+ expect(first.pages.map((p) => p.id)).toEqual([id(1), id(2)]);
+ expect(first.rest).toEqual([id(3), id(4)]);
+ // One page over the budget on its own still comes back.
+ stored = [{ ...page(5), body_html: 'x'.repeat(3_500_000) }];
+ expect((await getContentDraftPages(BRAND, [id(5)])).pages).toHaveLength(1);
+ });
+});
diff --git a/web/src/lib/actions/content-drafts.ts b/web/src/lib/actions/content-drafts.ts
index fee72ccf2..952360c28 100644
--- a/web/src/lib/actions/content-drafts.ts
+++ b/web/src/lib/actions/content-drafts.ts
@@ -5,6 +5,7 @@ import { API_BASE_URL } from '@/config/api';
import { createClient } from '@/lib/supabase/server';
import { readAssignees } from '@/lib/action-center/assignees';
import {
+ DRAFT_PAGES_PER_CALL,
DRAFT_STATUSES,
LIBRARY_TABS,
PAGE_TYPES,
@@ -249,44 +250,58 @@ export interface DraftPage {
sourceUrl: string | null;
}
-/** Ids read at a time: the list goes in the request's address. */
-const PAGES_PER_READ = 100;
+/**
+ * A call's pages stop at about this many characters, well inside a
+ * serverless response's 4.5 MB: fifty pages can each be up to 500 KB.
+ */
+const PAGES_RESPONSE_BUDGET = 3_000_000;
+
+/** What one Export call returns: the pages, and the ids it left for the next. */
+export interface DraftPagesRead {
+ pages: DraftPage[];
+ /** Ids not read for the response's size, in order; empty when all were. */
+ rest: string[];
+}
/**
* The Library's Export (CE-002): the stored page of each draft given, of the
- * brand, in the order given; at most the Library's own count. A draft gone
- * meanwhile is left out.
+ * brand, in the order given. At most DRAFT_PAGES_PER_CALL a call, and pages
+ * up to PAGES_RESPONSE_BUDGET, since every page comes back in the response:
+ * the Library reads a longer list in calls, starting each with `rest`. Always
+ * at least one page, so a list moves on. A draft gone meanwhile is left out.
*/
-export async function getContentDraftPages(brandId: string, ids: string[]): Promise<DraftPage[]> {
- const parsed = z.array(z.guid()).max(LIBRARY_LIMIT).safeParse(ids);
+export async function getContentDraftPages(
+ brandId: string,
+ ids: string[],
+): Promise<DraftPagesRead> {
+ const parsed = z.array(z.guid()).max(DRAFT_PAGES_PER_CALL).safeParse(ids);
if (!draftId.safeParse(brandId).success || !parsed.success) throw new Error('invalid');
const supabase = await createClient();
- const chunks = [];
- for (let i = 0; i < parsed.data.length; i += PAGES_PER_READ) {
- chunks.push(parsed.data.slice(i, i + PAGES_PER_READ));
- }
- const reads = await Promise.all(
- chunks.map((chunk) =>
- supabase
- .from('content_drafts')
- .select('id, title, body_html, source_url')
- .eq('brand_id', brandId)
- .in('id', chunk),
- ),
+ const { data, error } = await supabase
+ .from('content_drafts')
+ .select('id, title, body_html, source_url')
+ .eq('brand_id', brandId)
+ .in('id', parsed.data);
+ if (error) throw new Error(error.message);
+ const byId = new Map<string, DraftPage>(
+ (data ?? []).map((row) => [
+ row.id,
+ { id: row.id, title: row.title, bodyHtml: row.body_html, sourceUrl: row.source_url },
+ ]),
);
- const byId = new Map<string, DraftPage>();
- for (const { data, error } of reads) {
- if (error) throw new Error(error.message);
- for (const row of data ?? []) {
- byId.set(row.id, {
- id: row.id,
- title: row.title,
- bodyHtml: row.body_html,
- sourceUrl: row.source_url,
- });
+ const pages: DraftPage[] = [];
+ let size = 0;
+ for (const [i, id] of parsed.data.entries()) {
+ const page = byId.get(id);
+ if (!page) continue;
+ const pageSize = page.bodyHtml.length + page.title.length + (page.sourceUrl?.length ?? 0);
+ if (pages.length > 0 && size + pageSize > PAGES_RESPONSE_BUDGET) {
+ return { pages, rest: parsed.data.slice(i) };
}
+ pages.push(page);
+ size += pageSize;
}
- return parsed.data.flatMap((id) => byId.get(id) ?? []);
+ return { pages, rest: [] };
}
/** Opportunities still open to write for, the highest scored first. */
diff --git a/web/src/lib/content/draft-export.test.ts b/web/src/lib/content/draft-export.test.ts
index f449a2716..a779e72d6 100644
--- a/web/src/lib/content/draft-export.test.ts
+++ b/web/src/lib/content/draft-export.test.ts
@@ -228,11 +228,21 @@ describe('the Library’s Export', () => {
it('cuts a page past a spreadsheet cell’s limit', () => {
const long = `<h1>T</h1><p>${'a'.repeat(40_000)}</p>`;
const [{ data }] = libraryWorkbook([item({ html: long })]);
- const content = data[1][9] as string;
+ const content = (data[1][9] as { value: string }).value;
expect(content).toHaveLength(32_767);
expect(content.endsWith('…')).toBe(true);
});
+ it('wraps the page text, a block a line, so Excel keeps its lines apart', () => {
+ const [{ data }] = libraryWorkbook([item()]);
+ expect(data[1][9]).toEqual({
+ value: 'Pipeline forecasting\nTeams trust it.\n1. Import\n2. Stage',
+ wrap: true,
+ });
+ // A one-line value stays a plain cell.
+ expect(typeof data[1][0]).toBe('string');
+ });
+
it('puts every draft in one Word document, each on its own page', async () => {
const files = unzipSync(
await read(
diff --git a/web/src/lib/content/draft-export.ts b/web/src/lib/content/draft-export.ts
index f4c93b079..388b6ecdf 100644
--- a/web/src/lib/content/draft-export.ts
+++ b/web/src/lib/content/draft-export.ts
@@ -496,9 +496,10 @@ export function libraryWorkbook(items: LibraryExportItem[]): Sheet<Blob>[] {
...libraryRows(items).map((row) =>
LIBRARY_EXPORT_HEADERS.map((header) => {
const value = row[header];
- return typeof value === 'string' && value.length > CELL_LIMIT
- ? `${value.slice(0, CELL_LIMIT - 1)}…`
- : value;
+ if (typeof value !== 'string') return value;
+ const cell = value.length > CELL_LIMIT ? `${value.slice(0, CELL_LIMIT - 1)}…` : value;
+ // A page is a block a line: wrapped, so Excel doesn't run them together.
+ return cell.includes('\n') ? { value: cell, wrap: true } : cell;
}),
),
],
diff --git a/web/src/lib/content/drafts.ts b/web/src/lib/content/drafts.ts
index a7fdd587e..7d9fb20f3 100644
--- a/web/src/lib/content/drafts.ts
+++ b/web/src/lib/content/drafts.ts
@@ -86,6 +86,13 @@ export function contentEditorPath(draftId: string) {
export const LIBRARY_TABS = ['all', ...DRAFT_STATUSES, 'archived'] as const;
export type LibraryTab = (typeof LIBRARY_TABS)[number];
+/**
+ * Drafts' pages read per Export call: each comes back with its whole page
+ * (about 30 KB on average, up to 500 KB), so a list is read in calls of this
+ * many rather than in one response.
+ */
+export const DRAFT_PAGES_PER_CALL = 50;
+
export interface DraftAuthor {
id: string;
fullName: string | null;