Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

feat(ui): select and copy text in the phone PDF reader (#3020)

Call Whitee90b0e9f7c1c
diff --git a/ui/e2e/pdf-preview/pdf.ts b/ui/e2e/pdf-preview/pdf.ts
index 940977c1a..4a2f674f2 100644
--- a/ui/e2e/pdf-preview/pdf.ts
+++ b/ui/e2e/pdf-preview/pdf.ts
@@ -5,6 +5,9 @@
// rendered canvas to prove *which* page was drawn) and a "Page N" label. ``cjk`` adds a line set in
// a non-embedded Adobe-GB1 font through the predefined ``UniGB-UCS2-H`` CMap — the shape of PDFs
// many generators emit for Chinese text, and one pdf.js can only decode with its bundled CMaps.
+// ``radicals`` adds a "月份" line whose ToUnicode maps 月 to the Kangxi radical ⽉ (U+2F49), as the
+// fonts Chrome embeds do: it looks right, and copies as a different character unless unified.
+// ``rotate`` sets every page's /Rotate, as scanners and some exporters do for landscape pages.
export type Rgb = readonly [number, number, number];
@@ -27,7 +30,7 @@ export function pageColor(index: number): Rgb {
const unit = (v: number) => (v / 255).toFixed(3);
-export function buildPdf({ pages, cjk = false }: { pages: number; cjk?: boolean }): Buffer {
+export function buildPdf({ pages, cjk = false, radicals = false, rotate = 0 }: { pages: number; cjk?: boolean; radicals?: boolean; rotate?: 0 | 90 | 180 | 270 }): Buffer {
const objects: string[] = [];
const add = (body: string) => {
objects.push(body);
@@ -45,7 +48,19 @@ export function buildPdf({ pages, cjk = false }: { pages: number; cjk?: boolean
const cidFont = add(
`<< /Type /Font /Subtype /CIDFontType0 /BaseFont /STSong-Light /CIDSystemInfo << /Registry (Adobe) /Ordering (GB1) /Supplement 4 >> /FontDescriptor ${descriptor} 0 R /DW 1000 >>`,
);
- song = add(`<< /Type /Font /Subtype /Type0 /BaseFont /STSong-Light-UniGB-UCS2-H /Encoding /UniGB-UCS2-H /DescendantFonts [${cidFont} 0 R] >>`);
+ let toUnicode = '';
+ if (radicals) {
+ const cmap = [
+ '/CIDInit /ProcSet findresource begin 12 dict begin begincmap',
+ '/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def',
+ '/CMapName /Adobe-Identity-UCS def /CMapType 2 def',
+ '1 begincodespacerange <0000> <FFFF> endcodespacerange',
+ '6 beginbfchar <4E2D> <4E2D> <6587> <6587> <9884> <9884> <89C8> <89C8> <6708> <2F49> <4EFD> <4EFD> endbfchar',
+ 'endcmap CMapName currentdict /CMap defineresource pop end end',
+ ].join('\n');
+ toUnicode = ` /ToUnicode ${add(`<< /Length ${Buffer.byteLength(cmap)} >>\nstream\n${cmap}\nendstream`)} 0 R`;
+ }
+ song = add(`<< /Type /Font /Subtype /Type0 /BaseFont /STSong-Light-UniGB-UCS2-H /Encoding /UniGB-UCS2-H /DescendantFonts [${cidFont} 0 R]${toUnicode} >>`);
}
const kids: number[] = [];
@@ -56,12 +71,14 @@ export function buildPdf({ pages, cjk = false }: { pages: number; cjk?: boolean
`0 0 0 rg BT /F1 48 Tf 72 680 Td (Page ${i + 1}) Tj ET`,
];
// "中文预览" as UCS-2 code units.
- if (cjk) ops.push('0 0 0 rg BT /F2 40 Tf 72 620 Td <4E2D6587987889C8> Tj ET');
+ if (cjk) ops.push('0 0 0 rg BT /F2 40 Tf 72 620 Td <4E2D6587988489C8> Tj ET');
+ // "月份", in the band's upper edge.
+ if (radicals) ops.push('0 0 0 rg BT /F2 32 Tf 72 575 Td <67084EFD> Tj ET');
const stream = ops.join('\n');
const content = add(`<< /Length ${Buffer.byteLength(stream)} >>\nstream\n${stream}\nendstream`);
const fonts = `/F1 ${helvetica} 0 R${cjk ? ` /F2 ${song} 0 R` : ''}`;
kids.push(
- add(`<< /Type /Page /Parent ${pagesNode} 0 R /MediaBox [0 0 ${PAGE_WIDTH} ${PAGE_HEIGHT}] /Contents ${content} 0 R /Resources << /Font << ${fonts} >> >> >>`),
+ add(`<< /Type /Page /Parent ${pagesNode} 0 R /MediaBox [0 0 ${PAGE_WIDTH} ${PAGE_HEIGHT}]${rotate ? ` /Rotate ${rotate}` : ''} /Contents ${content} 0 R /Resources << /Font << ${fonts} >> >> >>`),
);
}
objects[catalog - 1] = `<< /Type /Catalog /Pages ${pagesNode} 0 R >>`;
diff --git a/ui/e2e/pdf-preview/preview.spec.ts b/ui/e2e/pdf-preview/preview.spec.ts
index f666ed549..cb65eea9f 100644
--- a/ui/e2e/pdf-preview/preview.spec.ts
+++ b/ui/e2e/pdf-preview/preview.spec.ts
@@ -18,6 +18,10 @@ const DOCS: Record<string, Buffer> = {
doc_five: buildPdf({ pages: 5 }),
doc_long: buildPdf({ pages: 14 }),
doc_cjk: buildPdf({ pages: 2, cjk: true }),
+ doc_radicals: buildPdf({ pages: 1, cjk: true, radicals: true }),
+ doc_rot90: buildPdf({ pages: 1, rotate: 90 }),
+ doc_rot180: buildPdf({ pages: 1, rotate: 180 }),
+ doc_rot270: buildPdf({ pages: 1, rotate: 270 }),
doc_broken: Buffer.from('%PDF-1.4\nthis is not a PDF document\n'),
};
@@ -69,6 +73,54 @@ const expectDrawn = async (dialog: Locator, n: number) => {
expect(close(r, er) && close(g, eg) && close(b, eb), `page ${n} band is rgb(${r}, ${g}, ${b}), expected rgb(${er}, ${eg}, ${eb})`).toBe(true);
};
+// The text layer's span for ``text`` on page ``n``, measured against the page's own drawing: the
+// share of dark pixels under the span, and beside it on the same line, on whichever side has room
+// (blank in every fixture). Text that is selectable but not where it is drawn would highlight the
+// wrong words.
+const textOverInk = (dialog: Locator, n: number, text: string) =>
+ pageBox(dialog, n).evaluate((box, wanted) => {
+ const span = Array.from(box.querySelectorAll('.vr-pdf-text > span')).find((el) => el.textContent === wanted);
+ const canvas = box.querySelector('canvas');
+ if (!span || !canvas) return null;
+ const page = box.getBoundingClientRect();
+ const rect = span.getBoundingClientRect();
+ const sx = canvas.width / page.width;
+ const sy = canvas.height / page.height;
+ const ctx = canvas.getContext('2d')!;
+ const ink = (x: number, y: number, w: number, h: number) => {
+ const data = ctx.getImageData(Math.round((x - page.left) * sx), Math.round((y - page.top) * sy), Math.max(1, Math.round(w * sx)), Math.max(1, Math.round(h * sy))).data;
+ let dark = 0;
+ for (let i = 0; i < data.length; i += 4) if (data[i] < 100 && data[i + 1] < 100 && data[i + 2] < 100 && data[i + 3] > 128) dark += 1;
+ return dark / (data.length / 4);
+ };
+ const besideX = rect.right + 4 + rect.width <= page.right ? rect.right + 4 : rect.left - 4 - rect.width;
+ return { over: ink(rect.left, rect.top, rect.width, rect.height), beside: ink(besideX, rect.top, rect.width, rect.height) };
+ }, text);
+
+const expectTextOverInk = async (dialog: Locator, n: number, text: string) => {
+ await expect(pageBox(dialog, n)).toHaveAttribute('data-pdf-text', 'true');
+ const ink = await textOverInk(dialog, n, text);
+ expect(ink, `page ${n} has a text span "${text}"`).not.toBeNull();
+ expect(ink!.over, `ink under "${text}"`).toBeGreaterThan(0.08);
+ expect(ink!.beside, `ink beside "${text}"`).toBeLessThan(0.01);
+};
+
+const spanBox = async (dialog: Locator, n: number, text: string) =>
+ (await pageBox(dialog, n).locator('.vr-pdf-text > span').filter({ hasText: text }).first().boundingBox())!;
+
+const selectionText = (page: Page) => page.evaluate(() => document.getSelection()?.toString() ?? '');
+
+// Press on the start of the first line and drag to a point, as a mouse or a dragged selection handle
+// does; reports the selection while the pointer is still down.
+const dragSelect = async (page: Page, from: { x: number; y: number }, to: { x: number; y: number }) => {
+ await page.mouse.move(from.x, from.y);
+ await page.mouse.down();
+ for (let i = 1; i <= 12; i += 1) await page.mouse.move(from.x + ((to.x - from.x) * i) / 12, from.y + ((to.y - from.y) * i) / 12);
+ const selected = await selectionText(page);
+ await page.mouse.up();
+ return selected;
+};
+
const scrollToPage = async (dialog: Locator, n: number) => {
await scroller(dialog).evaluate((el, top) => el.scrollTo({ top }), await pageBox(dialog, n).evaluate((p) => (p as HTMLElement).offsetTop));
};
@@ -116,6 +168,7 @@ test.describe('phones and tablets', () => {
await scroller(dialog).evaluate((el) => el.scrollTo({ top: el.scrollHeight }));
await expectDrawn(dialog, 14);
await expect(pageBox(dialog, 1).locator('canvas')).toHaveCount(0);
+ await expect(pageBox(dialog, 1).locator('.vr-pdf-text')).toHaveCount(0);
expect(await dialog.locator('[data-pdf-page] canvas').count()).toBeLessThan(14);
});
@@ -139,6 +192,16 @@ test.describe('phones and tablets', () => {
expect(centreY).toBeLessThan(third.y + third.height);
await expect(position(dialog)).toHaveText('3 / 5');
await expectDrawn(dialog, 3);
+ // Once the page is redrawn at the new size, its selectable text is still over the drawing.
+ await expect
+ .poll(() =>
+ pageBox(dialog, 3).evaluate((box) => {
+ const canvas = box.querySelector('canvas');
+ return !!canvas && canvas.width >= Math.floor(box.getBoundingClientRect().width * window.devicePixelRatio) - 1;
+ }),
+ )
+ .toBe(true);
+ await expectTextOverInk(dialog, 3, 'Page 3');
await dialog.getByRole('button', { name: 'Reset zoom' }).click();
await expect(dialog.getByText('100%')).toBeVisible();
@@ -180,6 +243,75 @@ test.describe('phones and tablets', () => {
const dialog = await openPreview(page, 'doc_cjk');
await expectDrawn(dialog, 1);
expect(cmapRequests.length).toBeGreaterThan(0);
+ // And it is selectable as the characters it shows, so a copy gives back real Chinese text.
+ await expect(pageBox(dialog, 1).locator('.vr-pdf-text')).toContainText('中文预览');
+ await expectTextOverInk(dialog, 1, '中文预览');
+ });
+
+ test('characters a font maps to look-alike radicals are selected as the characters shown', async ({ page }) => {
+ const dialog = await openPreview(page, 'doc_radicals');
+ await expectTextOverInk(dialog, 1, '月份');
+ await expect(pageBox(dialog, 1).locator('.vr-pdf-text')).not.toContainText('\u2F49');
+ });
+
+ test('on a page with /Rotate the text is turned with the drawing', async ({ page }) => {
+ for (const file of ['doc_rot90', 'doc_rot180', 'doc_rot270']) {
+ const dialog = await openPreview(page, file);
+ await expectTextOverInk(dialog, 1, 'Page 1');
+ await dialog.getByRole('button', { name: 'Close' }).click();
+ await expect(dialog).toHaveCount(0);
+ }
+ });
+
+ test("each page's text lies over its drawing and is what a touch on it reaches", async ({ page }) => {
+ const dialog = await openPreview(page, 'doc_five');
+ await expectTextOverInk(dialog, 1, 'Page 1');
+ const box = await spanBox(dialog, 1, 'Page 1');
+ const reached = await page.evaluate(({ x, y }) => document.elementFromPoint(x, y)?.textContent, { x: box.x + box.width / 4, y: box.y + box.height / 2 });
+ expect(reached).toBe('Page 1');
+
+ await scrollToPage(dialog, 4);
+ await expectTextOverInk(dialog, 4, 'Page 4');
+ });
+
+ // Long-press word selection is the browser's own gesture (and only some platforms' Chromium
+ // performs it under automation), so what is held here is the reader's side of it: a one-finger
+ // press on the text reaches the browser uncancelled. The pinch handler must stay two-finger only.
+ test('a one-finger press on the text is left to the browser, so its long press can select', async ({ page, browserName }) => {
+ test.skip(browserName !== 'chromium', 'touch input is driven through the Chromium DevTools protocol');
+ const dialog = await openPreview(page, 'doc_five');
+ await expectTextOverInk(dialog, 1, 'Page 1');
+ const box = await spanBox(dialog, 1, 'Page 1');
+ await page.evaluate(() => {
+ const seen: string[] = [];
+ Object.assign(window, { __touches: seen });
+ for (const type of ['touchstart', 'touchend']) {
+ window.addEventListener(type, (event) => seen.push(`${type}:${(event.target as Element).textContent}:${event.defaultPrevented}`), { passive: true });
+ }
+ });
+ const cdp = await page.context().newCDPSession(page);
+ const finger = { x: box.x + box.width / 5, y: box.y + box.height / 2, id: 1 };
+ await cdp.send('Input.dispatchTouchEvent', { type: 'touchStart', touchPoints: [finger] });
+ await page.waitForTimeout(700);
+ await cdp.send('Input.dispatchTouchEvent', { type: 'touchEnd', touchPoints: [] });
+ expect(await page.evaluate(() => (window as unknown as { __touches: string[] }).__touches)).toEqual([
+ 'touchstart:Page 1:false',
+ 'touchend:Page 1:false',
+ ]);
+ });
+
+ test('extending a selection into blank space keeps it on the text', async ({ page }) => {
+ const dialog = await openPreview(page, 'doc_cjk');
+ await expectTextOverInk(dialog, 1, 'Page 1');
+ const first = await spanBox(dialog, 1, 'Page 1');
+ const sheet = (await pageBox(dialog, 1).boundingBox())!;
+ const start = { x: first.x + 1, y: first.y + first.height / 2 };
+ // Past the end of the line, in the margin: WebKit would otherwise snap the selection's end to the
+ // start of the page and select nothing.
+ expect(await dragSelect(page, start, { x: sheet.x + sheet.width - 20, y: start.y })).toMatch(/^Page/);
+ await page.evaluate(() => document.getSelection()?.removeAllRanges());
+ // Below both lines of text, above the coloured band.
+ expect(await dragSelect(page, start, { x: sheet.x + sheet.width / 2, y: sheet.y + sheet.height * 0.26 })).toMatch(/^Page 1/);
});
test('a file that is not a valid PDF says so instead of drawing blank pages', async ({ page }) => {
@@ -211,4 +343,16 @@ test.describe('desktop browsers', () => {
await expect(dialog.locator('[data-pdf-page]')).toHaveCount(5);
await expectDrawn(dialog, 1);
});
+
+ test("the page reader's text can be selected with the mouse and copied", async ({ page, context }) => {
+ await reportViewer(page, false);
+ await context.grantPermissions(['clipboard-read', 'clipboard-write']);
+ const dialog = await openPreview(page, 'doc_five');
+ await expectTextOverInk(dialog, 1, 'Page 1');
+ const box = await spanBox(dialog, 1, 'Page 1');
+ const selected = await dragSelect(page, { x: box.x + 1, y: box.y + box.height / 2 }, { x: box.x + box.width - 1, y: box.y + box.height / 2 });
+ expect(selected).toBe('Page 1');
+ await page.keyboard.press('ControlOrMeta+C');
+ await expect.poll(() => page.evaluate(() => navigator.clipboard.readText())).toBe('Page 1');
+ });
});
diff --git a/ui/src/components/ui/pdf-pages.tsx b/ui/src/components/ui/pdf-pages.tsx
index da326e29e..04b907c39 100644
--- a/ui/src/components/ui/pdf-pages.tsx
+++ b/ui/src/components/ui/pdf-pages.tsx
@@ -2,11 +2,13 @@ import * as React from 'react';
import { useTranslation } from 'react-i18next';
import { Loader2, Maximize2, ZoomIn, ZoomOut } from 'lucide-react';
import clsx from 'clsx';
-import type { PDFDocumentLoadingTask, PDFDocumentProxy, PDFWorker, RenderTask } from 'pdfjs-dist/legacy/build/pdf.mjs';
+import type { PDFDocumentLoadingTask, PDFDocumentProxy, PDFWorker, RenderTask, TextLayer } from 'pdfjs-dist/legacy/build/pdf.mjs';
import PdfjsWorker from 'pdfjs-dist/legacy/build/pdf.worker.mjs?worker';
import { Button } from '@/components/ui/button';
import { PDF_MAX_ZOOM, PDF_MIN_ZOOM, clampPdfZoom, pageIndexAt, pdfOutputScale, stepPdfZoom } from '@/lib/pdfPreview';
+import { unifyCjkRadicals } from '@/lib/cjkRadicals';
+import { PDF_TEXT_LAYER_CLASS, trackPdfTextLayer } from '@/lib/pdfTextSelection';
// ── PDF pages ────────────────────────────────────────────────────────────────
// Every page of a PDF drawn to its own canvas by pdf.js, stacked in one scroller: the reader used
@@ -18,6 +20,9 @@ import { PDF_MAX_ZOOM, PDF_MIN_ZOOM, clampPdfZoom, pageIndexAt, pdfOutputScale,
// keeps the point under the fingers / centre in place.
// - Only pages within a screen and a half of the viewport hold a canvas. Others are released, which
// is what keeps a long document inside iOS's canvas memory budget.
+// - Each drawn page carries pdf.js's text layer: transparent text laid exactly over the drawing, so
+// the text can be long-pressed or dragged to select and copied (styles: ``.vr-pdf-text`` in
+// index.css; selection behaviour: lib/pdfTextSelection.ts).
// - The legacy build is used on purpose: the modern one relies on JavaScript features newer than the
// mobile Safari versions people still run.
@@ -86,7 +91,9 @@ class PageWatch {
}
}
-type DocState = { status: 'loading' } | { status: 'error' } | { status: 'ready'; doc: PDFDocumentProxy; ratio: number };
+type TextLayerClass = typeof TextLayer;
+
+type DocState = { status: 'loading' } | { status: 'error' } | { status: 'ready'; doc: PDFDocumentProxy; ratio: number; TextLayer: TextLayerClass };
type Anchor = { page: number; fx: number; fy: number; ax: number; ay: number };
@@ -103,9 +110,11 @@ export default function PdfPages({ url, className }: { url: string; className?:
let port: Worker | null = null;
let worker: PDFWorker | null = null;
let task: PDFDocumentLoadingTask | null = null;
+ let textLayers: TextLayerClass | null = null;
(async () => {
try {
const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs');
+ textLayers = pdfjs.TextLayer;
if (!alive) return;
// A worker per document, terminated with it, so closing the preview gives its memory back.
// (pdf.js's generated typings declare ``port`` as null-only; it takes a Worker at runtime.)
@@ -122,7 +131,7 @@ export default function PdfPages({ url, className }: { url: string; className?:
const doc = await task.promise;
const first = await doc.getPage(1);
const viewport = first.getViewport({ scale: 1 });
- if (alive) setState({ status: 'ready', doc, ratio: viewport.height / viewport.width });
+ if (alive) setState({ status: 'ready', doc, ratio: viewport.height / viewport.width, TextLayer: pdfjs.TextLayer });
} catch {
if (alive) setState({ status: 'error' });
}
@@ -135,6 +144,8 @@ export default function PdfPages({ url, className }: { url: string; className?:
.finally(() => {
worker?.destroy();
port?.terminate();
+ // Drops the hidden measuring canvas pdf.js keeps in <body> once no text layer is pending.
+ textLayers?.cleanup();
});
};
}, [url]);
@@ -149,10 +160,10 @@ export default function PdfPages({ url, className }: { url: string; className?:
</div>
);
}
- return <PdfReader doc={state.doc} ratio={state.ratio} className={className} />;
+ return <PdfReader doc={state.doc} ratio={state.ratio} TextLayer={state.TextLayer} className={className} />;
}
-function PdfReader({ doc, ratio, className }: { doc: PDFDocumentProxy; ratio: number; className?: string }) {
+function PdfReader({ doc, ratio, TextLayer, className }: { doc: PDFDocumentProxy; ratio: number; TextLayer: TextLayerClass; className?: string }) {
const { t } = useTranslation();
const [scroller, setScroller] = React.useState<HTMLDivElement | null>(null);
const contentRef = React.useRef<HTMLDivElement>(null);
@@ -314,7 +325,9 @@ function PdfReader({ doc, ratio, className }: { doc: PDFDocumentProxy; ratio: nu
{/* ``relative``: the pages' offsets are measured against this column, the scroller's content. */}
<div ref={contentRef} className="relative flex w-max min-w-full flex-col items-center gap-3 px-3 pb-16 pt-3">
{pageWidth > 0 &&
- pages.map((n) => <PdfPage key={n} doc={doc} pageNumber={n} width={pageWidth} fallbackRatio={ratio} watch={watch} label={t('preview.pdfPage', { page: n })} />)}
+ pages.map((n) => (
+ <PdfPage key={n} doc={doc} TextLayer={TextLayer} pageNumber={n} width={pageWidth} fallbackRatio={ratio} watch={watch} label={t('preview.pdfPage', { page: n })} />
+ ))}
</div>
</div>
{/* Control cluster, as in the image preview; the bottom padding of the column keeps the last
@@ -344,8 +357,20 @@ function PdfReader({ doc, ratio, className }: { doc: PDFDocumentProxy; ratio: nu
);
}
+// What a page box shows: its canvas and text layer, swapped in together. The previous text layer
+// stops being tracked for selection when it leaves the document.
+const trackedLayers = new WeakMap<HTMLElement, () => void>();
+
+function showPage(box: HTMLElement, canvas: HTMLCanvasElement | null, layer: HTMLElement | null) {
+ trackedLayers.get(box)?.();
+ trackedLayers.delete(box);
+ box.replaceChildren(...[canvas, layer].filter((node): node is HTMLCanvasElement | HTMLElement => node !== null));
+ if (layer) trackedLayers.set(box, trackPdfTextLayer(layer));
+}
+
const PdfPage = React.memo(function PdfPage({
doc,
+ TextLayer,
pageNumber,
width,
fallbackRatio,
@@ -353,6 +378,7 @@ const PdfPage = React.memo(function PdfPage({
label,
}: {
doc: PDFDocumentProxy;
+ TextLayer: TextLayerClass;
pageNumber: number;
width: number;
fallbackRatio: number;
@@ -367,53 +393,83 @@ const PdfPage = React.memo(function PdfPage({
React.useEffect(() => {
const box = boxRef.current;
if (!box) return;
- return watch.observe(box, setNear);
+ const stop = watch.observe(box, setNear);
+ return () => {
+ stop();
+ showPage(box, null, null);
+ };
}, [watch]);
- // The canvas lives outside React's tree (the box has no React children), so a re-render at a new
- // width keeps showing the previous bitmap, stretched, until the sharper one replaces it.
+ // The canvas and text layer live outside React's tree (the box has no React children), so a
+ // re-render at a new width keeps showing the previous ones, stretched, until the new pair replaces
+ // them together.
React.useEffect(() => {
const box = boxRef.current;
if (!box) return;
if (!near) {
- box.replaceChildren();
+ showPage(box, null, null);
delete box.dataset.pdfRendered;
+ delete box.dataset.pdfText;
return;
}
let cancelled = false;
let task: RenderTask | null = null;
+ let text: TextLayer | null = null;
(async () => {
const page = await doc.getPage(pageNumber);
if (cancelled) return;
const base = page.getViewport({ scale: 1 });
const pageRatio = base.height / base.width;
setRatio(pageRatio);
+ const cssScale = width / base.width;
+ // The text layer sizes itself and its text from this variable. Set before drawing, so a layer
+ // still showing from the previous width already follows the resized box.
+ box.style.setProperty('--total-scale-factor', String(cssScale));
const outputScale = pdfOutputScale(width, width * pageRatio, window.devicePixelRatio);
- const viewport = page.getViewport({ scale: (width / base.width) * outputScale });
+ const viewport = page.getViewport({ scale: cssScale * outputScale });
const canvas = document.createElement('canvas');
canvas.width = Math.max(1, Math.floor(viewport.width));
canvas.height = Math.max(1, Math.floor(viewport.height));
canvas.className = 'absolute inset-0 h-full w-full';
task = page.render({ canvas, viewport });
+ // Text is laid out in CSS pixels, in parallel with the drawing. A page whose text cannot be
+ // extracted is still shown, just without selectable text. Radical look-alikes are unified
+ // first, so what is selected copies as the characters on screen.
+ const layer = document.createElement('div');
+ layer.className = PDF_TEXT_LAYER_CLASS;
+ const textReady = page
+ .getTextContent()
+ .then((content) => {
+ if (cancelled) return false;
+ for (const item of content.items) if ('str' in item) item.str = unifyCjkRadicals(item.str);
+ text = new TextLayer({ textContentSource: content, container: layer, viewport: page.getViewport({ scale: cssScale }) });
+ return text.render().then(() => true);
+ })
+ .catch(() => false);
await task.promise;
+ const hasText = await textReady;
if (cancelled) return;
- box.replaceChildren(canvas);
+ showPage(box, canvas, hasText ? layer : null);
box.dataset.pdfRendered = 'true';
+ if (hasText) box.dataset.pdfText = 'true';
+ else delete box.dataset.pdfText;
})().catch(() => undefined); // cancelled renders reject; a page that fails to draw stays blank
return () => {
cancelled = true;
task?.cancel();
+ text?.cancel();
};
- }, [doc, pageNumber, width, near]);
+ }, [doc, TextLayer, pageNumber, width, near]);
+ // A group, not an image: the page's text is readable by assistive technology through its layer.
return (
<div
ref={boxRef}
- role="img"
+ role="group"
aria-label={label}
data-pdf-page={pageNumber}
className="relative shrink-0 bg-white shadow-[0_1px_4px_rgba(0,0,0,0.5)]"
- style={{ width, height: Math.round(width * (ratio ?? fallbackRatio)) }}
+ style={{ width, height: Math.round(width * (ratio ?? fallbackRatio)), ['--scale-round-x' as string]: '1px', ['--scale-round-y' as string]: '1px' }}
/>
);
});
diff --git a/ui/src/index.css b/ui/src/index.css
index af70dbdb4..123a28736 100644
--- a/ui/src/index.css
+++ b/ui/src/index.css
@@ -1031,6 +1031,65 @@ pre,
}
.vr-fileview-text .vr-fileview-md pre code { font-size: 0.92em; }
+/* PDF text layer (components/ui/pdf-pages.tsx). pdf.js lays one transparent span per run of text
+ over the page canvas, sized from --total-scale-factor (CSS px per PDF point, set on the page box)
+ and the --font-height / --scale-x / --rotate it writes on each span, so the text can be selected
+ and copied where it is drawn. The layer is laid out in unrotated page space; a page's /Rotate
+ arrives as data-main-rotation and turns the layer onto the drawing. Adapted from pdf.js's viewer stylesheet (Apache-2.0); the end block
+ is what keeps a selection from jumping across the page (lib/pdfTextSelection.ts). */
+.vr-pdf-text {
+ --min-font-size: 1;
+ --text-scale-factor: calc(var(--total-scale-factor) * var(--min-font-size));
+ --min-font-size-inv: calc(1 / var(--min-font-size));
+ position: absolute;
+ inset: 0;
+ z-index: 0;
+ overflow: clip;
+ line-height: 1;
+ text-align: initial;
+ letter-spacing: normal;
+ word-spacing: normal;
+ -webkit-text-size-adjust: none;
+ text-size-adjust: none;
+ forced-color-adjust: none;
+ transform-origin: 0 0;
+}
+.vr-pdf-text[data-main-rotation="90"] { transform: rotate(90deg) translateY(-100%); }
+.vr-pdf-text[data-main-rotation="180"] { transform: rotate(180deg) translate(-100%, -100%); }
+.vr-pdf-text[data-main-rotation="270"] { transform: rotate(270deg) translateX(-100%); }
+.vr-pdf-text :is(span, br) {
+ position: absolute;
+ color: transparent;
+ white-space: pre;
+ cursor: text;
+ transform-origin: 0% 0%;
+ -webkit-user-select: text;
+ user-select: text;
+}
+.vr-pdf-text > span {
+ --font-height: 0;
+ --scale-x: 1;
+ --rotate: 0deg;
+ z-index: 1;
+ font-size: calc(var(--text-scale-factor) * var(--font-height));
+ transform: rotate(var(--rotate)) scaleX(var(--scale-x)) scale(var(--min-font-size-inv));
+}
+.vr-pdf-text ::selection {
+ background: color-mix(in srgb, var(--mint) 35%, transparent);
+ color: transparent;
+}
+.vr-pdf-text br::selection { background: transparent; }
+.vr-pdf-text-end {
+ position: absolute;
+ inset: 100% 0 0;
+ z-index: 0;
+ display: block;
+ cursor: default;
+ -webkit-user-select: none;
+ user-select: none;
+}
+.vr-pdf-text.selecting .vr-pdf-text-end { top: 0; }
+
/* The project header remains a scroll/tap target until a touch hold activates. */
.project-drag-header {
cursor: pointer;
diff --git a/ui/src/lib/cjkRadicals.test.ts b/ui/src/lib/cjkRadicals.test.ts
new file mode 100644
index 000000000..26b48cde8
--- /dev/null
+++ b/ui/src/lib/cjkRadicals.test.ts
@@ -0,0 +1,25 @@
+import { describe, expect, it } from 'vitest';
+
+import { unifyCjkRadicals } from './cjkRadicals';
+
+describe('unifyCjkRadicals', () => {
+ it('maps the Kangxi radicals Chrome-made PDFs extract to the characters they show', () => {
+ // As extracted from a Chrome-printed PDF set in PingFang: ⽉ ⾃ ⽣ ⾏ ⾄ are Kangxi radicals.
+ expect(unifyCjkRadicals('10 ⽉ · ⾃动⽣成 · 运⾏稳定,降⾄ 2.1%')).toBe('10 月 · 自动生成 · 运行稳定,降至 2.1%');
+ });
+
+ it('maps CJK Radicals Supplement forms, including simplified ones NFKC leaves alone', () => {
+ expect(unifyCjkRadicals('共 4 ⻚')).toBe('共 4 页');
+ expect(unifyCjkRadicals('⻥⻢⻛')).toBe('鱼马风');
+ expect('⻚'.normalize('NFKC')).toBe('⻚');
+ });
+
+ it('keeps an equivalent outside the BMP whole', () => {
+ expect(unifyCjkRadicals('⺇')).toBe('\u{20628}');
+ });
+
+ it('leaves full-width punctuation, Latin text and ordinary ideographs as they are', () => {
+ const text = '第 1.1 节说明:本季度 Agent 运行稳定,The weekly digest(摘要)!';
+ expect(unifyCjkRadicals(text)).toBe(text);
+ });
+});
diff --git a/ui/src/lib/cjkRadicals.ts b/ui/src/lib/cjkRadicals.ts
new file mode 100644
index 000000000..e81033133
--- /dev/null
+++ b/ui/src/lib/cjkRadicals.ts
@@ -0,0 +1,32 @@
+// CJK radicals → the unified ideograph each one is equivalent to, for text extracted from PDFs.
+//
+// Fonts that Chrome and other generators embed often map common characters (月, 自, 行, 页 …) to the
+// Kangxi Radicals and CJK Radicals Supplement code points that look the same. The text reads
+// correctly on screen but copies as different characters, so pasted text no longer matches a search
+// or the original. pdf.js's own normalization leaves both blocks alone, and NFKC would also turn
+// full-width punctuation into ASCII, so these blocks are mapped on their own.
+//
+// Generated from Unicode's EquivalentUnifiedIdeograph-18.0.0.txt (U+2E80–U+2FDF entries), as
+// consecutive (radical, ideograph) pairs. Regenerate rather than edit by hand.
+const PAIRS =
+ '⺁厂⺂乛⺃乚⺄乙⺅亻⺆冂⺇𠘨⺈刀⺉刂⺊卜⺋㔾⺌小⺍小⺎兀⺏尣⺐尢⺑𡯂⺒巳⺓幺⺔彑⺕𫜹⺖忄⺗心⺘扌⺙攵⺛旡⺜日⺝月⺞歺⺟母⺠民⺡氵⺢氺⺣灬⺤爫⺥爫⺦丬⺧牛⺨犭⺩王' +
+ '⺪𤴔⺫目⺬示⺭礻⺮𥫗⺯糹⺰纟⺱罓⺲罒⺳㓁⺴冗⺵𦉫⺶羊⺷𦍌⺸𦍋⺹耂⺺肀⺻聿⺼肉⺽𦥑⺾艹⺿艹⻀艹⻁虎⻂衤⻃覀⻄西⻅见⻆角⻇𧢲⻈讠⻉贝⻊𧾷⻋车⻌辶⻍辶⻎辶⻏邑⻐钅⻑長' +
+ '⻒镸⻓长⻔门⻕𨸏⻖阝⻗雨⻘青⻙韦⻚页⻛风⻜飞⻝食⻞𩙿⻟飠⻠饣⻡𩠐⻢马⻣骨⻤鬼⻥鱼⻦鸟⻧卤⻨麦⻩黄⻪黾⻫斉⻬齐⻭歯⻮齿⻯竜⻰龙⻱龜⻲亀⻳龟⼀一⼁丨⼂丶⼃丿⼄乙⼅亅' +
+ '⼆二⼇亠⼈人⼉儿⼊入⼋八⼌冂⼍冖⼎冫⼏几⼐凵⼑刀⼒力⼓勹⼔匕⼕匚⼖匸⼗十⼘卜⼙卩⼚厂⼛厶⼜又⼝口⼞囗⼟土⼠士⼡夂⼢夊⼣夕⼤大⼥女⼦子⼧宀⼨寸⼩小⼪尢⼫尸⼬屮⼭山' +
+ '⼮巛⼯工⼰己⼱巾⼲干⼳幺⼴广⼵廴⼶廾⼷弋⼸弓⼹彐⼺彡⼻彳⼼心⼽戈⼾戶⼿手⽀支⽁攴⽂文⽃斗⽄斤⽅方⽆无⽇日⽈曰⽉月⽊木⽋欠⽌止⽍歹⽎殳⽏毋⽐比⽑毛⽒氏⽓气⽔水⽕火' +
+ '⽖爪⽗父⽘爻⽙爿⽚片⽛牙⽜牛⽝犬⽞玄⽟玉⽠瓜⽡瓦⽢甘⽣生⽤用⽥田⽦疋⽧疒⽨癶⽩白⽪皮⽫皿⽬目⽭矛⽮矢⽯石⽰示⽱禸⽲禾⽳穴⽴立⽵竹⽶米⽷糸⽸缶⽹网⽺羊⽻羽⽼老⽽而' +
+ '⽾耒⽿耳⾀聿⾁肉⾂臣⾃自⾄至⾅臼⾆舌⾇舛⾈舟⾉艮⾊色⾋艸⾌虍⾍虫⾎血⾏行⾐衣⾑襾⾒見⾓角⾔言⾕谷⾖豆⾗豕⾘豸⾙貝⾚赤⾛走⾜足⾝身⾞車⾟辛⾠辰⾡辵⾢邑⾣酉⾤釆⾥里' +
+ '⾦金⾧長⾨門⾩阜⾪隶⾫隹⾬雨⾭靑⾮非⾯面⾰革⾱韋⾲韭⾳音⾴頁⾵風⾶飛⾷食⾸首⾹香⾺馬⾻骨⾼高⾽髟⾾鬥⾿鬯⿀鬲⿁鬼⿂魚⿃鳥⿄鹵⿅鹿⿆麥⿇麻⿈黃⿉黍⿊黑⿋黹⿌黽⿍鼎' +
+ '⿎鼓⿏鼠⿐鼻⿑齊⿒齒⿓龍⿔龜⿕龠';
+
+// By code point: a few ideographs are outside the BMP (two UTF-16 units each).
+const CHARS = Array.from(PAIRS);
+const EQUIVALENT = new Map<string, string>();
+for (let i = 0; i < CHARS.length; i += 2) EQUIVALENT.set(CHARS[i], CHARS[i + 1]);
+
+const RADICAL = /[\u2E80-\u2FDF]/g;
+
+/** ``text`` with every CJK radical that has an equivalent unified ideograph replaced by it. */
+export function unifyCjkRadicals(text: string): string {
+ return text.replace(RADICAL, (ch) => EQUIVALENT.get(ch) ?? ch);
+}
diff --git a/ui/src/lib/pdfTextSelection.test.ts b/ui/src/lib/pdfTextSelection.test.ts
new file mode 100644
index 000000000..6af523305
--- /dev/null
+++ b/ui/src/lib/pdfTextSelection.test.ts
@@ -0,0 +1,83 @@
+/* @vitest-environment jsdom */
+
+import { afterEach, describe, expect, it } from 'vitest';
+
+import { PDF_TEXT_LAYER_CLASS, trackPdfTextLayer } from './pdfTextSelection';
+
+const stops: Array<() => void> = [];
+
+const layerWith = (...words: string[]) => {
+ const layer = document.createElement('div');
+ layer.className = PDF_TEXT_LAYER_CLASS;
+ for (const word of words) {
+ const span = document.createElement('span');
+ span.textContent = word;
+ layer.append(span);
+ }
+ document.body.append(layer);
+ stops.push(trackPdfTextLayer(layer));
+ return layer;
+};
+
+const end = (layer: HTMLElement) => layer.querySelector<HTMLElement>('.vr-pdf-text-end');
+
+afterEach(() => {
+ stops.splice(0).forEach((stop) => stop());
+ document.getSelection()?.removeAllRanges();
+ document.body.replaceChildren();
+});
+
+describe('trackPdfTextLayer', () => {
+ it('gives each layer an end-of-content block after its text', () => {
+ const layer = layerWith('Page', 'one');
+ expect(layer.lastElementChild).toBe(end(layer));
+ });
+
+ it('marks a layer as selecting from the press until the pointer is released', () => {
+ const layer = layerWith('Page');
+ layer.dispatchEvent(new MouseEvent('mousedown', { bubbles: true }));
+ expect(layer.classList.contains('selecting')).toBe(true);
+ document.dispatchEvent(new Event('pointerup'));
+ expect(layer.classList.contains('selecting')).toBe(false);
+ });
+
+ it('marks only the layers the selection reaches', () => {
+ const first = layerWith('Page', 'one');
+ const second = layerWith('Page', 'two');
+ const range = document.createRange();
+ range.selectNodeContents(first.firstElementChild!);
+ document.getSelection()!.addRange(range);
+ document.dispatchEvent(new Event('selectionchange'));
+ expect(first.classList.contains('selecting')).toBe(true);
+ expect(second.classList.contains('selecting')).toBe(false);
+
+ document.getSelection()!.removeAllRanges();
+ document.dispatchEvent(new Event('selectionchange'));
+ expect(first.classList.contains('selecting')).toBe(false);
+ expect(first.lastElementChild).toBe(end(first));
+ });
+
+ it('leaves a released layer alone, and tracks layers again after every one was released', () => {
+ const released = layerWith('Page', 'one');
+ const kept = layerWith('Page', 'two');
+ stops.shift()!();
+ const selectFirstWord = (layer: HTMLElement) => {
+ const range = document.createRange();
+ range.selectNodeContents(layer.firstElementChild!);
+ document.getSelection()!.removeAllRanges();
+ document.getSelection()!.addRange(range);
+ document.dispatchEvent(new Event('selectionchange'));
+ };
+ selectFirstWord(released);
+ expect(released.classList.contains('selecting')).toBe(false);
+ selectFirstWord(kept);
+ expect(kept.classList.contains('selecting')).toBe(true);
+
+ // Releasing the last layer stops the shared listeners; the next layer must start them again
+ // (a reopened preview, or the first page drawn after scrolling back).
+ stops.shift()!();
+ const next = layerWith('Page', 'three');
+ selectFirstWord(next);
+ expect(next.classList.contains('selecting')).toBe(true);
+ });
+});
diff --git a/ui/src/lib/pdfTextSelection.ts b/ui/src/lib/pdfTextSelection.ts
new file mode 100644
index 000000000..e21ec45ac
--- /dev/null
+++ b/ui/src/lib/pdfTextSelection.ts
@@ -0,0 +1,137 @@
+// Text selection over the PDF page reader's text layers (components/ui/pdf-pages.tsx).
+//
+// pdf.js places one transparent, absolutely positioned span per run of text over each page canvas,
+// so the page's text can be long-pressed (touch) or dragged (mouse) to select and copy it. Between
+// those spans there is only empty layer. WebKit — every iPhone and iPad — resolves a selection end
+// dragged into that empty space to the start of the layer, so the selection collapses to nothing as
+// soon as a finger or handle leaves the text. pdf.js's own viewer prevents that with an "end of
+// content" block that it moves next to the selection's moving end while a selection is in progress;
+// this is that technique, ported from its ``TextLayerBuilder`` (Apache-2.0) without the rest of the
+// viewer.
+//
+// Current Chromium keeps the end on the nearest text by itself, and measured worse with the block in
+// place, so there (and in Firefox, which pdf.js also exempts) the layers are left to the browser.
+
+export const PDF_TEXT_LAYER_CLASS = 'vr-pdf-text';
+const END_CLASS = 'vr-pdf-text-end';
+const SELECTING_CLASS = 'selecting';
+
+// Text layer → its end-of-content block, for every layer currently in the document.
+const layers = new Map<HTMLElement, HTMLElement>();
+let listeners: AbortController | null = null;
+
+const reset = (end: HTMLElement, layer: HTMLElement) => {
+ layer.append(end);
+ end.style.width = '';
+ end.style.height = '';
+ end.style.userSelect = '';
+ layer.classList.remove(SELECTING_CLASS);
+};
+
+// Firefox, and Chromium from 148 (the version pdf.js's viewer also keys on).
+function handlesGapsNatively(): boolean {
+ if (typeof CSS !== 'undefined' && typeof CSS.supports === 'function' && CSS.supports('-moz-user-select', 'none')) return true;
+ const brands = (navigator as Navigator & { userAgentData?: { brands: { brand: string; version: string }[] } }).userAgentData?.brands;
+ const chromium = brands ? brands.find(({ brand }) => brand === 'Chromium')?.version : /\bChrome\/(\d+)\b/.exec(navigator.userAgent)?.[1];
+ return !!chromium && Number.parseInt(chromium, 10) >= 148;
+}
+
+function listen() {
+ if (listeners) return;
+ const controller = new AbortController();
+ listeners = controller;
+ const { signal } = controller;
+ let pointerDown = false;
+ let previous: Range | null = null;
+ const resetAll = () => layers.forEach(reset);
+
+ document.addEventListener('pointerdown', () => (pointerDown = true), { signal });
+ document.addEventListener(
+ 'pointerup',
+ () => {
+ pointerDown = false;
+ resetAll();
+ },
+ { signal },
+ );
+ window.addEventListener(
+ 'blur',
+ () => {
+ pointerDown = false;
+ resetAll();
+ },
+ { signal },
+ );
+ document.addEventListener('keyup', () => !pointerDown && resetAll(), { signal });
+ document.addEventListener(
+ 'selectionchange',
+ () => {
+ const selection = document.getSelection();
+ if (!selection || selection.rangeCount === 0) {
+ resetAll();
+ return;
+ }
+ const active = new Set<HTMLElement>();
+ for (let i = 0; i < selection.rangeCount; i += 1) {
+ const range = selection.getRangeAt(i);
+ for (const layer of layers.keys()) if (range.intersectsNode(layer)) active.add(layer);
+ }
+ for (const [layer, end] of layers) {
+ if (active.has(layer)) layer.classList.add(SELECTING_CLASS);
+ else reset(end, layer);
+ }
+ if (active.size === 0) return;
+
+ // Move the end block right after the selection's moving end (or before it, when the start is
+ // the end being dragged), so empty space past that point resolves to the block, not the layer.
+ const range = selection.getRangeAt(0);
+ const movingStart =
+ !!previous &&
+ (range.compareBoundaryPoints(Range.END_TO_END, previous) === 0 || range.compareBoundaryPoints(Range.START_TO_END, previous) === 0);
+ let anchor: Node | null = movingStart ? range.startContainer : range.endContainer;
+ if (anchor?.nodeType === Node.TEXT_NODE) anchor = anchor.parentNode;
+ if (!movingStart && range.endOffset === 0 && anchor) {
+ // The end sits before the anchor's first character: it belongs to the previous text.
+ do {
+ while (anchor && !anchor.previousSibling) anchor = anchor.parentNode;
+ anchor = anchor?.previousSibling ?? null;
+ } while (anchor && !anchor.childNodes.length);
+ }
+ const parent = anchor?.parentElement;
+ const layer = parent?.closest<HTMLElement>(`.${PDF_TEXT_LAYER_CLASS}`);
+ const end = layer ? layers.get(layer) : undefined;
+ if (anchor && parent && layer && end) {
+ end.style.width = layer.style.width;
+ end.style.height = layer.style.height;
+ end.style.userSelect = 'text';
+ parent.insertBefore(end, movingStart ? anchor : anchor.nextSibling);
+ }
+ previous = range.cloneRange();
+ },
+ { signal },
+ );
+}
+
+/**
+ * Track a rendered text layer: appends its end-of-content block and starts the shared document
+ * listeners on the first layer. Returns the function that stops tracking it, which also removes the
+ * listeners once no layer is left. A no-op where the browser keeps selections on the text itself.
+ */
+export function trackPdfTextLayer(layer: HTMLElement): () => void {
+ if (handlesGapsNatively()) return () => undefined;
+ const end = document.createElement('div');
+ end.className = END_CLASS;
+ layer.append(end);
+ const onMouseDown = () => layer.classList.add(SELECTING_CLASS);
+ layer.addEventListener('mousedown', onMouseDown);
+ layers.set(layer, end);
+ listen();
+ return () => {
+ layer.removeEventListener('mousedown', onMouseDown);
+ if (layers.get(layer) === end) layers.delete(layer);
+ if (layers.size === 0) {
+ listeners?.abort();
+ listeners = null;
+ }
+ };
+}