Skip to content

Build your personal knowledge base with Trilium Notes

23.1 KB
Raw Blame
/**
 * Minimal Microsoft Graph client for the OneNote importer. Only the read endpoints needed to
 * enumerate the notebook tree and pull page content are implemented.
 *
 * Graph reference: https://learn.microsoft.com/en-us/graph/api/resources/onenote-api-overview
 */

import type { OneNoteNotebook, OneNoteSection, OneNoteSectionGroup } from "@triliumnext/commons";
import { getLog } from "@triliumnext/core";

import { safeFetch } from "../../safe_fetch.js";
import { extractPageId } from "./links.js";

const GRAPH_BASE = "https://graph.microsoft.com/v1.0";

/**
 * Supplies a currently-valid access token, refreshing it as needed. The importer passes one of these
 * rather than a fixed string so a long or heavily-throttled import — which outlives a single Graph
 * token — keeps authenticating instead of 401ing once the token captured up front expires. It is
 * re-read before every request attempt (see {@link graphFetch}), including after a throttle wait.
 */
export type AccessTokenProvider = () => Promise<string>;

// safeFetch's default 5s timeout is too tight for the importer: page content and binary resources can
// be large and slow. Allow a generous per-request budget instead.
const GRAPH_TIMEOUT_MS = 60_000;

// Graph throttles aggressively under load (HTTP 429, occasionally 503). When the response carries a
// Retry-After header that wait is used verbatim — but OneNote's 429s usually omit it
// (https://learn.microsoft.com/en-us/graph/throttling), so retries mostly fall back to exponential
// backoff. A request only gives up once it has spent MAX_THROTTLE_WAIT_MS waiting out throttles: a
// fixed retry count proved too small for huge imports. Microsoft publishes no hard OneNote limits —
// throttling is per app+user and time-based ("simply waiting will eventually reset the limit"), and
// a heavily-throttled user can stay throttled for the better part of an hour, so the budget must
// cover that; one page giving up aborts the whole import.
const BASE_RETRY_DELAY_MS = 2000;
const MAX_RETRY_DELAY_MS = 60_000;
const MAX_THROTTLE_WAIT_MS = 60 * 60_000;

// A 504 means the OneNote backend behind the Graph gateway timed out producing this one response.
// Usually transient (Microsoft's own SDK retry handlers treat 504 like 429/503), but a permanent
// per-resource variant is well documented — some pages/sections 504 on every fetch — so unlike
// throttling these retries are bounded by a small attempt count, not the hour-long wait budget: a
// poisoned page must fail within minutes instead of stalling the import for an hour.
const MAX_GATEWAY_TIMEOUT_RETRIES = 5;

// Shared throttle gate. Graph throttles per app/tenant, so a 429 on one request means every other
// in-flight and subsequent request is being throttled too. Rather than each request independently
// retrying — which just keeps hammering Graph and prolongs the throttle — a 429 pushes back a shared
// "don't send until" timestamp that ALL requests wait on, so the whole pool backs off together.
let throttledUntilMs = 0;

// Notified whenever the shared throttle gate is extended — i.e. Graph is actively throttling and every
// request is now waiting. The importer uses this to flip its progress toast to a "waiting out rate
// limiting" message: during a long throttle window no page completes, so the progress count sits still
// for up to an hour and is otherwise indistinguishable from a hang (users have killed multi-hour
// imports — losing all fetched work — over exactly that).
let throttleListener: ((waitMs: number) => void) | null = null;

/** Registers (or clears, with null) the single throttle-gate listener. */
export function setThrottleListener(listener: ((waitMs: number) => void) | null): void {
    throttleListener = listener;
}

// Aggregate throttle statistics for the current import, surfaced in the import report (they answer
// "why did this take hours?"). `requestCount` counts throttled (429/503) responses; `waitMs`
// accumulates net extensions of the shared gate — wall-clock time spent waiting out throttles —
// rather than a per-request sum, since concurrent requests wait on the same gate window.
let throttledRequestCount = 0;
let throttleWaitMs = 0;

/**
 * Fetches a Microsoft Graph URL through {@link safeFetch} so the request is hardened against SSRF —
 * pagination (`@odata.nextLink`) and resource URLs come from Graph responses and page HTML, so they
 * must not be trusted to point at the public Graph host. The bearer token is sent on every hop.
 *
 * Retries on throttling (429/503) via the shared {@link throttledUntilMs} gate, so concurrent
 * requests pause together instead of dog-piling the throttle. Gives up only once this request has
 * spent {@link MAX_THROTTLE_WAIT_MS} waiting, returning the throttled response for the caller to
 * report as an error.
 *
 * Also retries gateway timeouts (504), but privately: the backoff is slept here rather than pushed
 * onto the shared gate — a 504 is specific to the resource being fetched, not the app+user pool —
 * and gives up after {@link MAX_GATEWAY_TIMEOUT_RETRIES} attempts rather than drawing on the
 * hour-long throttle budget.
 */
async function graphFetch(getAccessToken: AccessTokenProvider, url: string): Promise<Response> {
    const giveUpAtMs = Date.now() + MAX_THROTTLE_WAIT_MS;
    let throttleAttempt = 0;
    let gatewayTimeoutRetry = 0;
    for (;;) {
        const gateWaitMs = throttledUntilMs - Date.now();
        if (gateWaitMs > 0) {
            await delay(gateWaitMs);
        }

        // Resolved after the gate wait, per attempt: a token captured before an hour of throttling
        // would already be expired by the time the request finally goes out. The provider hands back
        // the cached token while it is valid and refreshes only as expiry nears.
        const accessToken = await getAccessToken();
        const response = await safeFetch(url, {
            headers: { Authorization: `Bearer ${accessToken}` },
            signal: AbortSignal.timeout(GRAPH_TIMEOUT_MS)
        });

        if (response.status === 504) {
            if (gatewayTimeoutRetry >= MAX_GATEWAY_TIMEOUT_RETRIES) {
                return response;
            }
            const waitMs = backoffDelayMs(gatewayTimeoutRetry);
            gatewayTimeoutRetry++;

            // Drain the failed response so its connection is released before we back off.
            await response.body?.cancel();
            getLog().info(`OneNote import: Graph gateway timeout (HTTP 504) on ${sanitizeGraphUrl(url)}; retry ${gatewayTimeoutRetry}/${MAX_GATEWAY_TIMEOUT_RETRIES} after ${waitMs}ms`);
            await delay(waitMs);
            continue;
        }

        if (response.status !== 429 && response.status !== 503) {
            return response;
        }

        // Graph's Retry-After, when present, states exactly how long the throttle lasts — trust it
        // over the computed backoff.
        const waitMs = retryAfterMs(response.headers.get("retry-after")) ?? backoffDelayMs(throttleAttempt);
        if (Date.now() + waitMs > giveUpAtMs) {
            return response;
        }

        // Drain the throttled response so its connection is released before we back off.
        await response.body?.cancel();

        // Extend the shared gate (Math.max: simultaneous 429s converge on one window rather than
        // stacking). The wait itself happens at the top of the next iteration, shared across the pool.
        throttledRequestCount++;
        throttleWaitMs += Math.max(0, Date.now() + waitMs - Math.max(throttledUntilMs, Date.now()));
        throttledUntilMs = Math.max(throttledUntilMs, Date.now() + waitMs);
        throttleAttempt++;
        getLog().info(`OneNote import: Graph throttled (HTTP ${response.status}) on ${sanitizeGraphUrl(url)}; retry ${throttleAttempt} after ${waitMs}ms (${Math.round((giveUpAtMs - Date.now()) / 60_000)}min of wait budget left)`);
        throttleListener?.(waitMs);
    }
}

/** Exponential backoff before a throttling retry, capped at {@link MAX_RETRY_DELAY_MS}. */
export function backoffDelayMs(attempt: number): number {
    return Math.min(BASE_RETRY_DELAY_MS * 2 ** attempt, MAX_RETRY_DELAY_MS);
}

/**
 * Parses a Retry-After header (delta-seconds or HTTP-date) into a wait in milliseconds, or null when
 * the header is absent or unparseable.
 */
export function retryAfterMs(header: string | null): number | null {
    const trimmed = header?.trim();
    if (!trimmed) {
        return null;
    }
    const seconds = Number(trimmed);
    if (Number.isFinite(seconds)) {
        return Math.max(0, seconds * 1000);
    }
    const dateMs = Date.parse(trimmed);
    return Number.isNaN(dateMs) ? null : Math.max(0, dateMs - Date.now());
}

/** Resets the shared throttle gate; exported for tests only. */
export function resetThrottleGate(): void {
    throttledUntilMs = 0;
}

/** Returns the throttle statistics accumulated since the last {@link resetThrottleStats}. */
export function getThrottleStats(): { requestCount: number; waitMs: number } {
    return { requestCount: throttledRequestCount, waitMs: throttleWaitMs };
}

/** Resets the throttle statistics; called at the start of an import so its report covers only itself. */
export function resetThrottleStats(): void {
    throttledRequestCount = 0;
    throttleWaitMs = 0;
}

function delay(ms: number): Promise<void> {
    return new Promise((resolve) => setTimeout(resolve, ms));
}

export interface GraphAccount {
    name: string;
    email: string;
}

export interface OneNotePage {
    id: string;
    title: string;
    createdDateTime?: string;
    lastModifiedDateTime?: string;
    /** OneNote's indentation level in the page list: 0 for a top-level page, 1+ for a subpage. */
    level: number;
    /** The page-id GUID OneNote uses in `onenote:` links to this page; used to resolve cross-page links. */
    pageId?: string;
}

export async function getAccount(getAccessToken: AccessTokenProvider): Promise<GraphAccount> {
    const me = await graphGet<{ displayName?: string; mail?: string; userPrincipalName?: string }>(getAccessToken, "/me");
    return {
        name: me.displayName ?? me.userPrincipalName ?? "Unknown",
        email: me.mail ?? me.userPrincipalName ?? ""
    };
}

/**
 * Returns the notebooks with their direct sections and a tree of section groups — each group carrying
 * its own sections and nested groups — mirroring the OneNote structure so the importer can recreate it
 * as nested folders.
 *
 * The initial call expands each notebook's sections and whether it has any section groups, so the
 * common case (no section groups) is a single round-trip. Only notebooks/groups that actually contain
 * section groups follow their `sectionGroupsUrl`, and those follow-up requests run in parallel —
 * section groups nest arbitrarily and the notebooks endpoint won't $expand them more than one level.
 */
export async function listNotebooks(getAccessToken: AccessTokenProvider): Promise<OneNoteNotebook[]> {
    const url = "/me/onenote/notebooks?$select=id,displayName,createdDateTime,lastModifiedDateTime,sectionGroupsUrl&$expand=sections($select=id,displayName,createdDateTime,lastModifiedDateTime),sectionGroups($select=id)&$orderby=displayName";
    const notebooks = await graphGetAll<RawNotebook>(getAccessToken, url);

    return Promise.all(
        notebooks.map(async (notebook) => ({
            id: notebook.id,
            title: notebook.displayName,
            createdDateTime: notebook.createdDateTime,
            lastModifiedDateTime: notebook.lastModifiedDateTime,
            sections: mapSections(notebook.sections),
            sectionGroups: notebook.sectionGroups?.length && notebook.sectionGroupsUrl ? await fetchSectionGroups(getAccessToken, notebook.sectionGroupsUrl) : []
        }))
    );
}

/**
 * Fetches the section groups under `sectionGroupsUrl` (a notebook's or section group's link) as a tree,
 * each group carrying its own sections and nested groups. Sibling groups are fetched in parallel, and a
 * further round-trip happens only for groups that actually have nested groups.
 */
async function fetchSectionGroups(getAccessToken: AccessTokenProvider, sectionGroupsUrl: string): Promise<OneNoteSectionGroup[]> {
    const url = appendQuery(sectionGroupsUrl, "$select=id,displayName,createdDateTime,lastModifiedDateTime,sectionGroupsUrl&$expand=sections($select=id,displayName,createdDateTime,lastModifiedDateTime),sectionGroups($select=id)");
    const groups = await graphGetAll<RawSectionGroup>(getAccessToken, url);

    return Promise.all(
        groups.map(async (group) => ({
            id: group.id,
            title: group.displayName,
            createdDateTime: group.createdDateTime,
            lastModifiedDateTime: group.lastModifiedDateTime,
            sections: mapSections(group.sections),
            sectionGroups: group.sectionGroups?.length && group.sectionGroupsUrl ? await fetchSectionGroups(getAccessToken, group.sectionGroupsUrl) : []
        }))
    );
}

function mapSections(raw: RawSection[] | undefined): OneNoteSection[] {
    return (raw ?? []).map((s) => ({ id: s.id, title: s.displayName, createdDateTime: s.createdDateTime, lastModifiedDateTime: s.lastModifiedDateTime }));
}

function appendQuery(url: string, query: string): string {
    return url.includes("?") ? `${url}&${query}` : `${url}?${query}`;
}

export async function listPages(getAccessToken: AccessTokenProvider, sectionId: string): Promise<OneNotePage[]> {
    // `links` carries the page's own `onenote:` client URL, from which we recover the page-id GUID that
    // cross-page links reference (see links.ts).
    const url = `/me/onenote/sections/${sectionId}/pages?$select=id,title,createdDateTime,lastModifiedDateTime,level,links&$orderby=order&pagelevel=true`;
    const raw = await graphGetAll<RawPage>(getAccessToken, url);
    return raw.map((p) => ({
        id: p.id,
        title: p.title || "Untitled",
        createdDateTime: p.createdDateTime,
        lastModifiedDateTime: p.lastModifiedDateTime,
        level: p.level ?? 0,
        pageId: extractPageId(p.links?.oneNoteClientUrl?.href) ?? undefined
    }));
}

/**
 * Returns a page's content, split into its HTML body and (optional) InkML.
 *
 * We request `includeInkML=true` so handwriting/drawings — which the default HTML output drops,
 * leaving only `<!-- InkNode is not supported -->` comments — come back as a separate InkML part. With
 * that flag the response is a MIME multipart envelope (one `text/html` part, one
 * `application/inkml+xml` part), which parsePageContent splits apart. Pages without ink may still come
 * back as plain HTML, which the parser handles by returning the whole body as `html`.
 */
export async function getPageContent(getAccessToken: AccessTokenProvider, pageId: string): Promise<{ html: string; inkml: string }> {
    const url = `${GRAPH_BASE}/me/onenote/pages/${pageId}/content?includeInkML=true`;
    const response = await graphFetch(getAccessToken, url);
    if (!response.ok) {
        throw await graphRequestError("Failed to fetch OneNote page content", url, response);
    }
    return parsePageContent(await response.text());
}

/**
 * Splits a OneNote page response into its HTML and InkML parts. A page fetched with
 * `includeInkML=true` comes back as a MIME multipart body whose first line is the boundary; each part
 * carries header lines (including `Content-Type`), then a blank line, then the body. A non-multipart
 * response (no leading `--` boundary) is returned verbatim as the HTML part.
 */
export function parsePageContent(raw: string): { html: string; inkml: string } {
    const normalized = raw.replace(/\r\n/g, "\n");
    const boundary = normalized.slice(0, normalized.indexOf("\n")).trim();
    if (!boundary.startsWith("--")) {
        return { html: raw.trim(), inkml: "" };
    }

    let html = "";
    let inkml = "";
    for (const part of normalized.split(boundary)) {
        const separator = part.indexOf("\n\n");
        if (separator < 0) {
            continue;
        }
        const headers = part.slice(0, separator).toLowerCase();
        const body = part.slice(separator + 2).trim();
        if (headers.includes("text/html")) {
            html = body;
        } else if (headers.includes("application/inkml+xml")) {
            inkml = body;
        }
    }
    return { html, inkml };
}

/**
 * Downloads a binary resource (image or file attachment) referenced from page HTML. The URL is an
 * absolute Graph `…/resources/{id}/$value` link, so it is fetched directly rather than relative to the
 * API base. Returns the raw bytes plus the server-reported content type.
 */
export async function getResource(getAccessToken: AccessTokenProvider, url: string): Promise<{ content: Uint8Array; contentType: string }> {
    const response = await graphFetch(getAccessToken, url);
    if (!response.ok) {
        throw await graphRequestError("Failed to fetch OneNote resource", url, response);
    }
    const buffer = new Uint8Array(await response.arrayBuffer());
    const contentType = response.headers.get("content-type") ?? "application/octet-stream";
    getLog().info(`OneNote import: downloaded resource (${contentType}, ${buffer.length} bytes) from ${sanitizeGraphUrl(url)}`);
    return { content: buffer, contentType };
}

interface RawSection {
    id: string;
    displayName: string;
    createdDateTime?: string;
    lastModifiedDateTime?: string;
}

interface RawNotebook {
    id: string;
    displayName: string;
    createdDateTime?: string;
    lastModifiedDateTime?: string;
    sectionGroupsUrl?: string;
    sections?: RawSection[];
    /** id-only, requested just to detect whether the notebook has section groups to follow. */
    sectionGroups?: { id: string }[];
}

interface RawSectionGroup {
    id: string;
    displayName: string;
    createdDateTime?: string;
    lastModifiedDateTime?: string;
    sectionGroupsUrl?: string;
    sections?: RawSection[];
    /** id-only, requested just to detect whether the group has nested section groups to follow. */
    sectionGroups?: { id: string }[];
}

interface RawPage {
    id: string;
    title?: string;
    createdDateTime?: string;
    lastModifiedDateTime?: string;
    level?: number;
    links?: { oneNoteClientUrl?: { href?: string } };
}

async function graphGet<T>(getAccessToken: AccessTokenProvider, path: string): Promise<T> {
    const url = `${GRAPH_BASE}${path}`;
    const response = await graphFetch(getAccessToken, url);
    if (!response.ok) {
        throw await graphRequestError("Microsoft Graph request failed", url, response);
    }
    return response.json() as Promise<T>;
}

/** GETs a Graph collection endpoint, following @odata.nextLink pagination. */
async function graphGetAll<T>(getAccessToken: AccessTokenProvider, pathOrUrl: string): Promise<T[]> {
    const results: T[] = [];
    let next: string | null = pathOrUrl.startsWith("http") ? pathOrUrl : `${GRAPH_BASE}${pathOrUrl}`;

    while (next) {
        const response: Response = await graphFetch(getAccessToken, next);
        if (!response.ok) {
            // `next` rather than `pathOrUrl`: on a paginated collection the failing request may be a
            // follow-up @odata.nextLink, not the original URL.
            throw await graphRequestError("Microsoft Graph request failed", next, response);
        }
        const json = (await response.json()) as { value?: T[]; "@odata.nextLink"?: string };
        results.push(...(json.value ?? []));
        next = json["@odata.nextLink"] ?? null;
    }

    return results;
}

/**
 * A failed Graph request, carrying the HTTP status and (when present) the Graph error code from the
 * response body alongside the human-readable message, so callers can react to specific failures — e.g.
 * {@link isEncryptedSectionError} — instead of parsing the message string.
 */
export class GraphRequestError extends Error {
    readonly status: number;
    /** The `error.code` from Graph's JSON error envelope, if the body was one (e.g. "20185"). */
    readonly code?: string;

    constructor(message: string, status: number, code?: string) {
        super(message);
        this.name = "GraphRequestError";
        this.status = status;
        this.code = code;
    }
}

/**
 * The Graph error code returned (with HTTP 403) when a section is encrypted/password-protected: its
 * pages can't be listed or read through the API at all. Not in Microsoft's published error-code list —
 * that only documents the write-side 10004 ("can't create a page … protected by a password") — so it
 * was captured from a live import. See {@link isEncryptedSectionError}.
 */
export const ENCRYPTED_SECTION_ERROR_CODE = "20185";

/** Whether an error is Graph rejecting a section because it is encrypted/password-protected. */
export function isEncryptedSectionError(e: unknown): boolean {
    return e instanceof GraphRequestError && e.status === 403 && e.code === ENCRYPTED_SECTION_ERROR_CODE;
}

/**
 * Builds the error for a failed Graph request. A bare HTTP status is not actionable when an import of
 * thousands of pages fails on one of them, so the message carries the request URL plus whatever error
 * code/message Graph itself returned in the response body.
 */
async function graphRequestError(summary: string, url: string, response: Response): Promise<GraphRequestError> {
    let body = "";
    try {
        body = await response.text();
    } catch {
        // The status and URL are still worth reporting when the body cannot be read.
    }
    const { code } = parseGraphError(body);
    const detail = extractGraphErrorDetail(body);
    return new GraphRequestError(`${summary} (HTTP ${response.status}${detail ? `: ${detail}` : ""}) from ${sanitizeGraphUrl(url)}`, response.status, code);
}

/**
 * Redacts personal data from a Graph URL before it is logged or embedded in an error message. OneNote
 * resource URLs (taken from page HTML) address the mailbox by the signed-in user's email — e.g.
 * `…/users('jane@example.com')/onenote/resources/…` — so the raw URL is PII. The email (or user id) in
 * the `users('…')` / `users/{id}` segment is replaced with a placeholder; the rest of the path, which
 * is what makes the log line useful for debugging, is kept. The importer's own calls use the `/me`
 * alias and pass through unchanged.
 */
export function sanitizeGraphUrl(url: string): string {
    return url
        .replace(/users\('[^']*'\)/gi, "users('<redacted>')")
        .replace(/users\/[^/?#]+/gi, "users/<redacted>");
}

/**
 * Parses Graph's standard JSON error envelope (`{"error": {"code": "...", "message": "..."}}`) into its
 * code and message, or an empty object when the body is not one.
 */
function parseGraphError(body: string): { code?: string; message?: string } {
    try {
        const parsed = JSON.parse(body) as { error?: { code?: string; message?: string } };
        return { code: parsed?.error?.code, message: parsed?.error?.message };
    } catch {
        return {};
    }
}

/**
 * Extracts "code: message" from Graph's standard JSON error envelope
 * (`{"error": {"code": "...", "message": "..."}}`), or "" when the body is not one.
 */
export function extractGraphErrorDetail(body: string): string {
    const { code, message } = parseGraphError(body);
    return [code, message].filter(Boolean).join(": ");
}

export default {
    getAccount,
    listNotebooks,
    listPages,
    getPageContent,
    getResource,
    getThrottleStats,
    resetThrottleStats,
    setThrottleListener,
    isEncryptedSectionError
};