Skip to content

Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.

Add browser folder upload for ingest

anirudhkumar-nanonetsfcb3a2cce778
diff --git a/.gitignore b/.gitignore
index 5498508d..2c3b9659 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,10 @@ dist/
*.db
*.db-shm
*.db-wal
+
+# Docker runtime volume — holds the graph db and the generated access token
+# (.web-token / share-link.txt). Never commit; the token would leak.
+data/
# Ignore the local graph dir's contents (the SQLite replica) but keep the
# committable, git-native team-sync file (Mode A).
.context-graph/*
diff --git a/README.md b/README.md
index ee473574..9519de93 100644
--- a/README.md
+++ b/README.md
@@ -177,7 +177,7 @@ docker compose up -d
cat ./data/share-link.txt # the share link, with access token
```
-The graph persists to `./data/graph.db`; drop documents into `./docs` and ingest them from the UI as `/docs`. When exposed beyond localhost the API requires an access token. It's generated once, saved to `./data/share-link.txt` (and printed to `docker compose logs`), and **stays stable across restarts** so shared links keep working. Pin your own with `CONTEXT_GRAPH_WEB_TOKEN`.
+The graph persists to `./data/graph.db`. Ingest documents two ways: click **Choose folder…** in the UI to upload a folder straight from your computer (works for remote teammates), or drop files into `./docs` on the server and ingest the mounted `/docs` path. When exposed beyond localhost the API requires an access token. It's generated once, saved to `./data/share-link.txt` (and printed to `docker compose logs`), and **stays stable across restarts** so shared links keep working. Pin your own with `CONTEXT_GRAPH_WEB_TOKEN`.
No server to put it on? Run it on any machine and share it over [Tailscale](https://tailscale.com) without opening a port to the internet:
diff --git a/src/ingest/pdf.ts b/src/ingest/pdf.ts
index f34feed3..99198aec 100644
--- a/src/ingest/pdf.ts
+++ b/src/ingest/pdf.ts
@@ -8,9 +8,17 @@ import { readFile } from "node:fs/promises";
* story honest. Pages are merged into a single string for chunking.
*/
export async function extractPdfText(path: string): Promise<string> {
- const { extractText, getDocumentProxy } = await import("unpdf");
const buffer = await readFile(path);
- const pdf = await getDocumentProxy(new Uint8Array(buffer));
+ return extractPdfTextFromData(new Uint8Array(buffer));
+}
+
+/**
+ * Extract plain text from PDF bytes already in memory — used for browser
+ * uploads, where there is no file on the server's disk to read.
+ */
+export async function extractPdfTextFromData(data: Uint8Array): Promise<string> {
+ const { extractText, getDocumentProxy } = await import("unpdf");
+ const pdf = await getDocumentProxy(data);
const { text } = await extractText(pdf, { mergePages: true });
return Array.isArray(text) ? text.join("\n\n") : text;
}
diff --git a/src/web.ts b/src/web.ts
index 2a30356b..21ac3504 100644
--- a/src/web.ts
+++ b/src/web.ts
@@ -23,6 +23,7 @@ import { resolve, dirname, join } from "node:path";
import OpenAI from "openai";
import { ContextGraphEngine } from "./engine.js";
import { resolveConfig } from "./ai/providers.js";
+import { extractPdfTextFromData, isPdfPath } from "./ingest/pdf.js";
const PORT = Number(process.env.PORT ?? 4680);
const HOST = process.env.HOST ?? "127.0.0.1";
@@ -76,12 +77,12 @@ async function synthesizeAnswer(
return response.choices[0]?.message?.content ?? "";
}
-function readBody(req: IncomingMessage): Promise<string> {
+function readBody(req: IncomingMessage, maxBytes = 10_000_000): Promise<string> {
return new Promise((res, rej) => {
let data = "";
req.on("data", (c) => {
data += c;
- if (data.length > 10_000_000) rej(new Error("body too large"));
+ if (data.length > maxBytes) rej(new Error("body too large"));
});
req.on("end", () => res(data));
req.on("error", rej);
@@ -307,6 +308,32 @@ const server = createServer(async (req, res) => {
return;
}
+ if (req.method === "POST" && url.pathname === "/api/ingest-upload") {
+ // One uploaded file at a time (the browser folder picker can't hand us a
+ // server path, so it ships file contents instead). Text files arrive as
+ // `text`; PDFs and other binaries arrive base64-encoded in `dataBase64`.
+ // 60 MB cap leaves room for a large PDF plus base64's ~33% overhead.
+ const { name, text, dataBase64 } = JSON.parse(await readBody(req, 60_000_000)) as {
+ name?: string;
+ text?: string;
+ dataBase64?: string;
+ };
+ if (!name?.trim()) return json(res, 400, { error: "name is required" });
+
+ let content: string;
+ if (isPdfPath(name)) {
+ if (!dataBase64) return json(res, 400, { error: "PDF upload needs dataBase64" });
+ content = await extractPdfTextFromData(new Uint8Array(Buffer.from(dataBase64, "base64")));
+ } else {
+ content = text ?? "";
+ }
+ if (!content.trim()) return json(res, 200, { skipped: true, name, empty: true });
+
+ const result = await engine.ingest(content, { title: name, source: name });
+ json(res, 200, { result, stats: await engine.stats() });
+ return;
+ }
+
if (req.method === "POST" && url.pathname === "/api/contribute") {
const { learning, agentId } = JSON.parse(await readBody(req)) as {
learning?: string;
diff --git a/src/web/ui.html b/src/web/ui.html
index 53cada93..46a24502 100644
--- a/src/web/ui.html
+++ b/src/web/ui.html
@@ -204,6 +204,14 @@
.grow-status { font-family: var(--mono); font-size: 12px; margin-top: 10px; color: var(--ink-soft); white-space: pre-wrap; word-break: break-word; }
.grow-status.ok { color: var(--verdant-deep); }
.grow-status.err { color: #a03b2e; }
+ details.server-path { margin-top: 12px; border-top: 1px solid var(--line); padding-top: 10px; }
+ details.server-path summary { cursor: pointer; font-size: 12.5px; color: var(--ink-faint); list-style: none; }
+ details.server-path summary::-webkit-details-marker { display: none; }
+ details.server-path summary::before { content: "›"; margin-right: 6px; color: var(--verdant); }
+ details.server-path[open] summary::before { content: "⌄"; }
+ details.server-path .grow-hint code { font-family: var(--mono); font-size: 11.5px; }
+ details.server-path[open] { padding-bottom: 4px; }
+ details.server-path > *:not(summary) { margin-top: 8px; }
.doc-row { display: flex; gap: 8px; align-items: baseline; padding: 7px 2px; border-bottom: 1px solid var(--line); font-size: 13px; }
.doc-row .t { font-weight: 600; flex: 1; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
@@ -354,9 +362,15 @@
<details class="grow" id="ingest-panel">
<summary>Ingest a folder of documents</summary>
<div class="grow-body">
- <p class="grow-hint">Point it at a folder on this machine (.pdf, .md, .txt). Every file becomes chunks, entities, and relationships — watch the graph grow on the right.</p>
- <input class="dir" id="ingest-input" placeholder="/path/to/your/docs" />
- <button class="btn quiet" id="ingest-btn" type="button">Ingest folder</button>
+ <p class="grow-hint">Pick a folder from your computer (.pdf, .md, .txt) — every file becomes chunks, entities, and relationships, watch the graph grow on the right.</p>
+ <input type="file" id="folder-input" webkitdirectory multiple hidden />
+ <button class="btn" id="choose-folder-btn" type="button">Choose folder…</button>
+ <details class="server-path">
+ <summary>or ingest a folder on the server</summary>
+ <p class="grow-hint">Reads files from the server's own disk — in Docker, that's the mounted <code>/docs</code>.</p>
+ <input class="dir" id="ingest-input" placeholder="/path/to/your/docs" />
+ <button class="btn quiet" id="ingest-btn" type="button">Ingest server folder</button>
+ </details>
<div class="grow-status" id="ingest-status"></div>
</div>
</details>
@@ -837,6 +851,87 @@ $("ingest-btn").addEventListener("click", async () => {
}
});
+/* ---------- folder upload (browser picker → server ingest) ---------- */
+const UPLOAD_ACCEPT = /\.(pdf|md|markdown|txt)$/i;
+
+// Base64-encode file bytes in chunks (fromCharCode.apply chokes on huge arrays).
+async function fileToPayload(file) {
+ if (/\.pdf$/i.test(file.name)) {
+ const bytes = new Uint8Array(await file.arrayBuffer());
+ let binary = "";
+ for (let i = 0; i < bytes.length; i += 0x8000) {
+ binary += String.fromCharCode.apply(null, bytes.subarray(i, i + 0x8000));
+ }
+ return { name: file.name, dataBase64: btoa(binary) };
+ }
+ return { name: file.name, text: await file.text() };
+}
+
+$("choose-folder-btn").addEventListener("click", () => $("folder-input").click());
+
+$("folder-input").addEventListener("change", async (e) => {
+ const all = Array.from(e.target.files || []);
+ const files = all.filter((f) => UPLOAD_ACCEPT.test(f.name));
+ const status = $("ingest-status");
+ e.target.value = ""; // let the user re-pick the same folder later
+ if (!files.length) {
+ status.className = "grow-status err";
+ status.textContent = all.length
+ ? "No .pdf, .md, or .txt files in that folder."
+ : "No folder selected.";
+ return;
+ }
+ const btn = $("choose-folder-btn");
+ btn.disabled = true;
+ status.className = "grow-status";
+ const lines = [];
+ let ok = 0, skipped = 0, failed = 0, nodes = 0;
+ try {
+ for (let i = 0; i < files.length; i++) {
+ const f = files[i];
+ lines.push(`[${i + 1}/${files.length}] ${f.name}…`);
+ status.textContent = lines.slice(-6).join("\n");
+ try {
+ const r = await api("/api/ingest-upload", {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify(await fileToPayload(f)),
+ });
+ if (!r.ok) {
+ const er = await r.json().catch(() => ({}));
+ throw new Error(er.error || r.statusText);
+ }
+ const data = await r.json();
+ if (data.result && !data.result.skipped) {
+ ok++;
+ nodes += (data.result.nodesCreated || 0) + (data.result.nodesUpdated || 0);
+ } else {
+ skipped++;
+ }
+ // Grow the graph live between files, like the server-folder ingest.
+ refreshGraph({ diff: true }).catch(() => {});
+ refreshStats().catch(() => {});
+ } catch (err) {
+ failed++;
+ lines.push(` ✗ ${f.name}: ${err.message}`);
+ status.textContent = lines.slice(-6).join("\n");
+ }
+ }
+ status.className = failed ? "grow-status err" : "grow-status ok";
+ status.textContent =
+ `✓ ${ok} document${ok === 1 ? "" : "s"} ingested` +
+ (skipped ? ` (${skipped} already known, skipped)` : "") +
+ (failed ? `, ${failed} failed` : "") +
+ `, ${nodes} facts written or reinforced` +
+ (failed ? "\nExtraction needs OPENROUTER_API_KEY or a running Ollama." : "");
+ await refreshStats();
+ await refreshGraph({ diff: true });
+ await refreshDocs();
+ } finally {
+ btn.disabled = false;
+ }
+});
+
/* ---------- documents ---------- */
async function refreshDocs() {
const docs = await (await api("/api/documents")).json();