One storage API for Google Drive, Nextcloud, and S3-compatible clouds, in TypeScript and Python.
TypeScript byoccloud-storagecloudflare-r2encryptiongoogle-drivenextcloudoauth2pkcepythons3sdkstoragetypescriptwebdav
docs: add a runnable direct-upload demo
Ajay Raminenie9546055d74c
diff --git a/examples/direct-upload-demo/README.md b/examples/direct-upload-demo/README.mdnew file mode 100644index 0000000..b77708f--- /dev/null+++ b/examples/direct-upload-demo/README.md@@ -0,0 +1,79 @@+# Direct upload demo++A page that uploads a file to storage, and a server that never sees it.++The server does two things: serve the page, and sign a short-lived permission+to write one object. Every request it receives is logged with a byte count, so+the log is the proof — a small `GET` for the grant, then nothing at all while+hundreds of megabytes move.++```+01:11:38 GET /api/grant no body total through server: 0 B+01:11:38 GET /api/verify no body total through server: 0 B+```++## Run it++No cloud account needed. MinIO is a real S3 server and runs locally:++```bash+brew install minio+minio server /tmp/byoc-demo --address :9000+```++Create a `byoc-demo` bucket in it, then:++```bash+npm install+npm start+```++Open <http://localhost:4000>, drop in a large file, and watch the terminal.++To point at a real bucket instead:++```bash+S3_ENDPOINT=https://<account>.r2.cloudflarestorage.com \+S3_BUCKET=my-bucket \+S3_ACCESS_KEY=... S3_SECRET_KEY=... \+npm start+```++Your bucket needs a CORS rule allowing `PUT` from `http://localhost:4000`, or+the browser blocks the request before it is sent.++## What to look at++**`server.mjs`** is about 90 lines and the interesting part is how little it+does. `createUploadGrant()` returns plain JSON with no credential of yours in+it; that JSON goes to the browser and the browser talks to storage.++**`public/index.html`** loads `@byoc/browser` straight from a CDN, so there is+no build step. It is the same published package an application installs from+npm.++The `/api/verify` route exists so the demo proves the file *arrived*. A progress+bar reaching 100% proves nothing on its own — it asks the provider what actually+landed and reports the size.++## Recording it++For a GIF, put the browser and the server's terminal side by side. The+right-hand pane staying empty is the whole point, so give it as much room as+the browser. End the shot on the confirmed size rather than on the progress bar,+or a viewer can reasonably assume the bytes went nowhere.++```bash+ffmpeg -i demo.mov \+ -vf "fps=12,scale=1200:-1:flags=lanczos,split[a][b];[a]palettegen[p];[b][p]paletteuse" \+ -loop 0 demo.gif+```++Keep it under 5 MB, or GitHub serves it slowly enough that people scroll past.++## Caveats++A grant is a **bearer capability**: whoever holds it can write to that one path+until it expires. This demo hands one to anybody who asks, which is fine for a+demo and wrong for an application. In production, issue a grant only after+checking the caller is allowed to write there.diff --git a/examples/direct-upload-demo/package.json b/examples/direct-upload-demo/package.jsonnew file mode 100644index 0000000..dafaa44--- /dev/null+++ b/examples/direct-upload-demo/package.json@@ -0,0 +1,14 @@+{+ "name": "example-direct-upload-demo",+ "version": "0.4.0",+ "private": true,+ "description": "The bytes never touch this server. A runnable demonstration of BYOC upload grants.",+ "type": "module",+ "scripts": {+ "start": "node server.mjs"+ },+ "dependencies": {+ "@byoc/core": "^0.4.0",+ "@byoc/s3-compatible": "^0.4.0"+ }+}diff --git a/examples/direct-upload-demo/public/index.html b/examples/direct-upload-demo/public/index.htmlnew file mode 100644index 0000000..d6b2598--- /dev/null+++ b/examples/direct-upload-demo/public/index.html@@ -0,0 +1,95 @@+<!doctype html>+<meta charset="utf-8" />+<title>BYOC — the bytes never touch the server</title>+<style>+ :root { color-scheme: dark; }+ body {+ margin: 0; padding: 48px 40px;+ font: 15px/1.6 ui-sans-serif, -apple-system, "Segoe UI", sans-serif;+ background: #0d1117; color: #e6edf3;+ }+ h1 { font-size: 20px; margin: 0 0 4px; font-weight: 600; }+ p.sub { margin: 0 0 32px; color: #8b949e; }+ .drop {+ border: 1px dashed #30363d; border-radius: 10px; padding: 40px;+ text-align: center; cursor: pointer; transition: border-color .15s, background .15s;+ }+ .drop:hover, .drop.over { border-color: #58a6ff; background: #0f1620; }+ .bar { height: 6px; background: #21262d; border-radius: 999px; margin: 28px 0 10px; overflow: hidden; }+ .fill { height: 100%; width: 0; background: #3fb950; transition: width .12s linear; }+ .row { display: flex; justify-content: space-between; font-variant-numeric: tabular-nums; color: #8b949e; }+ .pct { color: #e6edf3; font-weight: 600; }+ .done { margin-top: 26px; padding: 14px 16px; border-radius: 8px;+ background: #0f2417; border: 1px solid #1f6f3f; display: none; }+ .done strong { color: #3fb950; }+ code { color: #79c0ff; }+</style>++<h1>Upload to your own cloud</h1>+<p class="sub">This page talks to storage directly. The server only signs a permission.</p>++<div class="drop" id="drop">+ <input type="file" id="file" hidden />+ Drop a large file here, or click to choose one+</div>++<div class="bar"><div class="fill" id="fill"></div></div>+<div class="row">+ <span id="status">waiting for a file</span>+ <span class="pct" id="pct">0%</span>+</div>++<div class="done" id="done"></div>++<script type="module">+ // Served from the CDN so the demo needs no bundler. This is the same+ // published package an application would install from npm.+ import { uploadWithGrant, reviveGrant } from "https://cdn.jsdelivr.net/npm/@byoc/[email protected]/+esm";++ const drop = document.getElementById("drop");+ const input = document.getElementById("file");+ const fill = document.getElementById("fill");+ const pct = document.getElementById("pct");+ const status = document.getElementById("status");+ const done = document.getElementById("done");++ drop.onclick = () => input.click();+ drop.ondragover = (e) => { e.preventDefault(); drop.classList.add("over"); };+ drop.ondragleave = () => drop.classList.remove("over");+ drop.ondrop = (e) => { e.preventDefault(); drop.classList.remove("over"); send(e.dataTransfer.files[0]); };+ input.onchange = () => send(input.files[0]);++ const mb = (n) => (n / 1048576).toFixed(1) + " MB";++ async function send(file) {+ if (!file) return;+ done.style.display = "none";+ status.textContent = `asking the server for permission…`;++ // The only thing the server is asked for: a short-lived, path-scoped grant.+ const grant = reviveGrant(+ await fetch(`/api/grant?name=${encodeURIComponent(file.name)}`).then((r) => r.json())+ );++ status.textContent = `uploading ${mb(file.size)} straight to storage`;+ const started = performance.now();++ await uploadWithGrant(grant, file, {+ onProgress: (p) => {+ fill.style.width = p.percentage + "%";+ pct.textContent = p.percentage.toFixed(0) + "%";+ }+ });++ const seconds = ((performance.now() - started) / 1000).toFixed(1);++ // Prove it arrived, rather than trusting a progress bar that reached 100%.+ const landed = await fetch(`/api/verify?name=${encodeURIComponent(file.name)}`).then((r) => r.json());++ status.textContent = `done in ${seconds}s`;+ done.style.display = "block";+ done.innerHTML =+ `<strong>${mb(landed.size)} in your bucket</strong> at <code>${landed.path}</code><br>` ++ `Bytes through the application server: <strong>0</strong>`;+ }+</script>diff --git a/examples/direct-upload-demo/server.mjs b/examples/direct-upload-demo/server.mjsnew file mode 100644index 0000000..d69fcb4--- /dev/null+++ b/examples/direct-upload-demo/server.mjs@@ -0,0 +1,109 @@+/**+ * A server that never sees your file.+ *+ * It does exactly two things: serve a page, and sign a short-lived permission+ * to write one object. The upload itself goes from the browser straight to the+ * storage account, so the bytes never enter this process.+ *+ * Every request is logged with its body size. That log is the point of the+ * demo: it shows a small POST for the grant, and then nothing at all while+ * hundreds of megabytes move.+ *+ * npm install && npm start+ *+ * Defaults target a local MinIO, so no cloud account is needed:+ * minio server /tmp/byoc-demo --address :9000+ */+import { createServer } from "node:http";+import { readFile } from "node:fs/promises";+import { fileURLToPath } from "node:url";+import path from "node:path";++import { BYOC } from "@byoc/core";+import { S3CompatibleProvider } from "@byoc/s3-compatible";++const HERE = path.dirname(fileURLToPath(import.meta.url));+const PORT = Number(process.env.PORT ?? 4000);++const storage = new BYOC({+ provider: new S3CompatibleProvider({+ endpoint: process.env.S3_ENDPOINT ?? "http://127.0.0.1:9000",+ bucket: process.env.S3_BUCKET ?? "byoc-demo",+ region: process.env.S3_REGION ?? "us-east-1",+ accessKeyId: process.env.S3_ACCESS_KEY ?? "minioadmin",+ secretAccessKey: process.env.S3_SECRET_KEY ?? "minioadmin",+ forcePathStyle: true+ })+});+await storage.connect();++/** Bytes this process has received in request bodies, across its whole life. */+let bytesThroughServer = 0;++function log(method, url, bodyBytes) {+ bytesThroughServer += bodyBytes;+ const stamp = new Date().toISOString().slice(11, 19);+ const size = bodyBytes ? `${bodyBytes} B body` : "no body";+ console.log(+ `${stamp} ${method.padEnd(4)} ${url.padEnd(34)} ${size.padStart(12)} ` ++ `total through server: ${bytesThroughServer} B`+ );+}++const server = createServer(async (req, res) => {+ const url = new URL(req.url, `http://${req.headers.host}`);++ // Count anything a client tries to send us, so the log cannot flatter us.+ let bodyBytes = 0;+ req.on("data", (chunk) => (bodyBytes += chunk.length));+ await new Promise((resolve) => req.on("end", resolve));+ log(req.method, url.pathname, bodyBytes);++ if (url.pathname === "/") {+ res.writeHead(200, { "content-type": "text/html; charset=utf-8" });+ res.end(await readFile(path.join(HERE, "public", "index.html")));+ return;+ }++ if (url.pathname === "/api/grant") {+ const name = url.searchParams.get("name");+ if (!name) {+ res.writeHead(400, { "content-type": "application/json" });+ res.end(JSON.stringify({ error: "name is required" }));+ return;+ }++ // In a real app this is where you check the caller may write here. A grant+ // is a bearer capability: whoever holds it can write to that one path until+ // it expires, so it must never be handed out unchecked.+ const grant = await storage.createUploadGrant(`demo/${name}`, {+ expiresInSeconds: 900+ });++ res.writeHead(200, { "content-type": "application/json" });+ res.end(JSON.stringify(grant));+ return;+ }++ if (url.pathname === "/api/verify") {+ // Ask the provider what actually landed, so the demo proves arrival+ // rather than just showing a progress bar reach 100%.+ const name = url.searchParams.get("name");+ try {+ const meta = await storage.metadata(`demo/${name}`);+ res.writeHead(200, { "content-type": "application/json" });+ res.end(JSON.stringify({ size: meta.size, path: meta.path }));+ } catch {+ res.writeHead(404, { "content-type": "application/json" });+ res.end(JSON.stringify({ error: "not found" }));+ }+ return;+ }++ res.writeHead(404).end();+});++server.listen(PORT, () => {+ console.log(`\n BYOC direct-upload demo on http://localhost:${PORT}`);+ console.log(" Watch this log while you upload. It should stay quiet.\n");+});diff --git a/package-lock.json b/package-lock.jsonindex 7044b91..75e818a 100644--- a/package-lock.json+++ b/package-lock.json@@ -21,6 +21,14 @@"node": ">=20"}},+ "examples/direct-upload-demo": {+ "name": "example-direct-upload-demo",+ "version": "0.4.0",+ "dependencies": {+ "@byoc/core": "^0.4.0",+ "@byoc/s3-compatible": "^0.4.0"+ }+ },"examples/node-quickstart": {"name": "example-node-quickstart","version": "0.4.0",@@ -570,6 +578,10 @@"@types/estree": "^1.0.0"}},+ "node_modules/example-direct-upload-demo": {+ "resolved": "examples/direct-upload-demo",+ "link": true+ },"node_modules/example-node-quickstart": {"resolved": "examples/node-quickstart","link": true