Record a sky camera around the clock, keep the footage on your own disk, and find what flew past.
TypeScript astronomycloudflare-workerscomputer-visionmcpmeteor-detectionobject-trackingrtspself-hostedsky-camerayolo
Encode IDs in website API paths; keep secrets out of setup output (#6)
Hunter Deibler63a87f778d74
diff --git a/package/dashboard/test-ui/api-paths.check.mjs b/package/dashboard/test-ui/api-paths.check.mjsnew file mode 100644index 0000000..1e3fc51--- /dev/null+++ b/package/dashboard/test-ui/api-paths.check.mjs@@ -0,0 +1,38 @@+import assert from 'node:assert/strict';+import fs from 'node:fs';+import path from 'node:path';+import { test } from 'node:test';++// A value placed in an API path unencoded can carry `../` or `/`, and the browser resolves those before the+// request leaves: a run ID taken from the page URL could then reach a different same-origin endpoint.+const files = fs+ .readdirSync('ui', { recursive: true })+ .filter(file => /\.tsx?$/.test(file))+ .map(file => path.join('ui', file));+const constantChoice = /^[\w.]+ === '[^']*' \? '[^']*' : '[^']*'$/;++/** The `${…}` expressions in the path part (before any `?` outside an expression) of a template literal. */+function pathExpressions(literal) {+ const expressions = [...literal.matchAll(/\$\{((?:[^{}]|\{[^{}]*\})*)\}/g)];+ let masked = literal;+ for (const m of expressions) masked = masked.slice(0, m.index) + 'x'.repeat(m[0].length) + masked.slice(m.index + m[0].length);+ const query = masked.indexOf('?');+ return expressions.filter(m => query === -1 || m.index < query).map(m => m[1].trim());+}++test('every value in a website API path is encoded', () => {+ const unencoded = [];+ for (const file of files) {+ for (const [literal] of fs.readFileSync(file, 'utf8').matchAll(/`\/api\/[^`]*`/g))+ for (const expression of pathExpressions(literal))+ if (!expression.startsWith('encodeURIComponent(') && !constantChoice.test(expression))+ unencoded.push(`${file}: \${${expression}}`);+ }+ assert.deepEqual(unencoded, []);+});++test('the check reads path and query parts correctly', () => {+ assert.deepEqual(pathExpressions('`/api/runs/${encodeURIComponent(id)}/review?cursor=${cursor}`'), ['encodeURIComponent(id)']);+ assert.deepEqual(pathExpressions("`/api/keys/${id}${type === 'rotate' ? '/rotate' : ''}`"), ['id', "type === 'rotate' ? '/rotate' : ''"]);+ assert.ok(constantChoice.test("kind === 'workflow' ? 'workflows' : 'buckets'"));+});diff --git a/package/dashboard/ui/components/api-access.tsx b/package/dashboard/ui/components/api-access.tsxindex 1d48cab..7edc57b 100644--- a/package/dashboard/ui/components/api-access.tsx+++ b/package/dashboard/ui/components/api-access.tsx@@ -376,7 +376,7 @@ function KeyAction({...(['models:sync', 'workshop:run'].includes(scope) ? { device_id: device } : {}),}): await apiRequest<{ key: ApiKeyInfo; token?: string }>(- `/api/keys/${action.key.id}${action.type === 'rotate' ? '/rotate' : ''}`,+ `/api/keys/${encodeURIComponent(action.key.id)}${action.type === 'rotate' ? '/rotate' : ''}`,action.type === 'rename' ? 'PATCH' : action.type === 'revoke' ? 'DELETE' : 'POST',action.type === 'rename' ? { name } : undefined,);diff --git a/package/dashboard/ui/components/models/model-devices.tsx b/package/dashboard/ui/components/models/model-devices.tsxindex 26ebbdb..912f201 100644--- a/package/dashboard/ui/components/models/model-devices.tsx+++ b/package/dashboard/ui/components/models/model-devices.tsx@@ -217,7 +217,7 @@ export function DeploymentEditor({setBusy(true);setError('');try {- await apiRequest(`/api/models/devices/${device.id}/deployment`, 'PUT', {+ await apiRequest(`/api/models/devices/${encodeURIComponent(device.id)}/deployment`, 'PUT', {previous_revision: device.revision,models: selection,});diff --git a/package/dashboard/ui/features/workshop/components/image-inspector.tsx b/package/dashboard/ui/features/workshop/components/image-inspector.tsxindex 49aa5a5..545fc21 100644--- a/package/dashboard/ui/features/workshop/components/image-inspector.tsx+++ b/package/dashboard/ui/features/workshop/components/image-inspector.tsx@@ -40,7 +40,7 @@ export function ImageInspector({ item, imageUrl, onClose }: ImageInspectorProps)role="img"aria-label={focused && selected ? `Magnified ${selected.class_name} prediction` : 'Full camera frame'}>- <image href={imageUrl ?? `/api/workshop/items/${item.id}/image`} width={width} height={height} />+ <image href={imageUrl ?? `/api/workshop/items/${encodeURIComponent(item.id)}/image`} width={width} height={height} />{outlines &&annotations.map((a, index) =>diff --git a/package/dashboard/ui/features/workshop/components/review-frame.tsx b/package/dashboard/ui/features/workshop/components/review-frame.tsxindex f1ce895..b8fc524 100644--- a/package/dashboard/ui/features/workshop/components/review-frame.tsx+++ b/package/dashboard/ui/features/workshop/components/review-frame.tsx@@ -10,7 +10,7 @@ export function ReviewFrame({ item, imageUrl, temporary, selected = false, onSel<article data-selected={selected}><button type="button" className="wk-review-image" onClick={onInspect} aria-label={`Inspect frame at ${time}`}><img- src={imageUrl ?? `/api/workshop/items/${item.id}/image`}+ src={imageUrl ?? `/api/workshop/items/${encodeURIComponent(item.id)}/image`}alt={`Camera frame at ${item.captured_at}`}width={width}height={height}diff --git a/package/dashboard/ui/features/workshop/hooks/use-workshop-run.ts b/package/dashboard/ui/features/workshop/hooks/use-workshop-run.tsindex 233abd2..66e16c0 100644--- a/package/dashboard/ui/features/workshop/hooks/use-workshop-run.ts+++ b/package/dashboard/ui/features/workshop/hooks/use-workshop-run.ts@@ -28,7 +28,7 @@ export function useWorkshopRun(id: string) {let delay = 4000;try {const result = await apiRequest<WorkshopRunDetail>(- `/api/workshop/runs/${id}`,+ `/api/workshop/runs/${encodeURIComponent(id)}`,'GET',undefined,controller.signal,@@ -92,7 +92,7 @@ export function useWorkshopRun(id: string) {}, [id, revision]);async function cancel() {try {- await apiRequest(`/api/workshop/runs/${id}/cancel`, 'POST', {});+ await apiRequest(`/api/workshop/runs/${encodeURIComponent(id)}/cancel`, 'POST', {});refresh();} catch (e) {setError(e instanceof Error ? e.message : 'Could not cancel.');diff --git a/package/docs/check-links.mjs b/package/docs/check-links.mjsindex 9c5fbff..f977221 100644--- a/package/docs/check-links.mjs+++ b/package/docs/check-links.mjs@@ -12,9 +12,10 @@ const ids = new Map();const broken = [];for (const file of pages) {- const html = fs.readFileSync(file, 'utf8').replace(/<script[\s\S]*?<\/script>/g, '');+ const html = fs.readFileSync(file, 'utf8');for (const [, link] of html.matchAll(/<a [^>]*href="([^"]+)"/g)) {- if (/^(https?:|mailto:)/.test(link)) continue;+ // Script templates such as the search results' href="${r.url}" are filled in later, not links.+ if (/^(https?:|mailto:)/.test(link) || link.includes('${')) continue;const [target, anchor] = link.split('#');const resolved = target? path.join(site, target.startsWith('/') ? target : path.relative(site, path.join(path.dirname(file), target)))diff --git a/setup/lib.check.mjs b/setup/lib.check.mjsindex 0b4fc79..f1a160f 100644--- a/setup/lib.check.mjs+++ b/setup/lib.check.mjs@@ -63,3 +63,9 @@ test('team domains are normalized, and bad input is reported', () => {assert.ok(lib.problems.camera('Sky 01'));assert.ok(lib.problems.folder('footage'));});++test('camera stream URLs and their passwords are removed from error text', () => {+ const url = 'rtsp://admin:s3cret%[email protected]:554/stream1';+ assert.equal(lib.redactUrl(`${url}: Connection refused`, url), '<camera stream>: Connection refused');+ assert.equal(lib.redactUrl('rtsp://admin:[email protected]/x: 401 Unauthorized', url), 'rtsp://10.1.1.1/x: 401 Unauthorized');+});diff --git a/setup/lib.mjs b/setup/lib.mjsindex f77fe07..13d673e 100644--- a/setup/lib.mjs+++ b/setup/lib.mjs@@ -98,3 +98,8 @@ export const problems = {path: v => (v.startsWith('/') ? undefined : 'The path starts with /.'),address: v => (/^[A-Za-z0-9.:-]+$/.test(v) ? undefined : 'Enter the camera address, such as 192.168.1.20.'),};++/** Text from a tool that was given `url`, without the URL or any user:password@ in it. */+export function redactUrl(text, url) {+ return text.replaceAll(url, '<camera stream>').replace(/\/\/[^/@\s]*@/g, '//');+}diff --git a/setup/setup.mjs b/setup/setup.mjsindex 4f4e8ac..f958686 100644--- a/setup/setup.mjs+++ b/setup/setup.mjs@@ -65,28 +65,40 @@ process.on('SIGINT', () => {});/**- * Asks until the answer passes `problem`. Enter keeps the current value. Secrets are not echoed. With --yes,- * the current value is the answer, and `key` names where to put one when there is none.+ * Asks until the answer passes `problem`. Enter keeps the current value. With --yes, the current value is the+ * answer, and `key` names where to put one when there is none. Secrets go through askSecret instead.*/-async function ask(label, { initial = '', secret = false, problem, key } = {}) {+async function ask(label, { initial = '', problem, key } = {}) {if (assumeYes) {const issue = initial ? problem?.(initial) : `none is set; put it in ${key}`;if (issue) throw new Error(`${label}: ${issue}.`);- say(`${label}: ${secret ? dim('(from ' + key + ')') : initial}`);+ say(`${label}: ${initial}`);return initial;}for (;;) {- const current = initial ? dim(secret ? ' [Enter keeps the saved one]' : ` [${initial}]`) : '';- process.stdout.write(` ${label}${current}: `);- if (secret) echo(false);- const answer = (await nextLine().finally(() => secret && echo(true))).trim();- if (secret && process.stdin.isTTY) process.stdout.write('\n');- const value = answer || initial;+ process.stdout.write(` ${label}${initial ? dim(` [${initial}]`) : ''}: `);+ const value = (await nextLine()).trim() || initial;const issue = value ? problem?.(value) : 'An answer is needed.';if (!issue) return value;console.log(` ${red(issue)}`);}}+/** Like ask, for a secret: typing is hidden, and the value never reaches the output, even with --yes. */+async function askSecret(label, { initial = '', key } = {}) {+ if (assumeYes) {+ if (!initial) throw new Error(`${label}: none is set; put it in ${key}.`);+ say(`${label}: ${dim(`from ${key}`)}`);+ return initial;+ }+ for (;;) {+ process.stdout.write(` ${label}${initial ? dim(' [Enter keeps the saved one]') : ''}: `);+ echo(false);+ const answer = (await nextLine().finally(() => echo(true))).trim();+ if (process.stdin.isTTY) process.stdout.write('\n');+ if (answer || initial) return answer || initial;+ console.log(` ${red('An answer is needed.')}`);+ }+}async function confirm(label, yes = true) {if (assumeYes) {say(`${label} ${dim(yes ? 'yes' : 'no')}`);@@ -341,7 +353,7 @@ const steps = [say('It needs no public hostname. On the install page, copy the command: the long token after --token is what you need.');openUrl('https://one.dash.cloudflare.com/');for (;;) {- const token = await ask('Tunnel token', { secret: true, initial: process.env.STORE_TUNNEL_TOKEN, key: 'the environment variable STORE_TUNNEL_TOKEN' });+ const token = await askSecret('Tunnel token', { initial: process.env.STORE_TUNNEL_TOKEN, key: 'the environment variable STORE_TUNNEL_TOKEN' });try {tunnelId = lib.parseTunnelToken(token).tunnelId;const wrote = ssh(host, 'mkdir -p ~/.config/skyvision-store && umask 077 && cat > ~/.config/skyvision-store/tunnel.token', token.trim());@@ -479,7 +491,7 @@ const steps = [say('The camera must stream H.264 or H.265 over RTSP (see docs/requirements.md).');const ip = await ask('Camera address', { initial: env.CAMERA_IP_ADDRESS, problem: lib.problems.address, key: 'CAMERA_IP_ADDRESS' });const user = await ask('Camera user name', { initial: env.CAMERA_USERNAME || 'admin', key: 'CAMERA_USERNAME' });- const password = await ask('Camera password', { initial: env.CAMERA_PASSWORD, secret: true, key: 'CAMERA_PASSWORD' });+ const password = await askSecret('Camera password', { initial: env.CAMERA_PASSWORD, key: 'CAMERA_PASSWORD' });const port = await ask('RTSP port', { initial: env.CAMERA_RTSP_PORT || '554', problem: lib.problems.port, key: 'CAMERA_RTSP_PORT' });say(dim('Stream paths: Dahua /cam/realmonitor?channel=1&subtype=0 · Hikvision /Streaming/Channels/101 · Tapo /stream1'));const streamPath = await ask('RTSP path', { initial: env.CAMERA_RTSP_PATH || '/cam/realmonitor?channel=1&subtype=0', problem: lib.problems.path, key: 'CAMERA_RTSP_PATH' });@@ -492,7 +504,7 @@ const steps = [say(green(`The camera streams ${codec === 'hevc' ? 'H.265' : 'H.264'} at ${width}×${height}.`));break;}- tryAgain(probe.ok ? `The camera streams ${codec}; the relay records H.264 or H.265 only.` : `The relay cannot read the stream: ${probe.all.trim().split('\n').pop()}`);+ tryAgain(probe.ok ? `The camera streams ${codec}; the relay records H.264 or H.265 only.` : `The relay cannot read the stream: ${lib.redactUrl(probe.all.trim().split('\n').pop(), url)}`);}say('Installing the recorder and uploader…');if (!run(path.join(root, 'external/relay/deploy.sh'), [host]).ok) throw new Error('The relay install failed; see the output above.');@@ -526,7 +538,7 @@ const steps = [say('Setup uses it to register the processor; it is saved in .env for your own scripts too.');openUrl(site);if (assumeYes) throw new Error('OPENSKYCAM_AGENT_KEY is missing or refused: create an Agent key for every camera on the website and put it in .env.');- save({ OPENSKYCAM_AGENT_KEY: await ask('Agent key', { secret: true }) });+ save({ OPENSKYCAM_AGENT_KEY: await askSecret('Agent key') });}const host = await sshHost('SSH host of the processor machine', 'PROCESSOR_HOST');const python = ssh(host, 'PATH=/opt/homebrew/bin:/usr/local/bin:$PATH; command -v python3 && python3 -c "import sys; sys.exit(sys.version_info < (3, 11))"');@@ -540,7 +552,7 @@ const steps = [say('');say(`Create a second key: Purpose ${bold('Run Workshop jobs')}, device ${bold(device)}, every camera.`);openUrl(site);- const key = await ask('Run Workshop jobs key', { secret: true, initial: process.env.OPENSKYCAM_WORKSHOP_KEY, key: 'the environment variable OPENSKYCAM_WORKSHOP_KEY' });+ const key = await askSecret('Run Workshop jobs key', { initial: process.env.OPENSKYCAM_WORKSHOP_KEY, key: 'the environment variable OPENSKYCAM_WORKSHOP_KEY' });const me = json((await api('/v1/me', key)).text);if (me?.scope === 'workshop:run') {ssh(host, 'mkdir -p ~/.config/openskycam && umask 077 && cat > ~/.config/openskycam/workshop.key', key);