Skip to content

Record a sky camera around the clock, keep the footage on your own disk, and find what flew past.

Encode IDs in website API paths; keep secrets out of setup output (#6)

Hunter Deibler63a87f778d74
diff --git a/package/dashboard/test-ui/api-paths.check.mjs b/package/dashboard/test-ui/api-paths.check.mjs
new file mode 100644
index 0000000..1e3fc51
--- /dev/null
+++ b/package/dashboard/test-ui/api-paths.check.mjs
@@ -0,0 +1,38 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import path from 'node:path';
+import { test } from 'node:test';
+
+// A value placed in an API path unencoded can carry `../` or `/`, and the browser resolves those before the
+// request leaves: a run ID taken from the page URL could then reach a different same-origin endpoint.
+const files = fs
+ .readdirSync('ui', { recursive: true })
+ .filter(file => /\.tsx?$/.test(file))
+ .map(file => path.join('ui', file));
+const constantChoice = /^[\w.]+ === '[^']*' \? '[^']*' : '[^']*'$/;
+
+/** The `${…}` expressions in the path part (before any `?` outside an expression) of a template literal. */
+function pathExpressions(literal) {
+ const expressions = [...literal.matchAll(/\$\{((?:[^{}]|\{[^{}]*\})*)\}/g)];
+ let masked = literal;
+ for (const m of expressions) masked = masked.slice(0, m.index) + 'x'.repeat(m[0].length) + masked.slice(m.index + m[0].length);
+ const query = masked.indexOf('?');
+ return expressions.filter(m => query === -1 || m.index < query).map(m => m[1].trim());
+}
+
+test('every value in a website API path is encoded', () => {
+ const unencoded = [];
+ for (const file of files) {
+ for (const [literal] of fs.readFileSync(file, 'utf8').matchAll(/`\/api\/[^`]*`/g))
+ for (const expression of pathExpressions(literal))
+ if (!expression.startsWith('encodeURIComponent(') && !constantChoice.test(expression))
+ unencoded.push(`${file}: \${${expression}}`);
+ }
+ assert.deepEqual(unencoded, []);
+});
+
+test('the check reads path and query parts correctly', () => {
+ assert.deepEqual(pathExpressions('`/api/runs/${encodeURIComponent(id)}/review?cursor=${cursor}`'), ['encodeURIComponent(id)']);
+ assert.deepEqual(pathExpressions("`/api/keys/${id}${type === 'rotate' ? '/rotate' : ''}`"), ['id', "type === 'rotate' ? '/rotate' : ''"]);
+ assert.ok(constantChoice.test("kind === 'workflow' ? 'workflows' : 'buckets'"));
+});
diff --git a/package/dashboard/ui/components/api-access.tsx b/package/dashboard/ui/components/api-access.tsx
index 1d48cab..7edc57b 100644
--- a/package/dashboard/ui/components/api-access.tsx
+++ b/package/dashboard/ui/components/api-access.tsx
@@ -376,7 +376,7 @@ function KeyAction({
...(['models:sync', 'workshop:run'].includes(scope) ? { device_id: device } : {}),
})
: await apiRequest<{ key: ApiKeyInfo; token?: string }>(
- `/api/keys/${action.key.id}${action.type === 'rotate' ? '/rotate' : ''}`,
+ `/api/keys/${encodeURIComponent(action.key.id)}${action.type === 'rotate' ? '/rotate' : ''}`,
action.type === 'rename' ? 'PATCH' : action.type === 'revoke' ? 'DELETE' : 'POST',
action.type === 'rename' ? { name } : undefined,
);
diff --git a/package/dashboard/ui/components/models/model-devices.tsx b/package/dashboard/ui/components/models/model-devices.tsx
index 26ebbdb..912f201 100644
--- a/package/dashboard/ui/components/models/model-devices.tsx
+++ b/package/dashboard/ui/components/models/model-devices.tsx
@@ -217,7 +217,7 @@ export function DeploymentEditor({
setBusy(true);
setError('');
try {
- await apiRequest(`/api/models/devices/${device.id}/deployment`, 'PUT', {
+ await apiRequest(`/api/models/devices/${encodeURIComponent(device.id)}/deployment`, 'PUT', {
previous_revision: device.revision,
models: selection,
});
diff --git a/package/dashboard/ui/features/workshop/components/image-inspector.tsx b/package/dashboard/ui/features/workshop/components/image-inspector.tsx
index 49aa5a5..545fc21 100644
--- a/package/dashboard/ui/features/workshop/components/image-inspector.tsx
+++ b/package/dashboard/ui/features/workshop/components/image-inspector.tsx
@@ -40,7 +40,7 @@ export function ImageInspector({ item, imageUrl, onClose }: ImageInspectorProps)
role="img"
aria-label={focused && selected ? `Magnified ${selected.class_name} prediction` : 'Full camera frame'}
>
- <image href={imageUrl ?? `/api/workshop/items/${item.id}/image`} width={width} height={height} />
+ <image href={imageUrl ?? `/api/workshop/items/${encodeURIComponent(item.id)}/image`} width={width} height={height} />
{outlines &&
annotations.map(
(a, index) =>
diff --git a/package/dashboard/ui/features/workshop/components/review-frame.tsx b/package/dashboard/ui/features/workshop/components/review-frame.tsx
index f1ce895..b8fc524 100644
--- a/package/dashboard/ui/features/workshop/components/review-frame.tsx
+++ b/package/dashboard/ui/features/workshop/components/review-frame.tsx
@@ -10,7 +10,7 @@ export function ReviewFrame({ item, imageUrl, temporary, selected = false, onSel
<article data-selected={selected}>
<button type="button" className="wk-review-image" onClick={onInspect} aria-label={`Inspect frame at ${time}`}>
<img
- src={imageUrl ?? `/api/workshop/items/${item.id}/image`}
+ src={imageUrl ?? `/api/workshop/items/${encodeURIComponent(item.id)}/image`}
alt={`Camera frame at ${item.captured_at}`}
width={width}
height={height}
diff --git a/package/dashboard/ui/features/workshop/hooks/use-workshop-run.ts b/package/dashboard/ui/features/workshop/hooks/use-workshop-run.ts
index 233abd2..66e16c0 100644
--- a/package/dashboard/ui/features/workshop/hooks/use-workshop-run.ts
+++ b/package/dashboard/ui/features/workshop/hooks/use-workshop-run.ts
@@ -28,7 +28,7 @@ export function useWorkshopRun(id: string) {
let delay = 4000;
try {
const result = await apiRequest<WorkshopRunDetail>(
- `/api/workshop/runs/${id}`,
+ `/api/workshop/runs/${encodeURIComponent(id)}`,
'GET',
undefined,
controller.signal,
@@ -92,7 +92,7 @@ export function useWorkshopRun(id: string) {
}, [id, revision]);
async function cancel() {
try {
- await apiRequest(`/api/workshop/runs/${id}/cancel`, 'POST', {});
+ await apiRequest(`/api/workshop/runs/${encodeURIComponent(id)}/cancel`, 'POST', {});
refresh();
} catch (e) {
setError(e instanceof Error ? e.message : 'Could not cancel.');
diff --git a/package/docs/check-links.mjs b/package/docs/check-links.mjs
index 9c5fbff..f977221 100644
--- a/package/docs/check-links.mjs
+++ b/package/docs/check-links.mjs
@@ -12,9 +12,10 @@ const ids = new Map(
);
const broken = [];
for (const file of pages) {
- const html = fs.readFileSync(file, 'utf8').replace(/<script[\s\S]*?<\/script>/g, '');
+ const html = fs.readFileSync(file, 'utf8');
for (const [, link] of html.matchAll(/<a [^>]*href="([^"]+)"/g)) {
- if (/^(https?:|mailto:)/.test(link)) continue;
+ // Script templates such as the search results' href="${r.url}" are filled in later, not links.
+ if (/^(https?:|mailto:)/.test(link) || link.includes('${')) continue;
const [target, anchor] = link.split('#');
const resolved = target
? path.join(site, target.startsWith('/') ? target : path.relative(site, path.join(path.dirname(file), target)))
diff --git a/setup/lib.check.mjs b/setup/lib.check.mjs
index 0b4fc79..f1a160f 100644
--- a/setup/lib.check.mjs
+++ b/setup/lib.check.mjs
@@ -63,3 +63,9 @@ test('team domains are normalized, and bad input is reported', () => {
assert.ok(lib.problems.camera('Sky 01'));
assert.ok(lib.problems.folder('footage'));
});
+
+test('camera stream URLs and their passwords are removed from error text', () => {
+ const url = 'rtsp://admin:s3cret%[email protected]:554/stream1';
+ assert.equal(lib.redactUrl(`${url}: Connection refused`, url), '<camera stream>: Connection refused');
+ assert.equal(lib.redactUrl('rtsp://admin:[email protected]/x: 401 Unauthorized', url), 'rtsp://10.1.1.1/x: 401 Unauthorized');
+});
diff --git a/setup/lib.mjs b/setup/lib.mjs
index f77fe07..13d673e 100644
--- a/setup/lib.mjs
+++ b/setup/lib.mjs
@@ -98,3 +98,8 @@ export const problems = {
path: v => (v.startsWith('/') ? undefined : 'The path starts with /.'),
address: v => (/^[A-Za-z0-9.:-]+$/.test(v) ? undefined : 'Enter the camera address, such as 192.168.1.20.'),
};
+
+/** Text from a tool that was given `url`, without the URL or any user:password@ in it. */
+export function redactUrl(text, url) {
+ return text.replaceAll(url, '<camera stream>').replace(/\/\/[^/@\s]*@/g, '//');
+}
diff --git a/setup/setup.mjs b/setup/setup.mjs
index 4f4e8ac..f958686 100644
--- a/setup/setup.mjs
+++ b/setup/setup.mjs
@@ -65,28 +65,40 @@ process.on('SIGINT', () => {
});
/**
- * Asks until the answer passes `problem`. Enter keeps the current value. Secrets are not echoed. With --yes,
- * the current value is the answer, and `key` names where to put one when there is none.
+ * Asks until the answer passes `problem`. Enter keeps the current value. With --yes, the current value is the
+ * answer, and `key` names where to put one when there is none. Secrets go through askSecret instead.
*/
-async function ask(label, { initial = '', secret = false, problem, key } = {}) {
+async function ask(label, { initial = '', problem, key } = {}) {
if (assumeYes) {
const issue = initial ? problem?.(initial) : `none is set; put it in ${key}`;
if (issue) throw new Error(`${label}: ${issue}.`);
- say(`${label}: ${secret ? dim('(from ' + key + ')') : initial}`);
+ say(`${label}: ${initial}`);
return initial;
}
for (;;) {
- const current = initial ? dim(secret ? ' [Enter keeps the saved one]' : ` [${initial}]`) : '';
- process.stdout.write(` ${label}${current}: `);
- if (secret) echo(false);
- const answer = (await nextLine().finally(() => secret && echo(true))).trim();
- if (secret && process.stdin.isTTY) process.stdout.write('\n');
- const value = answer || initial;
+ process.stdout.write(` ${label}${initial ? dim(` [${initial}]`) : ''}: `);
+ const value = (await nextLine()).trim() || initial;
const issue = value ? problem?.(value) : 'An answer is needed.';
if (!issue) return value;
console.log(` ${red(issue)}`);
}
}
+/** Like ask, for a secret: typing is hidden, and the value never reaches the output, even with --yes. */
+async function askSecret(label, { initial = '', key } = {}) {
+ if (assumeYes) {
+ if (!initial) throw new Error(`${label}: none is set; put it in ${key}.`);
+ say(`${label}: ${dim(`from ${key}`)}`);
+ return initial;
+ }
+ for (;;) {
+ process.stdout.write(` ${label}${initial ? dim(' [Enter keeps the saved one]') : ''}: `);
+ echo(false);
+ const answer = (await nextLine().finally(() => echo(true))).trim();
+ if (process.stdin.isTTY) process.stdout.write('\n');
+ if (answer || initial) return answer || initial;
+ console.log(` ${red('An answer is needed.')}`);
+ }
+}
async function confirm(label, yes = true) {
if (assumeYes) {
say(`${label} ${dim(yes ? 'yes' : 'no')}`);
@@ -341,7 +353,7 @@ const steps = [
say('It needs no public hostname. On the install page, copy the command: the long token after --token is what you need.');
openUrl('https://one.dash.cloudflare.com/');
for (;;) {
- const token = await ask('Tunnel token', { secret: true, initial: process.env.STORE_TUNNEL_TOKEN, key: 'the environment variable STORE_TUNNEL_TOKEN' });
+ const token = await askSecret('Tunnel token', { initial: process.env.STORE_TUNNEL_TOKEN, key: 'the environment variable STORE_TUNNEL_TOKEN' });
try {
tunnelId = lib.parseTunnelToken(token).tunnelId;
const wrote = ssh(host, 'mkdir -p ~/.config/skyvision-store && umask 077 && cat > ~/.config/skyvision-store/tunnel.token', token.trim());
@@ -479,7 +491,7 @@ const steps = [
say('The camera must stream H.264 or H.265 over RTSP (see docs/requirements.md).');
const ip = await ask('Camera address', { initial: env.CAMERA_IP_ADDRESS, problem: lib.problems.address, key: 'CAMERA_IP_ADDRESS' });
const user = await ask('Camera user name', { initial: env.CAMERA_USERNAME || 'admin', key: 'CAMERA_USERNAME' });
- const password = await ask('Camera password', { initial: env.CAMERA_PASSWORD, secret: true, key: 'CAMERA_PASSWORD' });
+ const password = await askSecret('Camera password', { initial: env.CAMERA_PASSWORD, key: 'CAMERA_PASSWORD' });
const port = await ask('RTSP port', { initial: env.CAMERA_RTSP_PORT || '554', problem: lib.problems.port, key: 'CAMERA_RTSP_PORT' });
say(dim('Stream paths: Dahua /cam/realmonitor?channel=1&subtype=0 · Hikvision /Streaming/Channels/101 · Tapo /stream1'));
const streamPath = await ask('RTSP path', { initial: env.CAMERA_RTSP_PATH || '/cam/realmonitor?channel=1&subtype=0', problem: lib.problems.path, key: 'CAMERA_RTSP_PATH' });
@@ -492,7 +504,7 @@ const steps = [
say(green(`The camera streams ${codec === 'hevc' ? 'H.265' : 'H.264'} at ${width}×${height}.`));
break;
}
- tryAgain(probe.ok ? `The camera streams ${codec}; the relay records H.264 or H.265 only.` : `The relay cannot read the stream: ${probe.all.trim().split('\n').pop()}`);
+ tryAgain(probe.ok ? `The camera streams ${codec}; the relay records H.264 or H.265 only.` : `The relay cannot read the stream: ${lib.redactUrl(probe.all.trim().split('\n').pop(), url)}`);
}
say('Installing the recorder and uploader…');
if (!run(path.join(root, 'external/relay/deploy.sh'), [host]).ok) throw new Error('The relay install failed; see the output above.');
@@ -526,7 +538,7 @@ const steps = [
say('Setup uses it to register the processor; it is saved in .env for your own scripts too.');
openUrl(site);
if (assumeYes) throw new Error('OPENSKYCAM_AGENT_KEY is missing or refused: create an Agent key for every camera on the website and put it in .env.');
- save({ OPENSKYCAM_AGENT_KEY: await ask('Agent key', { secret: true }) });
+ save({ OPENSKYCAM_AGENT_KEY: await askSecret('Agent key') });
}
const host = await sshHost('SSH host of the processor machine', 'PROCESSOR_HOST');
const python = ssh(host, 'PATH=/opt/homebrew/bin:/usr/local/bin:$PATH; command -v python3 && python3 -c "import sys; sys.exit(sys.version_info < (3, 11))"');
@@ -540,7 +552,7 @@ const steps = [
say('');
say(`Create a second key: Purpose ${bold('Run Workshop jobs')}, device ${bold(device)}, every camera.`);
openUrl(site);
- const key = await ask('Run Workshop jobs key', { secret: true, initial: process.env.OPENSKYCAM_WORKSHOP_KEY, key: 'the environment variable OPENSKYCAM_WORKSHOP_KEY' });
+ const key = await askSecret('Run Workshop jobs key', { initial: process.env.OPENSKYCAM_WORKSHOP_KEY, key: 'the environment variable OPENSKYCAM_WORKSHOP_KEY' });
const me = json((await api('/v1/me', key)).text);
if (me?.scope === 'workshop:run') {
ssh(host, 'mkdir -p ~/.config/openskycam && umask 077 && cat > ~/.config/openskycam/workshop.key', key);