Skip to content

Record a sky camera around the clock, keep the footage on your own disk, and find what flew past.

Fix bugs found in a review of the whole codebase (#11)

Hunter Deibler561e80d34be8
diff --git a/.env.example b/.env.example
index 11928c9..a12eee1 100644
--- a/.env.example
+++ b/.env.example
@@ -34,7 +34,7 @@ ACCESS_TEAM_DOMAIN=https://<team>.cloudflareaccess.com
ACCESS_AUD=<application-audience-tag>
# Comma-separated camera ids the relay may upload for (CAM_ID above is one of them).
CAMERAS=sky01
-# The Workers VPC service in front of the footage store's tunnel (npx wrangler vpc service list).
+# The Workers VPC service in front of the footage store's tunnel (npm run wrangler -- vpc service list).
STORE_SERVICE_ID=<vpc-service-id>
# Optional for deploys (the database is found by name); needed by `npm run dev`, which uses your real
# catalog: npm run wrangler -- d1 info skyvision-catalog
@@ -42,7 +42,8 @@ D1_DATABASE_ID=
# Optional: the host for the documentation site (package/docs), if you publish it with `npm run deploy` there.
DOCS_HOST=
-# Written by the guided setup (npm run setup), which also fills in everything above. Hosts are SSH hosts.
+# Written by the guided setup (npm run setup), which also fills in everything above except CAMERA_SCHEME,
+# CAMERA_PORT, MIN_FREE_GB and DOCS_HOST (their defaults apply). Hosts are SSH hosts.
# Cloudflare account, when your login has more than one.
CLOUDFLARE_ACCOUNT_ID=
# The footage store: its SSH host, footage folder, whether that folder is on the system disk, and its tunnel.
diff --git a/README.md b/README.md
index daf8c8f..2aab474 100644
--- a/README.md
+++ b/README.md
@@ -82,7 +82,7 @@ You need an RTSP camera, a Linux machine near it for the relay (a Raspberry Pi i
curl -fsSL https://raw.githubusercontent.com/Deibler/openskycam/main/setup/install.sh | bash
```
-It checks your tools, downloads openskycam to `~/openskycam` and starts a guided setup. The setup signs in to Cloudflare, creates the database and buckets, installs the footage store and the relay over SSH, deploys both Workers, and waits for the first recording to arrive. For the two things only the Cloudflare dashboard can do, a tunnel and the sign-in application, it explains each click and checks what you paste. Answers are saved in `.env`, so you can stop and run `npm run setup` again; `npm run setup -- --check` checks a running deployment and changes nothing.
+It checks your tools, downloads openskycam to `~/openskycam` and starts a guided setup. The setup signs in to Cloudflare, creates the database and buckets, installs the footage store and the relay over SSH, deploys both Workers, and waits for the first recording to arrive. For the things only the Cloudflare dashboard can do, a tunnel, the sign-in application and a rule that turns off Browser Integrity Check on the API host, it explains each click and checks the result. Answers are saved in `.env`, so you can stop and run `npm run setup` again; `npm run setup -- --check` checks a running deployment and changes nothing.
[Deploy your own](https://docs.openskycam.org/deploy/) explains every step the setup runs, if you would rather do it by hand. To work on the code:
@@ -108,7 +108,7 @@ openskycam is built for an AI agent to operate:
You ask in plain words: *"Did the camera record all night?"*, *"Find last night's meteors and show me the clearest three"*, *"Build a workflow that keeps moving planes between 02:00 and 09:30 UTC and run it every night"*, *"Review the waiting images in Planes"*. Three steps need a person:
- the first Cloudflare login;
-- creating the tunnel and the sign-in application in the Cloudflare dashboard;
+- creating the tunnel, the sign-in application and the Browser Integrity Check rule in the Cloudflare dashboard;
- creating API keys on the website.
[Run it with an agent](https://docs.openskycam.org/agents/) maps each request to the tool that answers it.
diff --git a/camera_mcp.py b/camera_mcp.py
index ec471a8..8c16a1d 100644
--- a/camera_mcp.py
+++ b/camera_mcp.py
@@ -8,6 +8,7 @@ import base64
import json
import os
import re
+import shlex
import ssl
import sys
import urllib.request
@@ -66,10 +67,21 @@ def load_env(path=ENV_PATH):
if not line or line.startswith("#") or "=" not in line:
continue
key, value = line.split("=", 1)
- env[key.strip()] = value.strip().strip('"').strip("'")
+ env[key.strip().removeprefix("export ").strip()] = env_value(value)
return env
+def env_value(raw):
+ """A value as bash reads it: the setup writes 'it'\\''s' for it's, and # starts a comment only as its own word."""
+ if raw[:1].isspace() and raw.lstrip().startswith("#"):
+ return ""
+ try:
+ words = shlex.split(raw)
+ except ValueError:
+ return raw.strip()
+ return words[0] if words else ""
+
+
def auth_key(key):
low = key.lower()
if "login" in low or any(word in low for word in SECRET):
@@ -116,7 +128,8 @@ class Camera:
scheme = env.get("CAMERA_SCHEME", "http")
if scheme not in ("http", "https"):
raise ValueError("CAMERA_SCHEME must be http or https")
- self.base = f"{scheme}://{env['CAMERA_IP_ADDRESS']}:{env['CAMERA_PORT']}"
+ port = env.get("CAMERA_PORT") or (443 if scheme == "https" else 80)
+ self.base = f"{scheme}://{env['CAMERA_IP_ADDRESS']}:{port}"
passwords = urllib.request.HTTPPasswordMgrWithDefaultRealm()
passwords.add_password(None, self.base, env["CAMERA_USERNAME"], self.password)
# ponytail: camera cert is self-signed; pin it if the wrong device answering matters
@@ -232,6 +245,9 @@ def demo():
else:
raise SystemExit("auth key was accepted")
assert "secret" not in redact("table.X.Password=secret\ntable.X.BitRate=1", "secret")
+ assert env_value("'it'\\''s-ok'") == "it's-ok" and env_value('"a b"') == "a b" and env_value("") == ""
+ assert env_value("Sky#2026") == "Sky#2026" and env_value("abc # note") == "abc" and env_value("'a # b' # x") == "a # b"
+ assert Camera({"CAMERA_PASSWORD": "p", "CAMERA_USERNAME": "u", "CAMERA_IP_ADDRESS": "10.0.0.2"}).base == "http://10.0.0.2:80"
print("ok")
diff --git a/docs/agents.md b/docs/agents.md
index df5d168..0a97773 100644
--- a/docs/agents.md
+++ b/docs/agents.md
@@ -6,7 +6,7 @@ description: An AI agent can set openskycam up, run it day to day and change its
openskycam is built so an AI agent can do the work. Every action on the website is also an API call under `/v1`. An [MCP server](api/mcp.md) gives agents seven tools that cover the website, with guides that teach them how to use them. The setup runs unattended from `.env`, and `make check` verifies any change an agent makes to the code.
-A person is needed for three things, all by design: creating API keys on the website, creating the tunnel and the sign-in application in the Cloudflare dashboard, and the first Cloudflare login in a browser. [What needs you](#what-needs-you) lists them.
+A person is needed for three things, all by design: creating API keys on the website, creating the tunnel, the sign-in application and the Browser Integrity Check rule in the Cloudflare dashboard, and the first Cloudflare login in a browser. [What needs you](#what-needs-you) lists them.
## What you can ask
diff --git a/docs/api.md b/docs/api.md
index c4dff0e..ae75dbf 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -58,7 +58,7 @@ Endpoints that read footage or images check the camera of what they return; a ke
Some answers contain download links that work without the bearer header, so they can be passed to `curl` or a browser: video, full-size images, photos and dataset manifests from the MCP server. A link is `https://<api-host><path>?key=<key id>&expires=<unix seconds>&sig=<HMAC>`:
-- It is valid for one path, for `GET` and `HEAD` only, and with exactly those three query parameters.
+- It is valid for one path, for `GET` (and `HEAD` on a video file) only, and with exactly those three query parameters.
- It lasts one hour, or until the key expires if that is sooner.
- It is signed with the key's secret hash, so revoking or rotating the key ends every link it made.
@@ -128,7 +128,9 @@ Workshop is one document with a revision number. Every write that changes it tak
| Most JSON bodies | 4,096 bytes |
| `PUT /v1/workshop` | 800,000 bytes; the saved document must stay under 750,000 bytes |
| `POST /v1/mcp` | 1,000,000 bytes |
-| `POST /v1/models` | 65,536 bytes |
+| `POST /v1/workshop/jobs/{id}/complete` | 8 MiB |
+| `POST /v1/workshop/jobs/{id}/progress` | 1 MiB |
+| `POST /v1/models`, `PUT /v1/workshop/items/{id}/review` | 65,536 bytes |
| `PUT /v1/models/devices/{id}/deployment`, `POST /v1/models/artifacts` | 32,768 bytes |
| `PUT /v1/models/chunks/{sha}`, `PUT /v1/workshop/jobs/{id}/images/{sha}` | 8 MiB |
diff --git a/docs/api/workshop.md b/docs/api/workshop.md
index 0e58858..1a0dd89 100644
--- a/docs/api/workshop.md
+++ b/docs/api/workshop.md
@@ -151,7 +151,7 @@ A full run's frames are bucket items, at `GET /v1/workshop/items/{id}/image`. A
| `GET /v1/workshop/items/{id}` | One image with its `recording`, `run` (`id`, `name`, `created_at`) and the model's `classes` |
| `GET /v1/workshop/items/{id}/thumbnail` | A small JPEG: `size=256` (default) or `512`, `mode=frame` (default) or `object` to crop to the detections. Supports `If-None-Match`. |
-`browse` filters: `run`, `camera`, `class`, `review` (`pending`, `accepted` or `rejected`), `saved=true`, `q` (text, up to 200 characters), `from` and `to` (capture time), `min_confidence` and `max_confidence` (0 to 1), `min_objects` and `max_objects` (0 to 100), `sort` (`newest` or `oldest`), `limit` (1 to 200, default 120) and `cursor` from the previous page.
+`browse` filters: `run`, `camera`, `class`, `review` (`pending`, `accepted` or `rejected`), `saved=true`, `q` (text, up to 200 characters), `from` and `to` (capture time), `min_confidence` and `max_confidence` (0 to 1), `min_objects` and `max_objects` (0 to 100; `max_objects=100`, the default, means 100 or more), `sort` (`newest` or `oldest`), `limit` (1 to 200, default 120) and `cursor` from the previous page.
### Review an image
@@ -175,7 +175,7 @@ PUT /v1/workshop/items/{id}/review
| `confidence` | 0 to 1 |
| `box` | Detection models only: `[left, top, right, bottom]` as fractions of the frame, left below right and top below bottom |
-At most 100 labels, and the request body is limited to 4,096 bytes. A label that differs from the model's is marked `edited`. A corrected box keeps the `track_id` of the model box it matches or overlaps by more than half, so trajectories survive review. The answer echoes `review`, and `annotations` when sent.
+At most 100 labels, and the request body is limited to 65,536 bytes. A label that differs from the model's is marked `edited`. A corrected box keeps the `track_id` of the model box it matches or overlaps by more than half, so trajectories survive review. The answer echoes `review`, and `annotations` when sent.
### Keep an image
diff --git a/docs/capture/ingest.md b/docs/capture/ingest.md
index 9d54d43..06ac1cd 100644
--- a/docs/capture/ingest.md
+++ b/docs/capture/ingest.md
@@ -35,7 +35,7 @@ Everything is checked before any storage call, in this order:
| Check | Failure |
|---|---|
-| `:cam` is in the `CAMERAS` list | `400 unknown camera: <cam>` |
+| `:cam` is in the `CAMERAS` list | `403 unknown camera: <cam>` |
| `:start` is a real UTC time in the form `YYYYMMDDTHHMMSSZ`. Impossible values such as month 13 are refused. | `400 start must look like 20260922T140500Z` |
| `:start` is no more than 40 days old and no more than 5 minutes ahead of the Worker's clock | `400 start is outside the retention window` |
| `Content-Length` is present and positive | `411` |
diff --git a/docs/capture/relay.md b/docs/capture/relay.md
index 1488b91..7ed28f7 100644
--- a/docs/capture/relay.md
+++ b/docs/capture/relay.md
@@ -66,7 +66,7 @@ ffmpeg -nostdin -hide_banner -loglevel warning \
| Option | Why |
|---|---|
| `-rtsp_transport tcp` | RTSP over UDP drops packets across the internet |
-| `-timeout 15000000` | Give up on a silent camera after 15 seconds (the value is in microseconds), so systemd restarts the recorder |
+| `-timeout 15000000` | Give up on a silent camera after 15 seconds (the value is in microseconds), so systemd restarts the recorder. ffmpeg 4 (Ubuntu 22.04, Debian 11) names this option `-stimeout`, and `record.sh` uses that name there |
| `-c copy` | No re-encoding: the stored bytes are what the camera encoded, so image quality is set on the camera only |
| `-tag:v hvc1` (H.265 only) | Safari and QuickTime refuse HEVC tagged `hev1`. Only the sample-entry label changes; decoded frames are identical. ffmpeg refuses the tag on H.264, which keeps `avc1`. |
| `-segment_time 60 -segment_atclocktime 1` | One file per minute, cut on the minute. The cut lands on the first keyframe after it, so files run about 59.4 to 60.4 seconds and no frames are lost between them. |
@@ -75,7 +75,7 @@ ffmpeg -nostdin -hide_banner -loglevel warning \
Segments are written to `/dev/shm/skyvision`, which is RAM, sparing the SD card 20 to 45 GB of writes a day while uploads keep up.
-The unit restarts `ffmpeg` whenever it exits (`Restart=always`). The delay grows from 2 seconds to at most 10 (`RestartSec=2`, `RestartSteps=5`, `RestartMaxDelaySec=10`), so a long camera outage does not cause a tight loop. `StartLimitIntervalSec=0` means systemd never stops retrying. A crash costs about 5 seconds of footage: the cut-off file keeps what was written, and the restart takes about 4 seconds.
+The unit restarts `ffmpeg` whenever it exits (`Restart=always`). The delay grows from 2 seconds to at most 10 (`RestartSec=2`, `RestartSteps=5`, `RestartMaxDelaySec=10`), so a long camera outage does not cause a tight loop. The growing delay needs systemd 254 or later; older systemd (Debian 12, Raspberry Pi OS Bookworm, Ubuntu 22.04) ignores those two keys and restarts every 2 seconds. `StartLimitIntervalSec=0` means systemd never stops retrying. A crash costs about 5 seconds of footage: the cut-off file keeps what was written, and the restart takes about 4 seconds.
## Uploading
@@ -84,7 +84,7 @@ The unit restarts `ffmpeg` whenever it exits (`Restart=always`). The delay grows
1. **Keep RAM small.** At most 3 finished segments stay in RAM. Older ones move to `~/skyvision/spool/pending/` on disk. A full RAM disk stops the recorder with "No space left on device" and loses footage.
2. **Upload a batch.** The next 3 files are uploaded in parallel: finished segments in RAM first, then the backlog in `pending/`, oldest first. One upload through the Worker often ran at about 4 Mbit/s, the camera's own rate, while three at once reached 24 to 28 Mbit/s together.
3. **Act on each response**, as shown in the table below.
-4. **Guard the disk.** While the spool's file system has less than `MIN_FREE_GB` GiB free, the oldest file in `pending/` is deleted. Other services on the host need the disk; a log line names each dropped file.
+4. **Guard the disk.** While the spool's file system has less than `MIN_FREE_GB` GiB free, the oldest file in `quarantine/` is deleted, then the oldest in `pending/`. Other services on the host need the disk; a log line names each dropped file.
5. **Sleep.** After a batch with no "unavailable" result, the loop runs again at once. Otherwise, or when there is nothing to upload, it waits 10 seconds.
For each file, the uploader:
@@ -107,10 +107,10 @@ The token reaches `curl` on stdin as a config line (`-K -`), so it never appears
| Worker response | Meaning | What the relay does |
|---|---|---|
| `200`, `201` | Stored (`200`: these exact bytes were already stored) | Deletes the local file |
-| `000`, `401`, `403`, `404`, `408`, `429`, `5xx` | Worker unreachable, not deployed, wrong token, rate-limited, or storage unavailable | Keeps the file. If any upload in the batch got one of these, it moves every finished RAM segment to `pending/` and waits 10 seconds. |
+| `000`, `401`, `403`, `404`, `408`, `429`, `5xx` | Worker unreachable, not deployed, wrong token, camera not in the Worker's `CAMERAS`, rate-limited, or storage unavailable | Keeps the file. If any upload in the batch got one of these, it moves every finished RAM segment to `pending/` and waits 10 seconds. |
| Anything else (`400`, `409`, `411`, `413`, `415`, `422`, …) | A problem with this file | Moves it to `quarantine/` and continues |
-`401` and `403` count as "unavailable" so that a wrong or missing token never discards footage. The backlog grows until you fix the token. A `workers.dev` URL with no Worker behind it answers `404`, so nothing is quarantined before the Worker exists.
+`401` and `403` count as "unavailable" so that a wrong or missing token, or a `CAM_ID` the deployed Worker does not list in `CAMERAS`, never discards footage. The backlog grows until you fix the token. A `workers.dev` URL with no Worker behind it answers `404`, so nothing is quarantined before the Worker exists.
The upload service runs at `Nice=10` and restarts 10 seconds after any exit.
@@ -120,7 +120,7 @@ The upload service runs at `Nice=10` and restarts 10 seconds after any exit.
|---|---|
| `/dev/shm/skyvision/` | Being recorded, or one of the next 3 finished segments to upload |
| `~/skyvision/spool/pending/` | Waiting: the Worker was unavailable or uploads fell behind. Retried oldest first. |
-| `~/skyvision/spool/quarantine/` | Unreadable, or rejected by the Worker. Kept for inspection, never retried or cleaned up. |
+| `~/skyvision/spool/quarantine/` | Unreadable, or rejected by the Worker. Kept for inspection and never retried. Deleted, oldest first, only when the disk runs low, before any backlog. |
A day of outage at 4 Mbit/s is about 43 GB of spool (4 Mbit/s × 86,400 s). The free disk divided by that is how long the relay can hold a backlog; past `MIN_FREE_GB`, the oldest footage goes first.
diff --git a/docs/capture/store.md b/docs/capture/store.md
index c077eda..c5245a8 100644
--- a/docs/capture/store.md
+++ b/docs/capture/store.md
@@ -61,7 +61,7 @@ The website asks for small JPEGs of Workshop images, and agents for previews, so
- **Without one**, the whole image is shrunk to fit and never enlarged.
- `size` is 1 to 2048.
- The output is JPEG quality 75. A 4K frame is decoded at reduced size (JPEG DCT scaling), so a thumbnail is fast.
-- Pillow is imported on the first thumbnail request, so recording never depends on it. Install it with `python3 -m pip install --user pillow` or your distribution's `python3-pil`.
+- Pillow is imported on the first thumbnail request, so recording never depends on it. Install your distribution's `python3-pil` package, or `brew install pillow` with Homebrew on macOS.
## Retention
diff --git a/docs/configuration.md b/docs/configuration.md
index 745e845..21d05ca 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -27,7 +27,7 @@ Copy `.env.example` to `.env`. Three programs read it: bash (`make deploy`, `mak
| `CAMERA_RTSP_PORT` | `554` | relay | RTSP port |
| `CAMERA_RTSP_PATH` | `/cam/realmonitor?channel=1&subtype=0` | relay | Stream path after the port. The default is the Dahua main stream |
| `CAMERA_SCHEME` | `http` | camera MCP | `http` or `https`, for the camera's web API |
-| `CAMERA_PORT` | none | camera MCP | The camera's web API port |
+| `CAMERA_PORT` | `80`, or `443` with `https` | camera MCP | The camera's web API port |
### Relay
@@ -55,7 +55,7 @@ Variables already set in the environment win over `.env`. In Workers Builds, set
### Guided setup
-[`npm run setup`](deploy.md#guided-setup) writes every key above and these, which only it reads. They let it check and redo steps later.
+[`npm run setup`](deploy.md#guided-setup) writes every key above except `CAMERA_SCHEME`, `CAMERA_PORT`, `MIN_FREE_GB` and `DOCS_HOST`, which keep their defaults, and these, which only it reads. They let it check and redo steps later.
| Key | Meaning |
|---|---|
@@ -209,4 +209,4 @@ The installers take absolute paths and install a service that survives reboots:
| `install-workshop.sh` | launchd agent `org.openskycam.workshop`, `Nice` 10 | systemd user service `skyvision-workshop`, `Restart=always`, `Nice=10` |
| `install-model-sync.sh` | launchd agent `org.openskycam.models.<device>`, every 60 s | systemd user timer `skyvision-models-<device>.timer`, every 60 s |
-Both installers check their arguments before writing anything. The runtime ID must look like `workshop-` followed by 32 hex characters, the origin must be HTTPS, and the key file must be mode 600. `install-model-sync.sh` runs one sync first, to prove the interpreter, key and assignment work. The optional dependency directory is added to `PYTHONPATH`. On Linux, run `loginctl enable-linger $USER` once so the services keep running after you log out.
+Both installers check their arguments before writing anything. The runtime ID must look like `workshop-` followed by 32 hex characters, the origin must be HTTPS with no path, and the key file must be mode 600. `install-workshop.sh` also checks that the Python imports `av`, `cv2`, `torch` and `ultralytics`. `install-model-sync.sh` runs one sync first, to prove the interpreter, key and assignment work. The optional dependency directory is added to `PYTHONPATH`. On Linux, each installer stops with an error if systemd cannot load the unit it wrote. Run `loginctl enable-linger $USER` once so the services keep running after you log out.
diff --git a/docs/deploy.md b/docs/deploy.md
index f3a3937..4d9b9c5 100644
--- a/docs/deploy.md
+++ b/docs/deploy.md
@@ -20,7 +20,7 @@ curl -fsSL https://raw.githubusercontent.com/Deibler/openskycam/main/setup/insta
The installer checks for Git, Node.js 24, Python 3 and SSH, downloads openskycam to `~/openskycam` (or `OPENSKYCAM_DIR`), installs its dependencies and starts `npm run setup`. In a clone you already have, run `npm install && npm run setup`.
-The setup does each step a command can do: it signs in to Cloudflare, creates the database, buckets and VPC service, installs the footage store and the relay over SSH, deploys both Workers with their secrets, and checks the camera's stream before recording it. Some steps happen only in a browser: creating the tunnel and the Access application in the Cloudflare dashboard, and keys on your website. For those it says what to click, opens the page, and checks what you paste. It saves every answer to `.env` and ends each step with the check this page describes, so a step either works or tells you why not.
+The setup does each step a command can do: it signs in to Cloudflare, creates the database, buckets and VPC service, installs the footage store and the relay over SSH, deploys both Workers with their secrets, and checks the camera's stream before recording it. Some steps happen only in a browser: creating the tunnel, the Access application and the Browser Integrity Check rule in the Cloudflare dashboard, and keys on your website. For those it says what to click, opens the page, and checks what you paste. It saves every answer to `.env` and ends each step with the check this page describes, so a step either works or tells you why not.
| Command | What it does |
|---|---|
@@ -31,7 +31,7 @@ The setup does each step a command can do: it signs in to Cloudflare, creates th
The setup needs key-based SSH login to each host (`ssh-copy-id <host>`). On Linux hosts it offers to run `loginctl enable-linger` and to install missing packages with `sudo`, which ask for your password there.
-The steps below are what the setup runs, for doing it by hand.
+The steps below do by hand what the setup does. The processor's folders differ: the setup puts each version in `~/openskycam-processor/<commit>/`, shares one virtual environment in `~/openskycam-processor/.venv`, and keeps the key in `~/.config/openskycam/workshop.key`.
## 1. Clone and check
@@ -69,7 +69,7 @@ Check: `npm run wrangler -- d1 list` and `npm run wrangler -- r2 bucket list` sh
The store is a small HTTP server on the host with the big disk. It listens on `127.0.0.1:8096` only; the Workers reach it through a Cloudflare Tunnel. See [Footage store](capture/store.md).
-1. **Prepare the host.** Install Python 3.10 or later, [`cloudflared`](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/downloads/) and `openssl`. Mount the footage disk. The footage root should be a mount point, such as `/mnt/footage`: the store refuses uploads while it is not mounted. For a folder on the system disk, add `--system-disk` to the install command below. For thumbnails, install Pillow for that `python3` (`python3 -m pip install --user pillow`, or the `python3-pil` package).
+1. **Prepare the host.** Install Python 3.10 or later, [`cloudflared`](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/downloads/) and `openssl`. Mount the footage disk. The footage root should be a mount point, such as `/mnt/footage`: the store refuses uploads while it is not mounted. For a folder on the system disk, add `--system-disk` to the install command below. For thumbnails, install Pillow for that `python3`: the `python3-pil` package on Debian, Ubuntu and Raspberry Pi OS, or `brew install pillow` with Homebrew on macOS.
2. **Create a tunnel.** In the Cloudflare dashboard, open Zero Trust → Networks → Tunnels and create a Cloudflared tunnel. It needs no public hostname. Copy its token and its tunnel ID.
@@ -85,7 +85,7 @@ The store is a small HTTP server on the host with the big disk. It listens on `1
external/store/deploy.sh <store-host> <footage-root> # for example: me@store-host /mnt/footage
```
- On Linux, also run `loginctl enable-linger $USER` on the store host once, so the services run without a login.
+ On Linux, also run `loginctl enable-linger $USER` on the store host once, so the services run without a login. On macOS the agents run in your login session: install while you are logged in at the Mac, and turn on automatic login (which FileVault prevents) so they start again after a restart.
5. **Create the VPC service** that the Workers bind to as `STORE`. The ID it prints is `STORE_SERVICE_ID`.
@@ -162,7 +162,7 @@ curl -s -H "Authorization: Bearer <INGEST_TOKEN>" https://skyvision-ingest.<your
2. **Fill in the camera** in `.env`:
- `CAMERA_IP_ADDRESS`, `CAMERA_USERNAME` and `CAMERA_PASSWORD`;
- - `CAMERA_RTSP_PORT` and `CAMERA_RTSP_PATH` (see [Requirements](requirements.md) for paths and the H.265 requirement);
+ - `CAMERA_RTSP_PORT` and `CAMERA_RTSP_PATH` (see [Requirements](requirements.md#camera) for stream paths and codecs);
- `CAM_ID`.
3. **Install the recorder and uploader.** This copies the scripts, writes `~/.config/skyvision/relay.env` (mode 600), and starts both systemd user services.
diff --git a/docs/develop/architecture.md b/docs/develop/architecture.md
index 49eca5d..8f47631 100644
--- a/docs/develop/architecture.md
+++ b/docs/develop/architecture.md
@@ -119,4 +119,4 @@ The processor is an installed Python package with the same six layers. Its `rout
## The documentation site
-`package/docs/` is an [Astro](https://astro.build) site built to static files. It has no Worker code; `wrangler.jsonc` there serves `dist/` as static assets at `DOCS_HOST`. Pages are Markdown in `src/content/docs/`, and the sidebar order is in `src/nav.ts`. `npm run build` builds the pages and a [Pagefind](https://pagefind.app) search index, then fails if any internal link or `#anchor` does not resolve. The build also fails when a page is missing from the sidebar or a sidebar entry has no page. Writing rules are in [`package/docs/README.md`](https://github.com/Deibler/openskycam/blob/main/package/docs/README.md).
+`package/docs/` is an [Astro](https://astro.build) site built to static files. It has no Worker code; `wrangler.jsonc` there serves `dist/` as static assets at `DOCS_HOST`. The pages are the Markdown files in the repository's `docs/` folder (loaded by `src/content.config.ts`), and the sidebar order is in `src/nav.ts`. `npm run build` builds the pages and a [Pagefind](https://pagefind.app) search index, then fails if any internal link or `#anchor` does not resolve. The build also fails when a page is missing from the sidebar or a sidebar entry has no page. Writing rules are in [`package/docs/README.md`](https://github.com/Deibler/openskycam/blob/main/package/docs/README.md).
diff --git a/docs/develop/local.md b/docs/develop/local.md
index 038b858..cd58a00 100644
--- a/docs/develop/local.md
+++ b/docs/develop/local.md
@@ -9,11 +9,11 @@ Everything runs from a clone with Node.js 24 (`.node-version`), Python 3 and `sh
## Install
```sh
-npm install # Biome at the root, then `npm ci` in package/dashboard and package/worker
+npm install # Biome at the root, then `npm ci` in package/dashboard, package/worker and package/docs
cp .env.example .env
```
-Each Worker installs from its own lockfile. The documentation site is separate: `cd package/docs && npm ci`.
+Each Worker and the documentation site install from their own lockfiles.
## Run the website
@@ -72,16 +72,19 @@ This is what CI runs. It stops at the first failing part:
| `docs-check` | In `package/docs`: builds this site and its search index, and fails on a broken internal link, a missing `#anchor`, or a page that is not in the sidebar |
| `processor-check` | `python3 -m unittest discover -s tests` in `processor/` |
| `store-check` | The same in `external/store/` |
-| `lint` | `shellcheck -x` on the relay, store and processor shell scripts |
+| `setup-check` | `node --test setup/lib.check.mjs`, the guided setup's parsing and validation |
+| `lint` | `shellcheck -x` on the relay, store and processor shell scripts and `setup/install.sh` |
| (last step) | `python3 camera_mcp.py --check`, the camera MCP self-test |
The processor's image tests need `numpy` and OpenCV, and the store's thumbnail tests need Pillow. CI installs them like this:
```sh
-python3 -m pip install numpy==2.4.3 opencv-python-headless==4.13.0.92
sudo apt-get install shellcheck python3-pil
+python3 -m pip install --user --break-system-packages numpy==2.4.3 opencv-python-headless==4.13.0.92
```
+On your own machine, a virtual environment avoids `--break-system-packages`: `python3 -m venv .venv && .venv/bin/pip install numpy==2.4.3 opencv-python-headless==4.13.0.92 pillow`, then `PATH="$PWD/.venv/bin:$PATH" make check`.
+
The GitHub workflow ([`.github/workflows/check.yml`](https://github.com/Deibler/openskycam/blob/main/.github/workflows/check.yml)) runs on every pull request and on pushes to `main`, on Ubuntu, with a 15-minute limit. Workers Builds runs each Worker's `npm run check` again before deploying it.
To run one part while working:
diff --git a/docs/how-it-works.md b/docs/how-it-works.md
index 5a39bd9..7ee7e72 100644
--- a/docs/how-it-works.md
+++ b/docs/how-it-works.md
@@ -24,11 +24,11 @@ Files are written to `/dev/shm/skyvision/`, a RAM disk, so the relay's SD card s
| Failure | What happens |
|---|---|
-| Worker unreachable, or answers `401`, `403`, `404`, `408`, `429` or `5xx` | The file is kept and retried after 10 s. Finished files move from RAM to `~/skyvision/spool/pending/` on disk and upload oldest first when the Worker is back. |
+| Worker unreachable, or answers `401`, `403` (wrong token, or a camera missing from `CAMERAS`), `404`, `408`, `429` or `5xx` | The file is kept and retried after 10 s. Finished files move from RAM to `~/skyvision/spool/pending/` on disk and upload oldest first when the Worker is back. |
| More than 3 finished files in RAM | The extras move to `spool/pending/`, so a full RAM disk never stops the recorder. |
| `ffprobe` cannot read the file | It moves to `spool/quarantine/` and is never retried. |
| Any other answer, such as `409` or `422` | Same: quarantined for inspection. |
-| Relay disk below `MIN_FREE_GB` (default 20) | The oldest pending file is deleted, one at a time, until there is room. |
+| Relay disk below `MIN_FREE_GB` (default 20) | Quarantined files are deleted first, then the oldest pending file, one at a time, until there is room. |
### 3. The ingest Worker checks and names it
diff --git a/docs/models.md b/docs/models.md
index 1c1df7d..6894fad 100644
--- a/docs/models.md
+++ b/docs/models.md
@@ -3,7 +3,7 @@ title: Models
description: The model registry, how to publish a trained model, and how devices and Workshop runs use it.
---
-The model registry stores every model version you publish: its weights and a model card that describes classes, task, input and evaluations. Versions are immutable. Workshop runs pin the exact checkpoint they use. A new deployment starts with an empty registry: openskycam ships no weights, so publish your own.
+The model registry stores every model version you publish: its weights and a model card that describes classes, task, input and evaluations. Versions are immutable. Workshop runs pin the exact checkpoint they use. A new deployment starts with an empty registry: publish your own model, or the night detector in `models/night-detector` ([below](#publish-the-night-detector)).
Model files live in the private `skyvision-models` R2 bucket, separate from footage retention, so they are never pruned. D1 holds the model cards, the artifact index, device assignments and device status.
diff --git a/docs/processor.md b/docs/processor.md
index b5a85f4..faeefc3 100644
--- a/docs/processor.md
+++ b/docs/processor.md
@@ -61,10 +61,10 @@ GPUs run the model in half precision (`half=True`); the CPU runs full precision.
| Argument | Meaning |
|---|---|
-| 1. Python | Absolute path of the interpreter. A virtual environment's interpreter is used as given, not resolved. |
+| 1. Python | Absolute path of the interpreter. A virtual environment's interpreter is used as given, not resolved. It must import `av`, `cv2`, `torch` and `ultralytics`, or the installer stops. |
| 2. Key file | Must exist with mode 600, or the installer stops. |
| 3. Runtime ID | Must match `workshop-[a-f0-9]{32}`. |
-| 4. API origin | Must be HTTPS. |
+| 4. API origin | Must be HTTPS, with no path: `https://<api-host>`. |
| 5. Dependency directory | Optional. Added to `PYTHONPATH` after the source directory. |
The service runs `python -m skyvision_processor workshop-run --base-url <origin> --key-file <key> --runtime <id> --watch`.
@@ -74,7 +74,7 @@ The service runs `python -m skyvision_processor workshop-run --base-url <origin>
| macOS | `~/Library/LaunchAgents/org.openskycam.workshop.plist`: `KeepAlive`, `Nice` 10, low-priority I/O, standard process type | `launchctl print gui/$(id -u)/org.openskycam.workshop` | `launchctl bootout gui/$(id -u) ~/Library/LaunchAgents/org.openskycam.workshop.plist` |
| Linux | `~/.config/systemd/user/skyvision-workshop.service`: `Restart=always`, `RestartSec=30`, `Nice=10`, best-effort I/O priority 7 | `systemctl --user status skyvision-workshop` | `systemctl --user disable --now skyvision-workshop` |
-On Linux, run `loginctl enable-linger $USER` once so the service runs without a login session. On macOS the agent runs at standard priority on purpose. Background priority confines it to efficiency cores, where decoding measured 4.8 times slower on a Mac mini.
+On Linux, run `loginctl enable-linger $USER` once so the service runs without a login session. If systemd cannot load the unit the installer wrote, the installer stops with an error before enabling it; `systemctl --user status skyvision-workshop` shows why. On macOS the agent runs at standard priority on purpose. Background priority confines it to efficiency cores, where decoding measured 4.8 times slower on a Mac mini.
Both services discard standard output and error. To see what the runner does, stop the service and run one job in the foreground:
@@ -117,6 +117,8 @@ To upgrade:
2. Print its fingerprint and register a new runtime.
3. Rerun `install-workshop.sh` from the new directory with the new runtime ID. The installer replaces the service definition and restarts it.
+If `npm run setup` installed the processor, update your clone (`git pull`) and run `npm run setup -- processor` instead. It copies the new source to `~/openskycam-processor/<commit>/`, registers its runtime and reinstalls the service.
+
Runs already created stay bound to the runtime they were created with. To roll back, rerun `install-workshop.sh` from the previous directory with the previous runtime ID; runtime records are never deleted.
## How a job runs
@@ -127,7 +129,7 @@ A Workshop run is split into jobs: one per recording, or one per image for runs
2. **Lease.** A claim leases the job for 5 minutes and increments its generation. A background thread renews it every 45 seconds (`POST …/jobs/{id}/heartbeat`). A job whose lease runs out goes back to the queue and is claimed again with a new generation. After its third lease expires it fails with `attempts_exhausted`. A stale attempt cannot publish: every write checks the generation and token, and returns 409 "The lease is no longer valid. Discard this attempt." The processor stops the job as soon as a renewal fails.
3. **Input.** `GET …/jobs/{id}/input` streams the original recording from the footage store, or the image. The download goes to a temporary directory and must match the pinned SHA-256 and size.
4. **Models.** `GET …/jobs/{id}/models/{sha256}` downloads a model's `.pt` file. The server allows only models the leased run uses. Files are cached by checksum, in `~/Library/Caches/openskycam-workshop/` on macOS or `$XDG_CACHE_HOME/openskycam-workshop/` (default `~/.cache/…`) on Linux. Before use, the checkpoint's class map and task must match the registry. One model stays loaded between jobs.
-5. **Frames.** The processor samples frames (see [Workflows](workshop/workflows.md)), runs the workflow graph on each, and tracks boxes across the recording (see [Tracking](workshop/tracking.md)). `POST …/jobs/{id}/progress` reports the phase (`downloading`, `processing`, `finalizing`), and each finished frame for the website's live view.
+5. **Frames.** The processor samples frames (see [Workflows](workshop/workflows.md)), runs the workflow graph on each, and tracks boxes across the recording (see [Tracking](workshop/tracking.md)). `POST …/jobs/{id}/progress` reports the phase (`downloading`, `loading_model`, `processing`, `uploading`, `finalizing`), and each finished frame for the website's live view.
6. **Upload.** Each kept frame is uploaded as `PUT …/jobs/{id}/images/{sha256}` (a JPEG of at most 8 MiB; the server checks the JPEG markers and the checksum).
7. **Complete or fail.** `POST …/jobs/{id}/complete` sends the fingerprint, the sample count, slots the recorder skipped (`empty_slots`), and every frame's labels and tracks. Repeating the same result is accepted. On any error the processor sends `POST …/jobs/{id}/fail` with `processor_failed`; the other accepted codes are `download_failed`, `decode_failed` and `model_failed`.
diff --git a/docs/requirements.md b/docs/requirements.md
index 33402e4..5301193 100644
--- a/docs/requirements.md
+++ b/docs/requirements.md
@@ -15,11 +15,11 @@ openskycam needs a camera, three roles on machines you own (relay, footage store
## Camera
-Any IP camera works if it serves an **H.265 (HEVC)** RTSP stream.
+Any IP camera works if it serves an **H.264 or H.265** RTSP stream.
- **Codec.** H.264 or H.265. `record.sh` copies the stream without re-encoding. It asks `ffprobe` for the codec first and tags H.265 as `hvc1`, so that Safari and QuickTime play it; H.264 keeps its usual `avc1` tag.
- **Stream path.** The relay builds `rtsp://<user>:<password>@<address>:<port><path>`. The default path, `/cam/realmonitor?channel=1&subtype=0`, is the Dahua main stream. Set `CAMERA_RTSP_PATH` for other brands, for example `/Streaming/Channels/101` (Hikvision) or `/stream1` (TP-Link Tapo).
-- **Credentials.** The username and password go into that URL unescaped, so they must not contain `@`, `:` or `/`.
+- **Credentials.** Write the username and password in `.env` as they are. The relay percent-encodes them when it builds the URL ([Relay](capture/relay.md)).
- **Bitrate.** The ingest Worker accepts segments of at most 95 MiB, because Workers on the Free and Pro plans reject request bodies over 100 MB. A 60-second segment stays under that up to about 13 Mbps.
- **Camera settings.** `camera_mcp.py`, which reads and changes camera settings from Claude Code, speaks the Dahua HTTP API only. Recording does not need it.
@@ -46,7 +46,7 @@ Measured on the reference camera: switching from constant bitrate (4.16 Mbps) to
A Raspberry Pi 4 or 5 is plenty. On a Pi 5, recording a 4 Mbps stream used under 1 second of CPU per minute.
- **RAM.** Segments are written to `/dev/shm`. The relay keeps the segment being recorded plus at most 3 finished ones there, about 30 MB each at 4 Mbps.
-- **Disk.** It holds the backlog while the Worker is unreachable, in `~/skyvision/spool/pending/`. The relay never lets free space drop below `MIN_FREE_GB` (20 GB by default), and deletes the oldest backlog first to keep it.
+- **Disk.** It holds the backlog while the Worker is unreachable, in `~/skyvision/spool/pending/`. The relay never lets free space drop below `MIN_FREE_GB` (20 GB by default): it deletes quarantined files first, then the oldest backlog.
- **Network.** It needs a steady link to the camera and an upload rate above the camera's bitrate. The relay sends three uploads at once. Measured on the reference relay, one upload through the ingest Worker often ran at about 4 Mbit/s, and three at once reached 24 to 28 Mbit/s together.
- **Lingering.** Both relay services are systemd user services. Run `loginctl enable-linger $USER` once so they run without a login session. No root access is needed.
diff --git a/docs/workshop/runs.md b/docs/workshop/runs.md
index 1ad27d8..9f6bacf 100644
--- a/docs/workshop/runs.md
+++ b/docs/workshop/runs.md
@@ -71,7 +71,7 @@ A job moves through `queued` → `leased` → `completed`, or ends `failed`, `ca
- **Claim.** The processor calls `POST /v1/workshop/claim` with its runtime ID and code SHA-256, using a `workshop:run` key bound to the runtime's device. The claim records that the runtime was seen, and leases the next job for a camera the key covers: photos first, then the oldest run, then job order. The answer includes the job's input and the run's snapshot.
- **Lease.** A lease lasts 5 minutes, never past the job's expiry. The processor renews it every 45 s (`/heartbeat`). Every upload, progress event and the final result carry the lease's `generation` and token; anything sent under an old lease is refused.
- **Retries.** If a processor dies or loses its network, its lease lapses and the job can be claimed again with a new generation. After a third lapsed lease, the job fails with `attempts_exhausted`.
-- **Failure.** When the processor itself hits an error (download, decode, model, upload), it reports `processor_failed` and the job fails without a retry. A recording that ends before its last scheduled slot fails this way.
+- **Failure.** When the processor itself hits an error (download, decode, model, upload), it reports `processor_failed` and the job fails without a retry. A dropped connection or a `5xx` answer to a heartbeat, download or upload is tried three times before it counts as an error. A recording that ends before its last scheduled slot fails this way.
- **Expiry.** A job expires with its source: the recording's 40 days, or the test's hour. Expired jobs are marked `expired` and their input is cleared.
- **Completion.** The result must account for every slot: frames sent plus `empty_slots` (slots the recorder skipped) must equal the expected count, and each frame must sit in its own slot and inside the window. A workflow with a Do nothing step may send fewer frames than it sampled. Results are committed only after the whole job succeeds, so a full run's bucket never holds part of a job. Committing the same result twice is accepted; a different result for a completed job is `409`.
diff --git a/docs/workshop/workflows.md b/docs/workshop/workflows.md
index 051a44d..5be903c 100644
--- a/docs/workshop/workflows.md
+++ b/docs/workshop/workflows.md
@@ -139,7 +139,7 @@ How each trigger behaves:
- A labeled-bucket input always takes only images added since the last scheduled run, whatever the trigger.
- **`onlyIf`** skips a set-time run when its window has no footage, and a follow run when the upstream run saved no images.
-A scheduled workflow's footage `day` must be `run-day` or `previous-day`; a fixed date would process the same footage every time and is refused. With `run-day`, `at` must be at least 10 minutes after the window's `until` so its last clips have uploaded. A window that would need a time after midnight must use `previous-day`.
+A scheduled workflow's footage `day` must be `run-day` or `previous-day`; a fixed date would process the same footage every time and is refused. With `run-day`, `at` must be at least 10 minutes after the window's `until` so its last clips have uploaded. A window that would need a time after midnight must use `previous-day`. A slot that starts late, after midnight, still takes the day of its slot. When new footage arrives needs `run-day` and a window that ends by 23:35 UTC, so its last clips upload before the day's final check at 23:45.
The dashboard Worker's cron runs every minute. A scheduled run's ID is `sched_` plus a hash of the workflow and trigger, so a repeated or retried minute never starts a second run. Scheduled runs go to the newest [processor runtime](runs.md) that has checked in; with none, the outcome is `failed` and the next minute tries again. `GET /api/workshop/schedules` lists each schedule's last outcome: `started` (with `run_id`), `skipped`, or `failed` (with `message`). The editor shows the same.
@@ -204,6 +204,6 @@ Why it is built this way:
- **The trigger is When new footage arrives.** A full run takes at most 100 recordings and 2,000 sampled frames. The whole 7.5-hour window has 450 one-minute recordings and 27,000 one-second slots, so a set-time run of it would be refused. Every 15 minutes the arrival trigger takes only the recordings uploaded since the last run, about 15 recordings and 900 frames, and outside 02:00–09:30 it finds nothing and records `skipped`.
- **The first arrival run has no earlier run to count from**, so it takes everything already uploaded in the window. Add the schedule outside 02:00–09:30. Added mid-window, the first run can exceed 100 recordings and fail each time until the window starts empty the next night.
-- **`run-day`** is required on any schedule, and here it means "tonight": the window is the UTC date each run is created on.
+- **`run-day`** is required for When new footage arrives, and here it means "tonight": the window is the UTC date each run is created on.
- **Frames with no Plane at all** take the `no` branch too, since neither rule matches. Put a condition with a `class` rule (`Plane`, `present`) in front of this one if those frames should go elsewhere, or to Do nothing.
- Run a [test](runs.md) of 20–100 frames first; it uses the same pinned models and sampling and saves nothing.
diff --git a/external/relay/record.sh b/external/relay/record.sh
index 0d5fab5..08df939 100755
--- a/external/relay/record.sh
+++ b/external/relay/record.sh
@@ -11,14 +11,19 @@ mkdir -p "$LIVE_DIR"
cd "$LIVE_DIR"
export TZ=UTC # filenames are UTC: no DST duplicates or gaps
-codec=$(ffprobe -v error -rtsp_transport tcp -timeout 15000000 -select_streams v:0 \
+# The RTSP socket timeout is -timeout from ffmpeg 5 on. ffmpeg 4 (Ubuntu 22.04, Debian 11) calls it -stimeout,
+# and reads -timeout as "wait for the camera to connect to us", which makes it listen instead of connect.
+timeout=-timeout
+[[ $(ffmpeg -hide_banner -h demuxer=rtsp 2>/dev/null) == *-stimeout* ]] && timeout=-stimeout
+
+codec=$(ffprobe -v error -rtsp_transport tcp "$timeout" 15000000 -select_streams v:0 \
-show_entries stream=codec_name -of csv=p=0 "$CAMERA_RTSP_URL")
tag=()
[ "$codec" = hevc ] && tag=(-tag:v hvc1)
# ponytail: the RTSP URL (with camera password) is visible in `ps` to other users on this Pi; ffmpeg can't read it from a file.
exec ffmpeg -nostdin -hide_banner -loglevel warning \
- -rtsp_transport tcp -timeout 15000000 -fflags +genpts \
+ -rtsp_transport tcp "$timeout" 15000000 -fflags +genpts \
-i "$CAMERA_RTSP_URL" \
-map 0:v:0 -c copy "${tag[@]}" \
-f segment -segment_time 60 -segment_atclocktime 1 -reset_timestamps 1 -strftime 1 \
diff --git a/external/relay/test_upload.sh b/external/relay/test_upload.sh
index 2c55b1a..043433a 100755
--- a/external/relay/test_upload.sh
+++ b/external/relay/test_upload.sh
@@ -23,11 +23,12 @@ has "$SPOOL_DIR/pending" 20260101T000200Z || fail "finished segment not parke
has "$LIVE_DIR" 20260101T000300Z || fail "newest segment (still being written) was touched"
[[ $(find "$LIVE_DIR" -name '*.mp4' | wc -l) == 1 ]] || fail "unexpected files left in RAM"
-# 2) Disk guard: with the floor above free space, the backlog is dropped oldest first.
+# 2) Disk guard: with the floor above free space, set-aside files go first, then the backlog oldest first.
MIN_FREE_GB=1000000 timeout 6 bash "$here/upload.sh" > "$T/log" 2>&1 || true
-[[ -z $(find "$SPOOL_DIR/pending" -name '*.mp4') ]] || fail "disk guard left backlog"
-grep -o 'dropped oldest backlog: [^ ]*' "$T/log" | head -2 | tr '\n' ' ' \
- | grep -q '000100Z.mp4 .*000200Z.mp4' || fail "backlog not dropped oldest first"
+[[ -z $(find "$SPOOL_DIR/pending" "$SPOOL_DIR/quarantine" -name '*.mp4') ]] || fail "disk guard left backlog"
+grep -o 'dropped oldest [^ ]*' "$T/log" | head -3 | tr '\n' ' ' \
+ | grep -q 'quarantine/sky01_20260101T000000Z.mp4 .*pending/sky01_20260101T000100Z.mp4 .*000200Z.mp4' \
+ || fail "quarantine and backlog not dropped in order"
has "$LIVE_DIR" 20260101T000300Z || fail "disk guard touched the segment being written"
# 3) A Worker that takes uploads but slower than the camera records: RAM keeps only the next few
diff --git a/external/relay/upload.sh b/external/relay/upload.sh
index 8d7b31b..76c31c7 100755
--- a/external/relay/upload.sh
+++ b/external/relay/upload.sh
@@ -59,11 +59,13 @@ while true; do
for g in "${live[@]}"; do [[ -e $g ]] && mv "$g" "$PENDING/"; done
fi
fi
- # Never fill the disk; other services on this Pi need it. Oldest backlog goes first.
+ # Never fill the disk; other services on this Pi need it. Files set aside go first, then the oldest backlog:
+ # a rejection that repeats every minute would otherwise fill the disk with quarantine and stop the recorder.
while (( $(df --output=avail -k "$PENDING" | tail -1) < MIN_FREE_GB * 1048576 )); do
- backlog=("$PENDING"/*.mp4)
+ backlog=("$BAD"/*.mp4)
+ (( ${#backlog[@]} )) || backlog=("$PENDING"/*.mp4)
(( ${#backlog[@]} )) || break
- echo "disk low, dropped oldest backlog: ${backlog[0]##*/}"; rm -f "${backlog[0]}"
+ echo "disk low, dropped oldest ${backlog[0]#"$SPOOL_DIR"/}"; rm -f "${backlog[0]}"
done
sleep "$pause"
done
diff --git a/external/store/README.md b/external/store/README.md
index 9c40797..fa91fc5 100644
--- a/external/store/README.md
+++ b/external/store/README.md
@@ -5,7 +5,7 @@ machine with Python 3, `cloudflared` and a large drive. Cloudflare Workers reach
[Workers VPC service](https://developers.cloudflare.com/workers-vpc/): the `STORE` binding in both
Workers. It has no public hostname and listens only on `127.0.0.1:8096`.
-Files keep the keys the Workers used in R2, under the footage root you choose:
+Files are stored under the object keys the Workers use, below the footage root you choose:
```
<footage-root>/raw/cam=sky01/date=2026-09-22/hour=14/sky01_20260922T140500Z.mp4
@@ -23,8 +23,8 @@ Every request needs `Authorization: Bearer <store token>`.
| `POST /thumbnail?size=<px>[&left=&top=&width=&height=]` with a sized JPEG body | `200` a JPEG: the crop (pixels) fit to `size`×`size`, or the whole image only shrunk · `400` bad size or crop · `413` over 32 MB · `422` not an image · `503` Pillow missing |
Thumbnails are for Workshop images and agent previews, so the website needs no paid image service.
-They use Pillow in the host's `python3` (`python3 -m pip install --user pillow`, or your distribution's
-`python3-pil` package), imported on first use, so footage never depends on it. On macOS the deploy prefers
+They use Pillow in the host's `python3` (your distribution's `python3-pil` package, or `brew install pillow`
+on macOS), imported on first use, so footage never depends on it. On macOS the deploy prefers
Homebrew's `python3`.
A file appears at its key only once it is complete, synced to disk and matches its SHA-256, and an
diff --git a/external/store/deploy.sh b/external/store/deploy.sh
index 6619552..70fd68b 100755
--- a/external/store/deploy.sh
+++ b/external/store/deploy.sh
@@ -24,11 +24,16 @@ set -euo pipefail
# Non-interactive ssh on macOS usually lacks Homebrew's PATH; prefer its python3, which has Pillow.
PATH=/opt/homebrew/bin:/usr/local/bin:$PATH
python=$(command -v python3) || { echo "python3 not found on the store host" >&2; exit 1; }
+"$python" -c 'import sys; sys.exit(sys.version_info < (3, 10))' \
+ || { echo "$python is older than Python 3.10; install a newer python3 (on macOS: brew install python)" >&2; exit 1; }
cloudflared=$(command -v cloudflared) || { echo "cloudflared not found on the store host; install it first" >&2; exit 1; }
config=$HOME/.config/skyvision-store
[ -s "$config/tunnel.token" ] || { echo "missing $config/tunnel.token" >&2; exit 1; }
[ -s "$config/store.token" ] || (umask 077 && openssl rand -hex 32 > "$config/store.token")
"$python" -c 'import PIL' 2>/dev/null || echo "warning: no Pillow for $python; thumbnails will answer 503" >&2
+# A folder on the system disk has no mount to wait for, and the store never creates its root.
+[ -z "$DISK" ] || mkdir -p "$ROOT"
+[ -w "$ROOT" ] || echo "warning: $ROOT is missing or not writable by $(id -un); uploads will fail until it is" >&2
case $(uname) in
Darwin)
@@ -65,6 +70,8 @@ PY
tail -n 3 "$HOME/Library/Logs/org.openskycam.store.log"
;;
Linux)
+ # ExecStart= expands % specifiers and $ variables, and reads \ and " as escapes inside quotes.
+ unit_quote() { local v=${1//\\/\\\\}; v=${v//%/%%}; v=${v//\$/\$\$}; printf '"%s"' "${v//\"/\\\"}"; }
units=$HOME/.config/systemd/user
mkdir -p "$units"
cat > "$units/skyvision-store.service" <<UNIT
@@ -75,7 +82,7 @@ StartLimitIntervalSec=0
[Service]
WorkingDirectory=$HOME/skyvision-store
Environment=PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1
-ExecStart="$python" -m skyvision_store --root "$ROOT" --token-file "$config/store.token" $DISK
+ExecStart=$(unit_quote "$python") -m skyvision_store --root $(unit_quote "$ROOT") --token-file $(unit_quote "$config/store.token") $DISK
Restart=always
RestartSec=10
@@ -88,7 +95,7 @@ Description=SkyVision footage store tunnel (cloudflared)
StartLimitIntervalSec=0
[Service]
-ExecStart="$cloudflared" tunnel --no-autoupdate run --token-file "$config/tunnel.token"
+ExecStart=$(unit_quote "$cloudflared") tunnel --no-autoupdate run --token-file $(unit_quote "$config/tunnel.token")
Restart=always
RestartSec=10
diff --git a/external/store/skyvision_store/images.py b/external/store/skyvision_store/images.py
index 1a3c345..b341345 100644
--- a/external/store/skyvision_store/images.py
+++ b/external/store/skyvision_store/images.py
@@ -15,7 +15,7 @@ def thumbnail(data: bytes, size: int, crop: tuple[int, int, int, int] | None) ->
image.draft('RGB', (round(width * scale), round(height * scale)))
k = image.size[0] / width
region = image.crop((round(left * k), round(top * k), round((left + w) * k), round((top + h) * k)))
- region = region.convert('RGB').resize((max(1, round(w * scale)), max(1, round(h * scale))), Image.Resampling.LANCZOS)
+ region = region.convert('RGB').resize((max(1, round(w * scale)), max(1, round(h * scale))), getattr(Image, 'Resampling', Image).LANCZOS) # Resampling is Pillow 9.1+; Ubuntu 22.04 has 9.0
out = io.BytesIO()
region.save(out, 'JPEG', quality=75)
return out.getvalue()
diff --git a/external/store/skyvision_store/server.py b/external/store/skyvision_store/server.py
index ef7cc54..71fb5f8 100644
--- a/external/store/skyvision_store/server.py
+++ b/external/store/skyvision_store/server.py
@@ -66,9 +66,9 @@ class Handler(BaseHTTPRequestHandler):
return self._reply(411, error='Content-Length is required', close=True)
if not self.server.mounted():
return self._reply(503, error='footage disk is not mounted', close=True)
- if shutil.disk_usage(self.server.root).free < self.server.min_free:
- return self._reply(507, error='footage disk is nearly full', close=True)
try:
+ if shutil.disk_usage(self.server.root).free < self.server.min_free:
+ return self._reply(507, error='footage disk is nearly full', close=True)
body = self._chunked_body() if chunked else self._sized_body(int(length))
created, size = objects.put(self.server.root, target, body, sha256)
except objects.Conflict as error:
diff --git a/external/store/tests/test_store.py b/external/store/tests/test_store.py
index d402819..d7788d9 100644
--- a/external/store/tests/test_store.py
+++ b/external/store/tests/test_store.py
@@ -5,6 +5,8 @@ import json
import os
import re
import shutil
+import subprocess
+import sys
import tempfile
import threading
import time
@@ -106,6 +108,11 @@ class StoreTest(unittest.TestCase):
self.assertEqual(response.status, 507)
self.assertIsNone(self.stored())
+ def test_answers_503_when_the_footage_folder_is_missing(self):
+ shutil.rmtree(self.root)
+ response, _ = self.put(os.urandom(1_000))
+ self.assertEqual(response.status, 503)
+
def test_accepts_writes_while_free_space_is_above_the_floor(self):
# The janitor stops pruning once free space reaches the floor, so a write must not demand more.
free = shutil.disk_usage(self.root).free
@@ -258,6 +265,19 @@ class DeployTest(unittest.TestCase):
for arguments in bootouts:
self.assertRegex(arguments, r'^"gui/\$\(id -u\)" "[^"]+\.plist"')
+ @unittest.skipUnless(shutil.which('systemd-analyze'), 'needs systemd-analyze')
+ def test_linux_unit_takes_any_footage_folder(self):
+ # ExecStart= expands % and $ and unescapes \ and ", so the folder is escaped for systemd.
+ script = (Path(__file__).parents[1] / 'deploy.sh').read_text()
+ quote = re.search(r'^\s*(unit_quote\(\).*)$', script, re.MULTILINE).group(1)
+ unit = Path(tempfile.mkdtemp()) / 'store.service'
+ root = '/mnt/foot%20age $x "q" back\\slash'
+ subprocess.run(['bash', '-c', quote + '; printf "[Service]\\nExecStart=%s -m skyvision_store --root %s\\n" '
+ '"$(unit_quote "$1")" "$(unit_quote "$2")" > "$3"', '-', sys.executable, root, str(unit)], check=True)
+ self.assertIn('--root "/mnt/foot%%20age $$x \\"q\\" back\\\\slash"', unit.read_text())
+ result = subprocess.run(['systemd-analyze', 'verify', '--man=no', str(unit)], capture_output=True, text=True)
+ self.assertEqual(0, result.returncode, result.stderr)
+
if __name__ == '__main__':
unittest.main()
diff --git a/package/dashboard/openapi.json b/package/dashboard/openapi.json
index ce40c8e..95b1891 100644
--- a/package/dashboard/openapi.json
+++ b/package/dashboard/openapi.json
@@ -751,7 +751,7 @@
},
"scope": {
"type": "string",
- "enum": ["footage:read", "processor:run", "models:write", "models:sync"],
+ "enum": ["footage:read", "processor:run", "models:write", "models:sync", "workshop:run", "agent"],
"default": "footage:read"
},
"pipeline_id": {
@@ -5663,7 +5663,7 @@
},
"scope": {
"type": "string",
- "enum": ["footage:read", "processor:run", "models:write", "models:sync", "workshop:run"]
+ "enum": ["footage:read", "processor:run", "models:write", "models:sync", "workshop:run", "agent"]
},
"cameras": {
"type": "array",
@@ -5783,7 +5783,7 @@
},
"scope": {
"type": "string",
- "enum": ["footage:read", "processor:run", "models:write", "models:sync", "workshop:run"]
+ "enum": ["footage:read", "processor:run", "models:write", "models:sync", "workshop:run", "agent"]
},
"cameras": {
"type": "array",
diff --git a/package/dashboard/src/db/FootageStorageOps.ts b/package/dashboard/src/db/FootageStorageOps.ts
index 10d61d5..e038127 100644
--- a/package/dashboard/src/db/FootageStorageOps.ts
+++ b/package/dashboard/src/db/FootageStorageOps.ts
@@ -4,6 +4,7 @@ export async function queryRecordings(
camera: string,
from: string,
to: string,
+ earliest: string,
asOf: string,
cursor: Cursor | null,
limit: number,
@@ -11,10 +12,10 @@ export async function queryRecordings(
const { results } = await db
.prepare(
`SELECT object_key, cam, start, size, sha256, duration_s, frames, width, height, bitrate, codec FROM segments
- WHERE cam = ? AND start < ? AND strftime('%Y-%m-%dT%H:%M:%fZ', start, '+' || COALESCE(duration_s, 60) || ' seconds') > ? AND uploaded_at <= ?
+ WHERE cam = ? AND start < ? AND start > ? AND strftime('%Y-%m-%dT%H:%M:%fZ', start, '+' || COALESCE(duration_s, 60) || ' seconds') > ? AND uploaded_at <= ?
AND (start > ? OR (start = ? AND object_key > ?)) ORDER BY start, object_key LIMIT ?`,
)
- .bind(camera, to, from, asOf, cursor?.start ?? '', cursor?.start ?? '', cursor?.key ?? '', limit + 1)
+ .bind(camera, to, earliest, from, asOf, cursor?.start ?? '', cursor?.start ?? '', cursor?.key ?? '', limit + 1)
.all<RecordingRow>();
return results;
}
diff --git a/package/dashboard/src/db/WorkshopInferenceOps.ts b/package/dashboard/src/db/WorkshopInferenceOps.ts
index 9483f81..bc9f2c3 100644
--- a/package/dashboard/src/db/WorkshopInferenceOps.ts
+++ b/package/dashboard/src/db/WorkshopInferenceOps.ts
@@ -190,6 +190,22 @@ export async function upload(env: Env, u: WorkshopUpload) {
.bind(u.object_key, u.job_id, u.generation, u.sha256, u.bytes, u.expires_at)
.run();
}
+// D1 caps a bound string at 2 MB, and one frame's tracks may take 512 KB, so finished frames go in chunks.
+const ITEM_CHUNK_BYTES = 1_000_000;
+function itemChunks(items: WorkshopItem[]) {
+ const chunks: WorkshopItem[][] = [];
+ let size = Infinity;
+ for (const item of items) {
+ const bytes = JSON.stringify(item).length;
+ if (size + bytes > ITEM_CHUNK_BYTES) {
+ chunks.push([]);
+ size = 0;
+ }
+ chunks.at(-1)!.push(item);
+ size += bytes;
+ }
+ return chunks;
+}
export async function complete(
env: Env,
j: WorkshopJob,
@@ -199,22 +215,28 @@ export async function complete(
test = false,
) {
const db = env.CATALOG.withSession('first-primary');
+ const previews = itemChunks(test ? items : []),
+ firstPosition = previews.map((_, i) => previews.slice(0, i).reduce((n, c) => n + c.length, 0));
const rs = await db.batch([
db
.prepare(
`UPDATE workshop_jobs SET status='completed',result_sha256=?,completed_at=? WHERE id=? AND status='leased' AND generation=? AND lease_token=? AND lease_until>? AND expires_at>?`,
)
.bind(digest, now, j.id, j.generation, j.lease_token, now, now),
- db
- .prepare(
- `INSERT OR IGNORE INTO workshop_items SELECT json_extract(v.value,'$.id'),json_extract(v.value,'$.bucket_id'),json_extract(v.value,'$.asset_id'),json_extract(v.value,'$.run_id'),json_extract(v.value,'$.job_id'),json_extract(v.value,'$.source_sha256'),json_extract(v.value,'$.cam'),json_extract(v.value,'$.captured_at'),json_extract(v.value,'$.object_key'),json_extract(v.value,'$.sha256'),json_extract(v.value,'$.expires_at'),json_extract(v.value,'$.annotation_json'),json_extract(v.value,'$.provenance_json'),'pending',? FROM json_each(?) v WHERE EXISTS(SELECT 1 FROM workshop_jobs WHERE id=? AND status='completed' AND result_sha256=? AND generation=? AND lease_token=?) AND EXISTS(SELECT 1 FROM workshop_state s,json_each(s.document_json,'$.buckets') b WHERE json_extract(b.value,'$.id')=json_extract(v.value,'$.bucket_id'))`,
- )
- .bind(now, JSON.stringify(test ? [] : items), j.id, digest, j.generation, j.lease_token),
- db
- .prepare(
- `INSERT OR REPLACE INTO workshop_previews SELECT ?,?,CAST(v.key AS INTEGER),json_extract(v.value,'$.id'),v.value,json_extract(v.value,'$.expires_at') FROM json_each(?) v WHERE EXISTS(SELECT 1 FROM workshop_jobs WHERE id=? AND status='completed' AND result_sha256=? AND generation=? AND lease_token=?) AND EXISTS(SELECT 1 FROM workshop_state s,json_each(s.document_json,'$.buckets') b WHERE json_extract(b.value,'$.id')=json_extract(v.value,'$.bucket_id'))`,
- )
- .bind(j.id, j.generation, JSON.stringify(test ? items : []), j.id, digest, j.generation, j.lease_token),
+ ...itemChunks(test ? [] : items).map(chunk =>
+ db
+ .prepare(
+ `INSERT OR IGNORE INTO workshop_items SELECT json_extract(v.value,'$.id'),json_extract(v.value,'$.bucket_id'),json_extract(v.value,'$.asset_id'),json_extract(v.value,'$.run_id'),json_extract(v.value,'$.job_id'),json_extract(v.value,'$.source_sha256'),json_extract(v.value,'$.cam'),json_extract(v.value,'$.captured_at'),json_extract(v.value,'$.object_key'),json_extract(v.value,'$.sha256'),json_extract(v.value,'$.expires_at'),json_extract(v.value,'$.annotation_json'),json_extract(v.value,'$.provenance_json'),'pending',? FROM json_each(?) v WHERE EXISTS(SELECT 1 FROM workshop_jobs WHERE id=? AND status='completed' AND result_sha256=? AND generation=? AND lease_token=?) AND EXISTS(SELECT 1 FROM workshop_state s,json_each(s.document_json,'$.buckets') b WHERE json_extract(b.value,'$.id')=json_extract(v.value,'$.bucket_id'))`,
+ )
+ .bind(now, JSON.stringify(chunk), j.id, digest, j.generation, j.lease_token),
+ ),
+ ...previews.map((chunk, i) =>
+ db
+ .prepare(
+ `INSERT OR REPLACE INTO workshop_previews SELECT ?,?,CAST(v.key AS INTEGER)+?,json_extract(v.value,'$.id'),v.value,json_extract(v.value,'$.expires_at') FROM json_each(?) v WHERE EXISTS(SELECT 1 FROM workshop_jobs WHERE id=? AND status='completed' AND result_sha256=? AND generation=? AND lease_token=?) AND EXISTS(SELECT 1 FROM workshop_state s,json_each(s.document_json,'$.buckets') b WHERE json_extract(b.value,'$.id')=json_extract(v.value,'$.bucket_id'))`,
+ )
+ .bind(j.id, j.generation, firstPosition[i], JSON.stringify(chunk), j.id, digest, j.generation, j.lease_token),
+ ),
db
.prepare(
`DELETE FROM workshop_previews WHERE job_id=? AND ?=0 AND EXISTS(SELECT 1 FROM workshop_jobs WHERE id=? AND status='completed' AND result_sha256=? AND generation=? AND lease_token=?)`,
diff --git a/package/dashboard/src/handler/workshopInferenceHandler.ts b/package/dashboard/src/handler/workshopInferenceHandler.ts
index 519cb93..4304b5e 100644
--- a/package/dashboard/src/handler/workshopInferenceHandler.ts
+++ b/package/dashboard/src/handler/workshopInferenceHandler.ts
@@ -36,7 +36,7 @@ export const deleteWorkflowHandler: Handler<App> = async (r, a, { id }) =>
export const deleteBucketHandler: Handler<App> = async (r, a, { id }) =>
privateJson(await deletion.deleteWorkshopEntry(a.env, 'bucket', id, await readBody(r)));
export const reviewHandler: Handler<App> = async (r, a, { id }) =>
- privateJson(await review.reviewItem(a.env, id, await readBody(r)));
+ privateJson(await review.reviewItem(a.env, id, await readBody(r, 65536)));
export const imageHandler: Handler<App> = async (_r, a, { id }) => {
const o = await workshop.websiteImage(a.env, id);
return new Response(o.body, {
@@ -57,7 +57,7 @@ export const heartbeatHandler: Handler<MachineApp> = async (r, a, { id }) =>
export const failHandler: Handler<MachineApp> = async (r, a, { id }) =>
privateJson(await processor.fail(a, id, await readBody(r)));
export const completeHandler: Handler<MachineApp> = async (r, a, { id }) =>
- privateJson(await processor.complete(a, id, await readBody(r, 1500000)));
+ privateJson(await processor.complete(a, id, await readBody(r, 8 * 1024 * 1024)));
export const inputHandler: Handler<MachineApp> = async (r, a, { id }) => {
const o = await processor.input(a, id, lease(r));
return new Response(o.body, {
@@ -123,7 +123,7 @@ export const previewHandler: Handler<App> = async (_r, a, { id, item }) => {
});
};
export const progressHandler: Handler<MachineApp> = async (r, a, { id }) =>
- privateJson(await processor.progress(a, id, await readBody(r, 100000)));
+ privateJson(await processor.progress(a, id, await readBody(r, 1024 * 1024)));
export const machinePreviewHandler: Handler<MachineApp> = async (_r, a, { id, item }) => {
const image = await run.machinePreview(a.env, a.principal, id, item);
return new Response(image.body, {
diff --git a/package/dashboard/src/models/workshopInferencemodel.ts b/package/dashboard/src/models/workshopInferencemodel.ts
index df7141d..8bccf1b 100644
--- a/package/dashboard/src/models/workshopInferencemodel.ts
+++ b/package/dashboard/src/models/workshopInferencemodel.ts
@@ -307,6 +307,7 @@ export interface WorkshopScheduleDue {
key: string;
label: string;
found?: boolean;
+ slot?: Date;
}
export interface WorkshopFinishedRun {
id: string;
diff --git a/package/dashboard/src/service/agentFootageStorageService.ts b/package/dashboard/src/service/agentFootageStorageService.ts
index b2da4c3..0696937 100644
--- a/package/dashboard/src/service/agentFootageStorageService.ts
+++ b/package/dashboard/src/service/agentFootageStorageService.ts
@@ -8,6 +8,7 @@ import { action, number, reply, text } from '../utils/agentSharedUtil';
import { getOverview } from './overviewStorageService';
import { getAvailability } from './calendarStorageService';
import { listRecordings } from './footageStorageService';
+import { timestamp } from './cursorStorageService';
import { requireApiCamera, signLinks } from './credentialSharedService';
import { photoDetail, requestPhoto } from './workshopPhotoInferenceService';
import { nextRunUpdate, runFinished } from './workshopLiveInferenceService';
@@ -85,7 +86,8 @@ async function list(app: MachineApp, args: JsonObject) {
async function download(app: MachineApp, args: JsonObject) {
const cam = camera(app, args),
- at = text(args, 'at');
+ given = text(args, 'at'),
+ at = given ? timestamp(given, 'at') : undefined;
const from = at ?? text(args, 'from', true)!,
to = at ? new Date(Date.parse(at) + 1).toISOString() : text(args, 'to', true)!;
// A recording that started before `at` still covers it, so look back one clip length.
diff --git a/package/dashboard/src/service/agentWorkflowInferenceService.ts b/package/dashboard/src/service/agentWorkflowInferenceService.ts
index 988ac6e..9290d87 100644
--- a/package/dashboard/src/service/agentWorkflowInferenceService.ts
+++ b/package/dashboard/src/service/agentWorkflowInferenceService.ts
@@ -180,6 +180,10 @@ async function remove(app: MachineApp, args: JsonObject, kind: 'workflow' | 'buc
}
async function saveWorkflow(app: MachineApp, s: WorkshopState, workflow: Workflow) {
+ // Refuse before saving: a scheduled workflow runs from the cron, which never sees this key.
+ const input = workflow.nodes.find(n => n.data.kind === 'input');
+ if (input?.data.bucketId === 'camera-archive' && input.data.camera)
+ requireApiCamera(app.principal, String(input.data.camera));
place(workflow.nodes, workflow.edges);
const others = s.document.workflows.filter(w => w.id !== workflow.id);
const saved = await saveWorkshop(app.env, {
diff --git a/package/dashboard/src/service/bucketBrowseInferenceService.ts b/package/dashboard/src/service/bucketBrowseInferenceService.ts
index a81cc4b..c3a9bbc 100644
--- a/package/dashboard/src/service/bucketBrowseInferenceService.ts
+++ b/package/dashboard/src/service/bucketBrowseInferenceService.ts
@@ -71,6 +71,8 @@ function filter(bucket: string, query: Record<string, string>, now: string): Buc
(f.from && f.to && f.from >= f.to)
)
invalid('Check the filter range.');
+ // 100 tops the filter's range and means 100 or more: a frame keeps up to 1,000 labels.
+ if (f.maxObjects === 100) f.maxObjects = 1000;
return f;
}
diff --git a/package/dashboard/src/service/footageStorageService.ts b/package/dashboard/src/service/footageStorageService.ts
index a000fb2..bef1a6a 100644
--- a/package/dashboard/src/service/footageStorageService.ts
+++ b/package/dashboard/src/service/footageStorageService.ts
@@ -2,6 +2,7 @@ import { ApplicationError } from '../../../shared/src/utils/errorSharedUtil';
import { requireApiCamera } from './credentialSharedService';
import { queryRecordings } from '../db/FootageStorageOps';
import { timestamp, readCursor, encodeCursor } from './cursorStorageService';
+import { LONGEST_RECORDING_MS } from '../utils/configStorageUtil';
import type { Env } from '../models/applicationSharedmodel';
import type { ApiPrincipal } from '../models/credentialSharedmodel';
import type { FootageQuery, FootagePage } from '../models/footageStoragemodel';
@@ -18,7 +19,9 @@ export async function listRecordings(env: Env, principal: ApiPrincipal, query: F
const rawCursor = query.cursor;
const cursor = rawCursor ? readCursor(rawCursor, camera, from, to) : null;
const asOf = cursor?.as_of ?? new Date().toISOString();
- const results = await queryRecordings(env.CATALOG, camera, from, to, asOf, cursor, limit);
+ // The lower bound on start lets the index skip the camera's older history instead of reading all of it.
+ const earliest = new Date(Date.parse(from) - LONGEST_RECORDING_MS).toISOString();
+ const results = await queryRecordings(env.CATALOG, camera, from, to, earliest, asOf, cursor, limit);
const more = results.length > limit;
const rows = results.slice(0, limit);
const last = rows.at(-1);
diff --git a/package/dashboard/src/service/workshopInferenceService.ts b/package/dashboard/src/service/workshopInferenceService.ts
index 7394772..cd7622d 100644
--- a/package/dashboard/src/service/workshopInferenceService.ts
+++ b/package/dashboard/src/service/workshopInferenceService.ts
@@ -89,7 +89,8 @@ export async function runtimeStatus(env: Env) {
};
}
export async function registerRuntime(env: Env, body: JsonObject) {
- if (!hash(body.code_sha256) || !identifier(body.device_id) || !(await models.findDevice(env, body.device_id)))
+ // Any registered device ID will do; device IDs may contain dots, which workflow identifiers may not.
+ if (!hash(body.code_sha256) || typeof body.device_id !== 'string' || !(await models.findDevice(env, body.device_id)))
invalid('Register a device and provide the processor code SHA-256.');
const manifest = {
schema_version: 1,
diff --git a/package/dashboard/src/service/workshopPhotoInferenceService.ts b/package/dashboard/src/service/workshopPhotoInferenceService.ts
index dc6658c..633e9c5 100644
--- a/package/dashboard/src/service/workshopPhotoInferenceService.ts
+++ b/package/dashboard/src/service/workshopPhotoInferenceService.ts
@@ -25,9 +25,13 @@ export async function requestPhoto(env: Env, principal: ApiPrincipal, camera: un
if (!time || !Number.isFinite(+time))
invalid('Give the photo time as an ISO 8601 UTC time, such as 2026-09-24T14:32:05Z.');
const iso = time.toISOString();
- // The same camera and time within an hour reuse one photo instead of decoding it again.
- const id = `photo_${(await sha256(`${cam}|${iso}|${Math.floor(Date.now() / photoLifetime)}`)).slice(0, 32)}`;
- if (await ops.run(env, id)) return id;
+ // The same camera and time within an hour reuse one photo instead of decoding it again. One that
+ // failed, was cancelled or expired is made again under the next ID, so asking again works.
+ let id = `photo_${(await sha256(`${cam}|${iso}|${Math.floor(Date.now() / photoLifetime)}`)).slice(0, 32)}`;
+ while (await ops.run(env, id)) {
+ if (!['failed', 'cancelled', 'expired'].includes((await runDetail(env, id)).status)) return id;
+ id = `photo_${(await sha256(id)).slice(0, 32)}`;
+ }
const rows = await ops.recordings(
env,
cam,
@@ -112,7 +116,12 @@ export async function requestPhoto(env: Env, principal: ApiPrincipal, camera: un
from: iso,
until: new Date(+time + 1000).toISOString(),
};
- const state = await loadWorkshop(env);
+ let state = await loadWorkshop(env);
+ // A Workshop never saved has no document row, and both the run and its frame are checked against one.
+ if (!state.revision) {
+ await ops.saveState(env, 0, JSON.stringify(state.document), created.toISOString());
+ state = await loadWorkshop(env);
+ }
await ops.insertRun(
env,
{
diff --git a/package/dashboard/src/service/workshopScheduleInferenceService.ts b/package/dashboard/src/service/workshopScheduleInferenceService.ts
index ff1cdb3..f85bb62 100644
--- a/package/dashboard/src/service/workshopScheduleInferenceService.ts
+++ b/package/dashboard/src/service/workshopScheduleInferenceService.ts
@@ -11,10 +11,9 @@ import type {
import * as ops from '../db/WorkshopScheduleInferenceOps';
import { run as existingRun } from '../db/WorkshopInferenceOps';
import { createRun, loadWorkshop } from './workshopInferenceService';
-import { noMedia } from './workshopValidationInferenceService';
+import { arrivalMinutes, noMedia } from './workshopValidationInferenceService';
// ponytail: fixed cadences. The cron fires every minute; these decide how often each trigger acts.
-const arrivalMinutes = 15; // new-footage runs batch this many minutes of uploads
const slotGrace = 60 * 60000; // a set-time slot still starts if the scheduler missed its minute
const followLookback = 60 * 60000; // a new follow schedule ignores upstream runs older than this
@@ -31,7 +30,7 @@ async function due(
// Yesterday's slot covers a scheduler minute missed just before midnight.
const slot = new Date(`${new Date(+now - back * 86400000).toISOString().slice(0, 10)}T${s.at}:00Z`);
if (s.days.includes(slot.getUTCDay()) && slot <= now && +now - +slot < slotGrace)
- return { key: slot.toISOString(), label: `Scheduled ${s.at} UTC` };
+ return { key: slot.toISOString(), label: `Scheduled ${s.at} UTC`, slot };
}
return null;
}
@@ -71,7 +70,12 @@ async function runSchedule(env: Env, state: WorkshopState, workflow: Workflow, s
await createRun(
env,
{ id, workflow_id: workflow.id, runtime_id: runtime.id, revision: state.revision, mode: 'run' },
- { now, name: `${workflow.name} · ${trigger.label}`, ...(raw && s.trigger !== 'arrival' ? {} : { since }) },
+ // A slot that starts late, after midnight, still processes the day of its slot.
+ {
+ now: trigger.slot ?? now,
+ name: `${workflow.name} · ${trigger.label}`,
+ ...(raw && s.trigger !== 'arrival' ? {} : { since }),
+ },
);
await record('started', trigger.label, id);
} catch (error) {
diff --git a/package/dashboard/src/service/workshopValidationInferenceService.ts b/package/dashboard/src/service/workshopValidationInferenceService.ts
index c1ebe09..7e23fa3 100644
--- a/package/dashboard/src/service/workshopValidationInferenceService.ts
+++ b/package/dashboard/src/service/workshopValidationInferenceService.ts
@@ -18,6 +18,8 @@ export const relativeDays: Record<string, number> = { 'run-day': 0, 'previous-da
export const isRelativeDay = (day: unknown) => typeof day === 'string' && Object.hasOwn(relativeDays, day);
// ponytail: fixed margin for the relay to upload a window's last clips; raise it if uploads lag.
export const uploadGraceMinutes = 10;
+// ponytail: fixed cadence; new-footage runs batch this many minutes of uploads.
+export const arrivalMinutes = 15;
export const noMedia = 'The selected input has no available media.';
export function validateDocument(value: unknown): WorkshopDocument {
if (
@@ -322,6 +324,18 @@ function validateSchedule(d: Record<string, unknown>, footage?: Record<string, u
);
}
}
+ if (d.trigger === 'arrival' && footage) {
+ // Each run takes the window of the day it runs on, so uploads for a window must land before the day's last check.
+ if (footage.day !== 'run-day')
+ invalid(
+ 'When new footage arrives takes recordings as they upload. Choose Day of each run in the footage filter.',
+ );
+ const latest = 1440 - arrivalMinutes - uploadGraceMinutes;
+ if (time(footage.until) && minutes(footage.until) > latest)
+ invalid(
+ `End the window by ${String(Math.floor(latest / 60)).padStart(2, '0')}:${String(latest % 60).padStart(2, '0')} UTC so its last clips upload before the day's final new-footage check.`,
+ );
+ }
if (d.trigger === 'workflow' && !identifier(d.workflowId)) invalid('Choose the workflow this one follows.');
}
export function resolveDay(day: unknown, now: Date) {
diff --git a/package/dashboard/src/utils/configStorageUtil.ts b/package/dashboard/src/utils/configStorageUtil.ts
index 5e47431..e6bfa45 100644
--- a/package/dashboard/src/utils/configStorageUtil.ts
+++ b/package/dashboard/src/utils/configStorageUtil.ts
@@ -4,5 +4,8 @@ export const DAY_MS = 24 * HOUR_MS;
/** A camera whose newest segment arrived longer ago than this is reported stale. */
export const STALE_AFTER_MS = 3 * 60_000;
+/** Recordings are 60-second segments; a listing looks this far before its window for one still running into it. */
+export const LONGEST_RECORDING_MS = HOUR_MS;
+
/** Longest time range one segments request may cover. */
export const MAX_RANGE_MS = 48 * HOUR_MS;
diff --git a/package/dashboard/test-ui/archive-links.test.tsx b/package/dashboard/test-ui/archive-links.test.tsx
new file mode 100644
index 0000000..aa6b47b
--- /dev/null
+++ b/package/dashboard/test-ui/archive-links.test.tsx
@@ -0,0 +1,149 @@
+import { forwardRef, useImperativeHandle } from 'react';
+import { act, cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react';
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+
+vi.hoisted(() => {
+ const tooltip = document.createElement('div');
+ tooltip.id = 'tooltip';
+ document.body.append(tooltip);
+});
+type Shown = { segment: { start: string }; offset: number } | null;
+const seen: { selection: Shown; onProgress: (start: string, offset: number) => void }[] = [];
+const skip = vi.fn();
+vi.mock('../ui/components/viewer', () => ({
+ Viewer: forwardRef(function Viewer(props: { selection: Shown; onProgress: never }, ref) {
+ useImperativeHandle(ref, () => ({ skip, toggle: vi.fn(), markLoop: vi.fn() }));
+ seen.push({ selection: props.selection, onProgress: props.onProgress });
+ return <div>viewer</div>;
+ }),
+}));
+import App from '../ui/App';
+import { TooltipProvider } from '../ui/components/ui/tooltip';
+
+// Yesterday has a recording every minute from 08:00 to 16:00 UTC.
+const day = new Date(Date.now() - 86400000).toISOString().slice(0, 10);
+const at = `${day}T10:05:00.000Z`;
+const shown = () => seen.at(-1)?.selection;
+let failures = 0;
+const noFootage = { healthy: false, newest_start: null, newest_uploaded_at: null, age_seconds: null };
+let health: object = noFootage;
+beforeEach(() => {
+ seen.length = 0;
+ failures = 0;
+ health = noFootage;
+ skip.mockClear();
+ vi.stubGlobal(
+ 'ResizeObserver',
+ class {
+ observe() {}
+ unobserve() {}
+ disconnect() {}
+ },
+ );
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(async (path: string) => {
+ const json = (body: unknown, status = 200) =>
+ new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } });
+ if (path.startsWith('/api/overview'))
+ return json({
+ viewer: { email: '[email protected]' },
+ cameras: ['sky01'],
+ cam: 'sky01',
+ generated_at: new Date().toISOString(),
+ health,
+ last_24h: { segments: 0, bytes: 0, seconds: 0 },
+ retained: { segments: 0, bytes: 0, seconds: 0 },
+ days: [],
+ projection: { bytes_per_day: null, steady_state_bytes: null },
+ });
+ if (path.startsWith('/api/segments')) {
+ if (failures > 0 && failures--) return json({ error: 'busy' }, 503);
+ const q = new URLSearchParams(path.split('?')[1]);
+ const from = Date.parse(q.get('from')!),
+ to = Date.parse(q.get('to')!);
+ const segments = [];
+ for (let t = Date.parse(`${day}T08:00:00Z`); t < Date.parse(`${day}T16:00:00Z`); t += 60000)
+ if (t >= from && t < to)
+ segments.push({
+ start: new Date(t).toISOString(),
+ size: 1,
+ duration_s: 60,
+ frames: null,
+ width: null,
+ height: null,
+ bitrate: null,
+ codec: null,
+ });
+ return json({ segments });
+ }
+ if (path.startsWith('/api/availability')) return json({ days: [] });
+ throw new Error(path);
+ }),
+ );
+});
+afterEach(() => {
+ cleanup();
+ vi.unstubAllGlobals();
+});
+const open = (search: string) => {
+ history.replaceState(null, '', `/?${search}`);
+ render(
+ <TooltipProvider>
+ <App />
+ </TooltipProvider>,
+ );
+};
+
+it('opens a linked moment even when the link names a different calendar day', async () => {
+ // A link copied in another time zone can name the day before or after the moment's day here.
+ const elsewhere = new Date(Date.parse(day) - 3 * 86400000).toISOString().slice(0, 10);
+ open(`day=${elsewhere}&tz=local&cam=sky01&at=${encodeURIComponent(at)}&t=12`);
+ await waitFor(() => expect(shown()?.segment.start).toBe(at));
+ expect(shown()!.offset).toBe(12);
+});
+
+it('opens the linked moment when Retry follows a failed first load', async () => {
+ failures = 1;
+ open(`day=${day}&tz=UTC&cam=sky01&at=${encodeURIComponent(at)}&t=30`);
+ const retry = await screen.findByRole('button', { name: 'Retry' });
+ await act(async () => {
+ fireEvent.click(retry);
+ });
+ await waitFor(() => expect(shown()?.segment.start).toBe(at));
+ expect(shown()!.offset).toBe(30);
+});
+
+it('keeps the playback position when the time zone changes', async () => {
+ open(`day=${day}&tz=UTC&cam=sky01&at=${encodeURIComponent(at)}&t=30`);
+ await waitFor(() => expect(shown()?.segment.start).toBe(at));
+ await act(async () => {
+ seen.at(-1)!.onProgress(at, 42);
+ });
+ fireEvent.click(screen.getByRole('button', { name: /Choose recording date/ }));
+ const zone = await screen.findByLabelText('Time zone', {}, { timeout: 10000 });
+ await act(async () => {
+ fireEvent.change(zone, { target: { value: 'local' } });
+ });
+ await waitFor(() => expect(location.search).toContain('tz=local'));
+ await waitFor(() => expect(shown()?.offset).toBe(42));
+ expect(shown()!.segment.start).toBe(at);
+});
+
+it('moves through the minute picker with the arrow keys without seeking the player', async () => {
+ open(`tz=UTC&day=${day}&cam=sky01`);
+ // The picker shows the hour of the newest recording, 15:00 to 15:59.
+ const minute = await waitFor(() => screen.getByRole('button', { name: /^15:05, 1 recording/ }));
+ minute.focus();
+ fireEvent.keyDown(minute, { key: 'ArrowRight' });
+ expect(document.activeElement?.getAttribute('data-minute')).toBe('6');
+ expect(skip).not.toHaveBeenCalled();
+});
+
+it("says footage is arriving by the server's measure, whatever this computer's clock says", async () => {
+ // The server saw an upload 20 seconds ago; this computer's clock runs an hour ahead of it.
+ const uploaded = new Date(Date.now() - 3600_000).toISOString();
+ health = { healthy: true, newest_start: uploaded, newest_uploaded_at: uploaded, age_seconds: 20 };
+ open(`day=${day}&tz=UTC&cam=sky01`);
+ expect(await screen.findByText('Receiving footage')).toBeTruthy();
+});
diff --git a/package/dashboard/test-ui/openapi.check.mjs b/package/dashboard/test-ui/openapi.check.mjs
index 7002637..c796098 100644
--- a/package/dashboard/test-ui/openapi.check.mjs
+++ b/package/dashboard/test-ui/openapi.check.mjs
@@ -1,6 +1,6 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
-import { readFileSync } from 'node:fs';
+import { readdirSync, readFileSync } from 'node:fs';
import SwaggerParser from '@apidevtools/swagger-parser';
import { withKeyPaths } from '../src/utils/openapiSharedUtil.ts';
@@ -16,6 +16,20 @@ test('the downloadable OpenAPI document validates and gives operations unique ID
assert.ok(ids.every(Boolean));
});
+test('every key scope the database accepts is in the document', () => {
+ const migrations = new URL('../db/migrations/', import.meta.url);
+ const latest = readdirSync(migrations)
+ .filter(f => f.endsWith('.sql'))
+ .sort()
+ .findLast(f => readFileSync(new URL(f, migrations), 'utf8').includes('CHECK(scope IN'));
+ const accepted = readFileSync(new URL(latest, migrations), 'utf8').match(/CHECK\(scope IN \(([^)]*)\)/)[1];
+ const scopes = [...accepted.matchAll(/'([^']+)'/g)].map(m => m[1]).sort();
+ const text = readFileSync(new URL('../openapi.json', import.meta.url), 'utf8');
+ const enums = [...text.matchAll(/"enum": \[("footage:read"[^\]]*)\]/g)].map(m => JSON.parse(`[${m[1]}]`).sort());
+ assert.ok(enums.length >= 3);
+ for (const listed of enums) assert.deepEqual(listed, scopes);
+});
+
test('the served document adds every website operation under /v1 for keys, except key management', async () => {
const file = JSON.parse(readFileSync(new URL('../openapi.json', import.meta.url), 'utf8'));
const served = withKeyPaths(structuredClone(file));
diff --git a/package/dashboard/test-ui/polled.test.tsx b/package/dashboard/test-ui/polled.test.tsx
new file mode 100644
index 0000000..27db13a
--- /dev/null
+++ b/package/dashboard/test-ui/polled.test.tsx
@@ -0,0 +1,45 @@
+import { act, cleanup, render } from '@testing-library/react';
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+import { DatasetVersions } from '../ui/features/workshop/components/dataset-versions';
+
+let hidden = true;
+beforeEach(() => {
+ hidden = true;
+ Object.defineProperty(document, 'hidden', { configurable: true, get: () => hidden });
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(
+ async () =>
+ new Response(JSON.stringify({ versions: [], accepted: 0 }), {
+ headers: { 'content-type': 'application/json' },
+ }),
+ ),
+ );
+});
+afterEach(() => {
+ cleanup();
+ vi.useRealTimers();
+ vi.unstubAllGlobals();
+});
+
+it('reads as soon as a page opened in a background tab is shown, then keeps one poll going', async () => {
+ vi.useFakeTimers();
+ render(<DatasetVersions bucketId="b" />);
+ expect(fetch).not.toHaveBeenCalled();
+ hidden = false;
+ await act(async () => {
+ document.dispatchEvent(new Event('visibilitychange'));
+ await vi.advanceTimersByTimeAsync(100);
+ });
+ expect(fetch).toHaveBeenCalledTimes(1);
+ // Showing the page again while the poll is current reads nothing extra.
+ await act(async () => {
+ document.dispatchEvent(new Event('visibilitychange'));
+ await vi.advanceTimersByTimeAsync(29000);
+ });
+ expect(fetch).toHaveBeenCalledTimes(1);
+ await act(async () => {
+ await vi.advanceTimersByTimeAsync(1500);
+ });
+ expect(fetch).toHaveBeenCalledTimes(2);
+});
diff --git a/package/dashboard/test-ui/viewer-loading.test.tsx b/package/dashboard/test-ui/viewer-loading.test.tsx
index 604414e..596207d 100644
--- a/package/dashboard/test-ui/viewer-loading.test.tsx
+++ b/package/dashboard/test-ui/viewer-loading.test.tsx
@@ -33,7 +33,7 @@ afterEach(() => {
vi.unstubAllGlobals();
});
-it('renders one indicator during initial loading and one during subsequent buffering', () => {
+function renderViewer() {
const segment: Segment = {
start: '2026-09-22T19:30:00.000Z',
duration_s: 60,
@@ -76,6 +76,10 @@ it('renders one indicator during initial loading and one during subsequent buffe
/>
</TooltipProvider>,
);
+}
+
+it('renders one indicator during initial loading and one during subsequent buffering', () => {
+ renderViewer();
const media = screen.getByLabelText('Recorded camera footage');
fireEvent.play(media);
fireEvent.waiting(media);
@@ -92,3 +96,24 @@ it('renders one indicator during initial loading and one during subsequent buffe
fireEvent.playing(media);
expect(screen.queryByRole('status')).toBeNull();
});
+
+it('tells a recording that cannot be downloaded from one the browser cannot decode', async () => {
+ for (const [response, message] of [
+ [new Response('', { status: 503 }), 'could not be loaded'],
+ [new Response('x', { status: 206, headers: { 'Content-Type': 'video/mp4' } }), 'cannot decode'],
+ ] as const) {
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(async () => response),
+ );
+ renderViewer();
+ const media = screen.getByLabelText('Recorded camera footage');
+ // Browsers report a failed HTTP load before metadata as MEDIA_ERR_SRC_NOT_SUPPORTED, like a codec they lack.
+ Object.defineProperty(media, 'error', { configurable: true, value: { code: 4 } });
+ await act(async () => {
+ fireEvent.error(media);
+ });
+ expect(screen.getByText(new RegExp(message))).toBeTruthy();
+ cleanup();
+ }
+});
diff --git a/package/dashboard/test-ui/workshop-schedules.test.tsx b/package/dashboard/test-ui/workshop-schedules.test.tsx
index 083c664..337ce7e 100644
--- a/package/dashboard/test-ui/workshop-schedules.test.tsx
+++ b/package/dashboard/test-ui/workshop-schedules.test.tsx
@@ -11,7 +11,7 @@ import {
} from '../ui/features/workshop/service/workflowInferenceService';
import { restoreWorkshop } from '../ui/features/workshop/service/workshopDraftInferenceService';
import { cameraArchive, createBucket } from '../ui/features/workshop/service/bucketInferenceService';
-import { nextSlot } from '../ui/features/workshop/utils/scheduleInferenceUtil';
+import { latestUntil, nextSlot } from '../ui/features/workshop/utils/scheduleInferenceUtil';
import type { WorkshopBucket } from '../ui/features/workshop/models/bucketInferencemodel';
import type { ModelVersion } from '../src/models/modelInferencemodel';
import type { ScheduleStep, WorkflowDraft } from '../ui/features/workshop/models/workflowInferencemodel';
@@ -97,6 +97,13 @@ it('waits for a same-day window to upload and requires days and a time', () => {
expect(messages(scheduled('Late', { at: '13:10' }, 'run-day'))).toEqual([]);
expect(messages(scheduled('No days', { days: [] }))).toContain('Choose the days and UTC time for this schedule.');
});
+it("keeps a new-footage schedule on the day of each run, ending before the day's last check", () => {
+ expect(messages(scheduled('Arrival', { trigger: 'arrival' }))).toContain(
+ 'When new footage arrives takes recordings as they upload. Choose Day of each run in the footage filter.',
+ );
+ expect(messages(scheduled('Arrival today', { trigger: 'arrival' }, 'run-day'))).toEqual([]);
+ expect(latestUntil({ ...schedule, trigger: 'arrival' })).toBe('23:35:00');
+});
it('flags workflows that would start each other in a loop', () => {
const a = scheduled('A'),
b = scheduled('B');
diff --git a/package/dashboard/test-ui/workshop-sightings.test.tsx b/package/dashboard/test-ui/workshop-sightings.test.tsx
index b7d5856..2221803 100644
--- a/package/dashboard/test-ui/workshop-sightings.test.tsx
+++ b/package/dashboard/test-ui/workshop-sightings.test.tsx
@@ -255,6 +255,26 @@ const openBucket = async (itemId = '', onItem = vi.fn()) => {
);
return screen.findByRole('button', { name: /^Plane, .*, 2 photos$/ });
};
+it('asks before Back from a bucket feed drops marks that are not saved yet', async () => {
+ const onLeave = vi.fn();
+ render(
+ <TooltipProvider>
+ <BucketExplorer bucketId="planes" title="Night planes" initialRun="run" onItem={vi.fn()} onLeave={onLeave} />
+ </TooltipProvider>,
+ );
+ const sighting = await screen.findByRole('button', { name: /^Plane, .*, 2 photos$/ });
+ await act(async () => {
+ fireEvent.click(sighting);
+ });
+ await screen.findByRole('article', { name: /Sighting 1 of 2/ });
+ await key('2'); // Meteorite, staged until Enter
+ await act(async () => {
+ fireEvent.click(screen.getByRole('button', { name: 'Back to bucket' }));
+ });
+ expect(onLeave).toHaveBeenCalledTimes(1);
+ expect(screen.getByRole('article', { name: /Sighting 1 of 2/ })).toBeTruthy();
+ expect(puts).toEqual([]);
+});
const tile = (photos: 1 | 2) =>
screen.getByRole('button', { name: new RegExp(`^Plane, .*, ${photos} ${photos === 1 ? 'photo' : 'photos'}`) });
const feedCard = (n: number) => screen.getByRole('article', { name: `Sighting ${n} of 2: Plane` });
@@ -487,6 +507,43 @@ const openReviewer = async (onItem = vi.fn()) => {
const sighting = (n: number) => screen.getByRole('article', { name: `Sighting ${n} of 2: Plane` });
const reviews = () => puts.map(p => [p.id, p.body.review]).sort();
+// jsdom's :focus-visible does not follow how the focus came; a browser's matches only keyboard focus.
+const focusVisible = (visible: boolean) => {
+ const matches = HTMLElement.prototype.matches;
+ vi.spyOn(HTMLElement.prototype, 'matches').mockImplementation(function (this: HTMLElement, selector: string) {
+ return selector === ':focus-visible' ? visible && this === document.activeElement : matches.call(this, selector);
+ });
+};
+it('leaves Enter and Space on a button reached from the keyboard to that button', async () => {
+ const first = await openReviewer();
+ const none = within(first).getByRole('button', { name: /None/ });
+ none.focus();
+ focusVisible(true); // as a browser reports a button reached with Tab
+ for (const key of ['Enter', ' '])
+ await act(async () => {
+ fireEvent.keyDown(none, { key });
+ });
+ expect(reviews()).toEqual([]);
+});
+
+it('keeps Enter as Confirm when a click left focus on a button', async () => {
+ const first = await openReviewer();
+ const confirm = within(first).getByRole('button', { name: 'Confirm' });
+ focusVisible(false); // a browser keeps focus on a clicked button, but not :focus-visible
+ confirm.focus();
+ await act(async () => {
+ fireEvent.click(confirm);
+ });
+ await act(async () => {
+ fireEvent.keyDown(confirm, { key: 'Enter' });
+ });
+ expect(reviews()).toEqual([
+ ['a', 'accepted'],
+ ['b', 'accepted'],
+ ['c', 'accepted'],
+ ]);
+});
+
it('plays the sighting first, and Confirm decides all its photos, moves on, and can be undone', async () => {
const onItem = vi.fn();
const first = await openReviewer(onItem);
diff --git a/package/dashboard/test/agent.test.ts b/package/dashboard/test/agent.test.ts
index 133bd92..94fac57 100644
--- a/package/dashboard/test/agent.test.ts
+++ b/package/dashboard/test/agent.test.ts
@@ -246,6 +246,25 @@ describe('Agent MCP server', () => {
expect(prompt.result.messages[0].content.text).toContain('Skill: verify your changes');
});
+ it('takes a photo before the Workshop document was ever saved, and again after it failed', async () => {
+ const p = await processor();
+ const start = minutesAgo(600);
+ await clip(start);
+ const at = new Date(+start + 5_000).toISOString();
+ const failed = await ok(agent.token, 'footage', { action: 'photo', at, wait_seconds: 0 });
+ const first = (await p.claim())!;
+ expect(first.run_id).toBe(failed.photo_id);
+ const lease = { generation: first.generation, lease_token: first.lease_token };
+ expect((await p.machine(`/jobs/${first.id}/fail`, { ...lease, error: 'decode_failed' })).status).toBe(200);
+ // Asking again makes a new photo instead of handing back the failed one.
+ const again = await ok(agent.token, 'footage', { action: 'photo', at, wait_seconds: 0 });
+ expect(again.photo_id).not.toBe(failed.photo_id);
+ const job = (await p.claim())!;
+ expect(job.run_id).toBe(again.photo_id);
+ await p.finish(job, { bucket_id: 'camera-archive', model_node: null, offset: 5, annotations: [] });
+ expect((await ok(agent.token, 'footage', { action: 'photo', at, wait_seconds: 0 })).photo_id).toBe(again.photo_id);
+ });
+
it('builds a workflow, protects edits with revisions, test-runs it, and wakes a watch on live progress', async () => {
const p = await processor();
const start = minutesAgo(90);
@@ -279,6 +298,16 @@ describe('Agent MCP server', () => {
});
expect(created).toMatchObject({ ready: true, issue: null });
expect(created.steps.find((s: { id: string }) => s.id === 'keep').bucketId).toBe(bucket.bucket_id);
+ // A key for sky01 cannot save a workflow, or a schedule the cron would run, for another camera.
+ expect(
+ (
+ await call(agent.token, 'workflows', {
+ action: 'create',
+ name: 'Elsewhere',
+ steps: [{ id: 'input', kind: 'input', camera: 'sky02' }],
+ })
+ ).error,
+ ).toContain('cannot access that camera');
const id = created.workflow_id;
expect(
(
@@ -441,6 +470,9 @@ describe('Agent MCP server', () => {
const temporary = await key('agent');
const [download] = (await ok(temporary.token, 'footage', { action: 'download', at })).files;
+ expect((await call(temporary.token, 'footage', { action: 'download', at: 'noon' })).error).toContain(
+ 'at must be an ISO 8601 timestamp',
+ );
expect(download.filename).toMatch(/^sky01_\d{8}T\d{6}Z\.mp4$/);
expect((await fetchLink(download.url)).status).toBe(404); // signed and authorized; the test clip has no bytes in the store
expect((await web(`/api/keys/${temporary.key.id}`, undefined, 'DELETE')).status).toBe(200);
diff --git a/package/dashboard/test/footage-window.test.ts b/package/dashboard/test/footage-window.test.ts
index a888d40..774ffc1 100644
--- a/package/dashboard/test/footage-window.test.ts
+++ b/package/dashboard/test/footage-window.test.ts
@@ -13,7 +13,9 @@ describe('footage window', () => {
for (const start of days) {
const from = new Date(+start + 63_060).toISOString(),
to = new Date(+start + 123_060).toISOString();
- if ((await queryRecordings(env.CATALOG, CAM, from, to, asOf, null, 10)).length) listed.push(start.toISOString());
+ const earliest = new Date(+start - 3_600_000).toISOString();
+ if ((await queryRecordings(env.CATALOG, CAM, from, to, earliest, asOf, null, 10)).length)
+ listed.push(start.toISOString());
}
expect(listed).toEqual([]);
});
diff --git a/package/dashboard/test/workshop.test.ts b/package/dashboard/test/workshop.test.ts
index fa812e1..7ea38e2 100644
--- a/package/dashboard/test/workshop.test.ts
+++ b/package/dashboard/test/workshop.test.ts
@@ -76,7 +76,7 @@ async function setup(
} = {},
) {
const suffix = crypto.randomUUID(),
- device = 'device-' + suffix;
+ device = 'device.' + suffix; // a dot, which device IDs allow
await web('/api/models/devices', { id: device });
const runtimeResponse = await web('/api/workshop/runtimes', {
device_id: device,
@@ -317,6 +317,11 @@ describe('bucket detection library', () => {
'review=wrong',
])
expect((await web(`/api/workshop/buckets/${s.bucket}/browse?${params}`)).status).toBe(400);
+ // A frame full of stars has more labels than the filter's top value, 100, which means 100 or more.
+ await env.CATALOG.prepare('UPDATE workshop_items SET annotation_json=? WHERE id=?')
+ .bind(JSON.stringify(Array(120).fill(label)), original.id)
+ .run();
+ expect((await browse('min_objects=100')).items.map(i => i.id)).toEqual([original.id]);
const detail = await (await web(`/api/workshop/items/${original.id}`)).json<
import('../src/models/bucketBrowseInferencemodel').BucketItemDetail
>();
@@ -1373,6 +1378,13 @@ describe('Workshop saved state and execution', () => {
const [first, second] = review.items,
label = JSON.parse(first.annotation_json)[0];
const decide = (id: string, body: unknown) => web(`/api/workshop/items/${id}/review`, body, 'PUT');
+ // A busy frame: as many labels as the service takes, as the website sends them, fit the body limit.
+ const busy = Array.from({ length: 100 }, () => ({
+ ...label,
+ model_node: 'n2',
+ track_id: 'track-' + '0'.repeat(32),
+ }));
+ expect((await decide(first.id, { review: 'pending', annotations: busy })).status).toBe(200);
// Relabel within the model's classes only, with a real box.
expect(
(await decide(first.id, { review: 'accepted', annotations: [{ ...label, class_name: 'Cat' }] })).status,
@@ -1428,8 +1440,8 @@ describe('Workshop schedules', () => {
// Footage is seeded on yesterday's UTC date; the scheduler runs at chosen times the next day.
const day = new Date(Date.now() - 86400000).toISOString().slice(0, 10);
const at = (time: string) => new Date(Date.parse(`${day}T${time}Z`) + 86400000);
- async function clip(time: string) {
- const start = new Date(`${day}T${time}Z`);
+ async function clip(time: string, date = day) {
+ const start = new Date(`${date}T${time}Z`);
await seedSegment({ start });
await env.CATALOG.prepare('UPDATE segments SET sha256=? WHERE start=?').bind(sourceHash, start.toISOString()).run();
}
@@ -1461,6 +1473,7 @@ describe('Workshop schedules', () => {
w.edges.push({ id: 'e9', source: 'n9', target: 'n0', sourceHandle: 'out', targetHandle: 'in' });
});
expect(response.status, await response.clone().text()).toBe(200);
+ await s.revoke(); // the runtime has checked in; schedules never use the key
return s.request.workflow_id;
}
const runs = async (workflow: string) =>
@@ -1509,8 +1522,9 @@ describe('Workshop schedules', () => {
});
});
it('runs new footage once per quarter hour and only on recordings uploaded since', async () => {
- await clip('04:00:00');
- const flow = await scheduled({ trigger: 'arrival' }, { day: 'previous-day', from: '04:00:00', until: '04:10:00' });
+ const today = at('00:00:00').toISOString().slice(0, 10);
+ await clip('04:00:00', today);
+ const flow = await scheduled({ trigger: 'arrival' }, { day: 'run-day', from: '04:00:00', until: '04:10:00' });
await runSchedules(env, at('04:15:00'));
await runSchedules(env, at('04:16:00'));
expect(await runs(flow)).toEqual([
@@ -1518,7 +1532,7 @@ describe('Workshop schedules', () => {
]);
await runSchedules(env, at('04:30:00'));
expect(await outcome(flow)).toMatchObject({ status: 'skipped', message: 'Nothing new to process.' });
- await clip('04:02:00');
+ await clip('04:02:00', today);
await runSchedules(env, at('04:45:00'));
const all = await runs(flow);
expect(all.map(r => r.jobs)).toEqual([1, 1]);
@@ -1527,7 +1541,29 @@ describe('Workshop schedules', () => {
.bind(all[1].id)
.all<{ input_json: string }>()
).results;
- expect(JSON.parse(input.input_json).recording.start).toBe(`${day}T04:02:00.000Z`);
+ expect(JSON.parse(input.input_json).recording.start).toBe(`${today}T04:02:00.000Z`);
+ });
+ it('refuses new-footage schedules that would miss uploads at midnight', async () => {
+ const flow = await scheduled({ trigger: 'arrival' }, { day: 'previous-day', from: '04:00:00', until: '04:10:00' });
+ await runSchedules(env, at('04:15:00'));
+ expect((await outcome(flow))?.message).toContain('Choose Day of each run');
+ await patch(flow, w => Object.assign(w.nodes[1].data, { day: 'run-day', from: '23:00:00', until: '23:50:00' }));
+ await runSchedules(env, at('04:30:00'));
+ expect((await outcome(flow))?.message).toBe(
+ "End the window by 23:35 UTC so its last clips upload before the day's final new-footage check.",
+ );
+ });
+ it('processes the day of a set-time slot that started late, after midnight', async () => {
+ const before = new Date(Date.parse(day) - 86400000).toISOString().slice(0, 10);
+ await clip('06:00:00', before);
+ await clip('06:00:00');
+ const flow = await scheduled({ at: '23:50' }, { day: 'previous-day', from: '06:00:00', until: '06:05:00' });
+ await runSchedules(env, at('00:20:00')); // yesterday's 23:50 slot, half an hour late
+ const [run] = await runs(flow);
+ const input = await env.CATALOG.prepare('SELECT input_json FROM workshop_jobs WHERE run_id=?')
+ .bind(run.id)
+ .first<{ input_json: string }>();
+ expect(JSON.parse(input!.input_json).recording.start).toBe(`${before}T06:00:00.000Z`);
});
it('follows another workflow only when its finished run saved images', async () => {
await clip('05:00:00');
@@ -1694,6 +1730,22 @@ describe('Workshop tracks', () => {
await s.revoke();
});
+ it('keeps a minute of frames that each carry a long track', async () => {
+ // A fixed light seen all minute rides, with its whole path, in every frame: 1.7 MB of results.
+ const s = await setup({ interval: 5 }),
+ { job } = await (await s.claim()).json<{ job: WorkshopJob }>();
+ const path = Array.from({ length: 4000 }, (_, i) => [i * 0.015, 0.2, 0.3, 0.002, 0.004, 0, 0.5]);
+ const light = { ...track(job.id, { motion: 'stationary', speed: 0, end: 59.985, detections: 4000, path }) };
+ delete (light as { scan?: unknown }).scan;
+ const body = tracked(await commit(s, job, 12, false), job.id, light);
+ expect(JSON.stringify(body).length).toBeGreaterThan(1_500_000);
+ const response = await api(s.token, `/jobs/${job.id}/complete`, body);
+ expect(response.status, await response.clone().text()).toBe(200);
+ const { items } = await (await web('/api/workshop/buckets/' + s.bucket)).json<{ items: WorkshopItem[] }>();
+ expect(items.filter(i => i.job_id === job.id)).toHaveLength(12);
+ await s.revoke();
+ });
+
it('keeps each frame with its tracks, refuses malformed ones, and exports them', async () => {
const s = await setup(),
{ job } = await (await s.claim()).json<{ job: WorkshopJob }>(),
diff --git a/package/dashboard/ui/App.tsx b/package/dashboard/ui/App.tsx
index 2e12d02..f924c16 100644
--- a/package/dashboard/ui/App.tsx
+++ b/package/dashboard/ui/App.tsx
@@ -1,7 +1,7 @@
import { BrandIndicator, LoadingStatus } from './components/brand-indicator';
import { BrandLogo } from './components/brand-logo';
import { createPortal } from 'react-dom';
-import { lazy, Suspense, useCallback, useEffect, useRef, useState } from 'react';
+import { lazy, Suspense, useCallback, useEffect, useMemo, useRef, useState } from 'react';
import {
CalendarDays,
ChevronDown,
@@ -187,9 +187,11 @@ function ArchiveApp({ theme, setTheme }: { theme: string; setTheme: (theme: stri
const isSaved = Boolean(segment && saved.has(`${cam}/${segment.start}`));
const previousDay = adjacentRecordedDay(availability.days, route.day, -1);
const nextDay = adjacentRecordedDay(availability.days, route.day, 1);
- const uploadAge = overview?.health.newest_uploaded_at
- ? Math.max(0, (now - Date.parse(overview.health.newest_uploaded_at)) / 1000)
- : null;
+ // The server measured the age; only the time since its answer arrived is counted here, so a computer whose
+ // clock is off neither hides nor invents footage.
+ const received = useMemo(() => Date.now(), [overview]);
+ const uploadAge =
+ overview?.health.age_seconds != null ? overview.health.age_seconds + Math.max(0, now - received) / 1000 : null;
const health = uploadAge === null ? 'Waiting for footage' : `Last upload ${formatAge(uploadAge)}`;
const changeDate = (key: string) => {
if (availability.days?.has(key)) {
@@ -272,8 +274,15 @@ function ArchiveApp({ theme, setTheme }: { theme: string; setTheme: (theme: stri
setBrowserOpen(false);
setDetailsOpen(true);
};
+ // The same recording at the same moment, shown in the other time zone.
const changeZone = (zone: Zone) =>
- a.changeDay(segment ? dayKey(new Date(segment.start), zone) : route.day, zone, segment?.start ?? null);
+ a.changeDay(
+ segment ? dayKey(new Date(segment.start), zone) : route.day,
+ zone,
+ segment?.start ?? null,
+ false,
+ segment && a.playbackPosition.current.start === segment.start ? a.playbackPosition.current.offset : 0,
+ );
function browseRecordings(exactTime = false) {
browserReturnFocus.current = document.activeElement as HTMLElement;
setFocusTime(exactTime);
@@ -291,7 +300,8 @@ function ArchiveApp({ theme, setTheme }: { theme: string; setTheme: (theme: stri
}, [compact, fullscreen, route.view]);
useEffect(() => {
const keydown = (e: KeyboardEvent) => {
- if (e.metaKey || e.ctrlKey || e.altKey || shortcuts || route.view !== 'recordings') return;
+ // A control that handled the key, such as the minute picker's arrows, keeps it from the player.
+ if (e.defaultPrevented || e.metaKey || e.ctrlKey || e.altKey || shortcuts || route.view !== 'recordings') return;
const target = e.target as HTMLElement;
if (
target.closest(
diff --git a/package/dashboard/ui/components/viewer.tsx b/package/dashboard/ui/components/viewer.tsx
index 455866e..5793462 100644
--- a/package/dashboard/ui/components/viewer.tsx
+++ b/package/dashboard/ui/components/viewer.tsx
@@ -910,14 +910,24 @@ export const Viewer = forwardRef<ViewerHandle, Props>(function Viewer(p, ref) {
onError={() => {
if (slot !== activeSlotRef.current) return;
steps.cancel();
- if (selected.current) {
- setError(
- video.current?.error?.code === 2
- ? 'The recording could not be loaded. Check your connection and select it to retry.'
- : 'This browser cannot decode this HEVC recording. Download the original to play it locally.',
- );
- setBuffering(false);
- }
+ if (!selected.current) return;
+ setBuffering(false);
+ const unreachable = 'The recording could not be loaded. Check your connection and select it to retry.';
+ const clip = clipOf(video.current);
+ if (video.current?.error?.code === 2 || !clip) return setError(unreachable);
+ // A failed download (a missing file, the footage store offline, a lapsed sign-in) is reported as an
+ // unsupported source too, so read one byte to tell it from a recording the browser cannot decode.
+ void fetch(clip, { headers: { Range: 'bytes=0-0' } })
+ .then(r => r.ok && (r.headers.get('Content-Type') ?? '').startsWith('video/'))
+ .catch(() => false)
+ .then(playable => {
+ if (clipOf(video.current) !== clip) return;
+ setError(
+ playable
+ ? 'This browser cannot decode this HEVC recording. Download the original to play it locally.'
+ : unreachable,
+ );
+ });
}}
/>
))}
diff --git a/package/dashboard/ui/features/workshop/components/bucket-explorer.tsx b/package/dashboard/ui/features/workshop/components/bucket-explorer.tsx
index f1fbd0c..8eb123d 100644
--- a/package/dashboard/ui/features/workshop/components/bucket-explorer.tsx
+++ b/package/dashboard/ui/features/workshop/components/bucket-explorer.tsx
@@ -24,6 +24,7 @@ export function BucketExplorer({
initialRun = '',
onItem,
onEditState,
+ onLeave = action => action(),
}: {
bucketId: string;
title?: string;
@@ -31,6 +32,8 @@ export function BucketExplorer({
initialRun?: string;
onItem?: (id: string, visit?: boolean) => void;
onEditState?: (dirty: boolean, busy: boolean) => void;
+ /** Runs `action` now, or after the person agrees to drop unsaved marks. */
+ onLeave?: (action: () => void) => void;
}) {
const [preferences, setPreferences] = useState<BrowserPreferences>(() => {
const saved = readBrowserPreferences(bucketId);
@@ -219,7 +222,7 @@ export function BucketExplorer({
focused.current = id;
onItem?.(id);
}}
- onBack={close}
+ onBack={() => onLeave(close)}
onEditState={onEditState}
/>
)}
diff --git a/package/dashboard/ui/features/workshop/components/bucket-workspace.tsx b/package/dashboard/ui/features/workshop/components/bucket-workspace.tsx
index 2fed8e3..43992ff 100644
--- a/package/dashboard/ui/features/workshop/components/bucket-workspace.tsx
+++ b/package/dashboard/ui/features/workshop/components/bucket-workspace.tsx
@@ -108,6 +108,7 @@ export function BucketWorkspace({
itemId={itemId}
onItem={onItem}
onEditState={onEditState}
+ onLeave={onLeave}
/>
)}
</TabsContent>
diff --git a/package/dashboard/ui/features/workshop/components/footage-fields.tsx b/package/dashboard/ui/features/workshop/components/footage-fields.tsx
index 5d02b77..0bce1ee 100644
--- a/package/dashboard/ui/features/workshop/components/footage-fields.tsx
+++ b/package/dashboard/ui/features/workshop/components/footage-fields.tsx
@@ -50,7 +50,11 @@ export function FootageFields({
// A schedule reruns this workflow, so its day must move with each run and a same-day window must upload first.
const limit = latestUntil(schedule),
endBy = data.day === 'run-day' ? limit : null;
- const dayOptions = Object.entries(relativeDays).filter(([value]) => !(value === 'run-day' && limit === ''));
+ // New footage takes recordings as they upload, which is always the day of the run.
+ const dayOptions = Object.entries(relativeDays).filter(
+ ([value]) =>
+ !(value === 'run-day' && limit === '') && !(value === 'previous-day' && schedule?.trigger === 'arrival'),
+ );
useEffect(() => {
setFrom(data.from);
setUntil(data.until);
@@ -332,7 +336,7 @@ export function FootageFields({
{endBy !== null && data.until && fullTime(data.until) > endBy && (
<p className="wk-error wk-hint" role="alert">
{endBy
- ? `End the window by ${endBy} UTC, or schedule later.`
+ ? `End the window by ${endBy} UTC${schedule?.trigger === 'time' ? ', or schedule later' : ''}.`
: 'This schedule runs too early for Day of each run. Use Day before each run.'}
</p>
)}
diff --git a/package/dashboard/ui/features/workshop/components/sighting-batch.tsx b/package/dashboard/ui/features/workshop/components/sighting-batch.tsx
index 68c2d20..da7b94d 100644
--- a/package/dashboard/ui/features/workshop/components/sighting-batch.tsx
+++ b/package/dashboard/ui/features/workshop/components/sighting-batch.tsx
@@ -7,6 +7,12 @@ import { classColor, modelClasses, sightingClass } from '../utils/sightingInfere
const typing = (target: EventTarget | null) =>
target instanceof HTMLElement && !!target.closest('input, textarea, select, [contenteditable=true]');
+// Enter and Space press a button reached from the keyboard. A clicked button keeps focus in most browsers,
+// but is not :focus-visible, so after a click the keys stay the reviewer's shortcuts.
+const pressing = (e: KeyboardEvent) =>
+ (e.key === 'Enter' || e.key === ' ') &&
+ e.target instanceof HTMLElement &&
+ !!e.target.closest('button, a[href], [role=button]')?.matches(':focus-visible');
const time = (iso: string, seconds = false) =>
new Date(iso).toLocaleTimeString([], {
hour: 'numeric',
@@ -153,6 +159,7 @@ export function SightingBatch({
e.ctrlKey ||
e.metaKey ||
typing(e.target) ||
+ pressing(e) ||
document.querySelector('[role=dialog]')
)
return;
diff --git a/package/dashboard/ui/features/workshop/components/sighting-feed.tsx b/package/dashboard/ui/features/workshop/components/sighting-feed.tsx
index bacdda8..bd4cb04 100644
--- a/package/dashboard/ui/features/workshop/components/sighting-feed.tsx
+++ b/package/dashboard/ui/features/workshop/components/sighting-feed.tsx
@@ -28,6 +28,12 @@ const SPEEDS = [1, 2, 4];
const typing = (target: EventTarget | null) =>
target instanceof HTMLElement && !!target.closest('input, textarea, select, [contenteditable=true]');
+// Enter and Space press a button reached from the keyboard. A clicked button keeps focus in most browsers,
+// but is not :focus-visible, so after a click the keys stay the reviewer's shortcuts.
+const pressing = (e: KeyboardEvent) =>
+ (e.key === 'Enter' || e.key === ' ') &&
+ e.target instanceof HTMLElement &&
+ !!e.target.closest('button, a[href], [role=button]')?.matches(':focus-visible');
const without = (marks: Record<string, PhotoMark>, frames: WorkshopRunImage[]) => {
const next = { ...marks };
for (const f of frames) delete next[f.id];
@@ -211,7 +217,14 @@ export function SightingFeed({
useEffect(() => {
if (editing !== null) return;
function down(e: KeyboardEvent) {
- if (e.defaultPrevented || e.altKey || e.ctrlKey || typing(e.target) || document.querySelector('[role=dialog]'))
+ if (
+ e.defaultPrevented ||
+ e.altKey ||
+ e.ctrlKey ||
+ typing(e.target) ||
+ pressing(e) ||
+ document.querySelector('[role=dialog]')
+ )
return;
const key = e.key.toLowerCase();
if (e.metaKey) {
diff --git a/package/dashboard/ui/features/workshop/hooks/use-polled.ts b/package/dashboard/ui/features/workshop/hooks/use-polled.ts
index 929d20f..ce95531 100644
--- a/package/dashboard/ui/features/workshop/hooks/use-polled.ts
+++ b/package/dashboard/ui/features/workshop/hooks/use-polled.ts
@@ -10,8 +10,10 @@ export function usePolled<T>(path: string | null, ms: number) {
if (!path) return;
const controller = new AbortController();
let timer: ReturnType<typeof setTimeout> | undefined;
+ let skipped = false; // the last poll fell while the page was hidden
async function load() {
- if (!document.hidden) {
+ skipped = document.hidden;
+ if (!skipped) {
try {
const value = await apiRequest<T>(path!, 'GET', undefined, controller.signal);
if (!controller.signal.aborted) {
@@ -24,10 +26,18 @@ export function usePolled<T>(path: string | null, ms: number) {
}
if (!controller.signal.aborted) timer = setTimeout(() => void load(), ms);
}
+ // A page opened in a background tab reads when it is shown, not a whole interval later.
+ const shown = () => {
+ if (document.hidden || !skipped) return;
+ clearTimeout(timer);
+ void load();
+ };
+ document.addEventListener('visibilitychange', shown);
void load();
return () => {
controller.abort();
clearTimeout(timer);
+ document.removeEventListener('visibilitychange', shown);
};
}, [path, ms, tick]);
return { data, error, refresh: () => setTick(value => value + 1) };
diff --git a/package/dashboard/ui/features/workshop/service/workflowInferenceService.ts b/package/dashboard/ui/features/workshop/service/workflowInferenceService.ts
index 0992612..0a6131a 100644
--- a/package/dashboard/ui/features/workshop/service/workflowInferenceService.ts
+++ b/package/dashboard/ui/features/workshop/service/workflowInferenceService.ts
@@ -16,7 +16,14 @@ import type { ModelVersion } from '../../../../src/models/modelInferencemodel';
import { modelKey } from '../../../lib/model-library';
import { conditionIssues, newCondition, upstreamModel } from './conditionInferenceService';
import { requiresPredictions } from '../utils/conditionInferenceUtil';
-import { clock, followLoop, isRelativeDay, minutesOf, uploadGraceMinutes } from '../utils/scheduleInferenceUtil';
+import {
+ arrivalMinutes,
+ clock,
+ followLoop,
+ isRelativeDay,
+ minutesOf,
+ uploadGraceMinutes,
+} from '../utils/scheduleInferenceUtil';
export function newStep(kind: StepKind): WorkflowStep {
switch (kind) {
@@ -222,6 +229,18 @@ export function validateWorkflow(
: `Schedule at ${clock(earliest)} UTC or later so the whole window has been recorded and uploaded.`,
);
}
+ if (data.trigger === 'arrival' && footage?.kind === 'footage' && footage.day && footage.day !== 'run-day')
+ add('When new footage arrives takes recordings as they upload. Choose Day of each run in the footage filter.');
+ const latest = 1440 - arrivalMinutes - uploadGraceMinutes;
+ if (
+ data.trigger === 'arrival' &&
+ footage?.kind === 'footage' &&
+ footage.until &&
+ minutesOf(footage.until) > latest
+ )
+ add(
+ `End the window by ${clock(latest)} UTC so its last clips upload before the day's final new-footage check.`,
+ );
if (
data.trigger === 'workflow' &&
!workflows.some(item => item.id === data.workflowId && item.id !== workflow.id)
diff --git a/package/dashboard/ui/features/workshop/utils/scheduleInferenceUtil.ts b/package/dashboard/ui/features/workshop/utils/scheduleInferenceUtil.ts
index cb59285..96b2b34 100644
--- a/package/dashboard/ui/features/workshop/utils/scheduleInferenceUtil.ts
+++ b/package/dashboard/ui/features/workshop/utils/scheduleInferenceUtil.ts
@@ -10,6 +10,8 @@ export const resolveDay = (day: string, now = new Date()) =>
isRelativeDay(day) ? new Date(+now + relativeDays[day].offset * 86400000).toISOString().slice(0, 10) : day;
// Matches the server's grace for the relay to upload a window's last clips.
export const uploadGraceMinutes = 10;
+// Matches the server: new-footage runs check every 15 minutes.
+export const arrivalMinutes = 15;
export const minutesOf = (time: string) => Number(time.slice(0, 2)) * 60 + Number(time.slice(3, 5));
export const clock = (minutes: number) =>
`${String(Math.floor(minutes / 60)).padStart(2, '0')}:${String(minutes % 60).padStart(2, '0')}`;
@@ -58,8 +60,10 @@ export function workflowSchedule(workflow: WorkflowDraft) {
const data = workflow.nodes.find(node => node.data.kind === 'schedule')?.data;
return data?.kind === 'schedule' ? data : undefined;
}
-/** Latest HH:MM:SS a Day-of-each-run window may end for a set-time schedule. '' means none fits; null means no limit. */
+/** Latest HH:MM:SS a Day-of-each-run window may end for a schedule. '' means none fits; null means no limit. */
export function latestUntil(schedule?: ScheduleStep) {
+ // New footage is checked every 15 minutes; the day's last check must see the window's last uploads.
+ if (schedule?.trigger === 'arrival') return `${clock(1440 - arrivalMinutes - uploadGraceMinutes)}:00`;
if (schedule?.trigger !== 'time' || !/^\d\d:\d\d$/.test(schedule.at)) return null;
const latest = minutesOf(schedule.at) - uploadGraceMinutes;
return latest < 0 ? '' : `${clock(latest)}:00`;
diff --git a/package/dashboard/ui/hooks/use-archive.ts b/package/dashboard/ui/hooks/use-archive.ts
index 3359933..8f72db0 100644
--- a/package/dashboard/ui/hooks/use-archive.ts
+++ b/package/dashboard/ui/hooks/use-archive.ts
@@ -26,7 +26,10 @@ function readRoute(): Route {
const q = new URLSearchParams(search);
const zone = q.get('tz') === 'UTC' ? 'UTC' : 'local';
const today = startOfDay(new Date(), zone);
- const date = parseDay(q.get('day') ?? '', zone) ?? today;
+ // A link names a moment; its day in this viewer's time zone can differ from the one in the link.
+ const at = q.get('at');
+ const moment = at && Number.isFinite(Date.parse(at)) ? startOfDay(new Date(at), zone) : null;
+ const date = moment ?? parseDay(q.get('day') ?? '', zone) ?? today;
const bounded = new Date(
Math.min(today.getTime(), Math.max(addDays(today, -RETENTION_DAYS, zone).getTime(), date.getTime())),
);
@@ -35,7 +38,7 @@ function readRoute(): Route {
day: dayKey(bounded, zone),
zone,
cam: q.get('cam'),
- at: q.get('at'),
+ at,
offset: Number.isFinite(offset) ? Math.max(0, Math.min(offset, 86400)) : 0,
view:
q.get('view') === 'workshop'
@@ -86,6 +89,12 @@ function writeRoute(route: Route, cam: string | null, at: string | null, push =
}
}
}
+/** The linked recording and position when the day has it, otherwise the day's newest recording from its start. */
+function linked(rows: Segment[], route: Route) {
+ const chosen = rows.find(s => s.start === route.at) ?? rows.at(-1);
+ const offset = chosen?.start === route.at ? Math.min(route.offset, Math.max(0, (chosen.duration_s ?? 60) - 0.01)) : 0;
+ return { chosen, offset };
+}
export function useArchive() {
const [route, setRoute] = useState(readRoute);
const [overview, setOverview] = useState<Overview | null>(null);
@@ -125,9 +134,7 @@ export function useArchive() {
const rows = await fetchSegments(summary.cam, date, addDays(date, 1, route.zone), controller.signal);
if (token !== generation.current) return;
setSegments(rows);
- const chosen = rows.find(s => s.start === route.at) ?? rows.at(-1);
- const offset =
- chosen?.start === route.at ? Math.min(route.offset, Math.max(0, (chosen.duration_s ?? 60) - 0.01)) : 0;
+ const { chosen, offset } = linked(rows, route);
playbackPosition.current = { start: chosen?.start ?? '', offset };
setSelection(chosen ? { segment: chosen, id: ++sequence.current, offset, play: route.play } : null);
writeRoute(current.current.route, summary.cam, chosen?.start ?? null, false, offset);
@@ -167,25 +174,37 @@ export function useArchive() {
writeRoute(next, next.cam ?? latest.overview?.cam ?? null, next.at, true);
setRoute(next);
}, []);
- const changeDay = (value: string, zone = route.zone, at: string | null = null, play = false) => {
- const max = dayKey(new Date(), zone);
- const min = dayKey(addDays(new Date(), -RETENTION_DAYS, zone), zone);
- if (parseDay(value, zone))
- navigate({
- day: value < min ? min : value > max ? max : value,
- zone,
- at,
- play,
- view: 'recordings',
- revision: Date.now(),
- });
- };
- const changeView = (view: Route['view']) => {
- const next = { ...route, view };
- writeRoute(next, cam, selection?.segment.start ?? null, true, playbackPosition.current.offset);
+ // Stable, so views that rebuild on a new callback (the Activity charts) keep their focus between refreshes.
+ const changeDay = useCallback(
+ (value: string, zone = current.current.route.zone, at: string | null = null, play = false, offset = 0) => {
+ const max = dayKey(new Date(), zone);
+ const min = dayKey(addDays(new Date(), -RETENTION_DAYS, zone), zone);
+ if (parseDay(value, zone))
+ navigate({
+ day: value < min ? min : value > max ? max : value,
+ zone,
+ at,
+ offset,
+ play,
+ view: 'recordings',
+ revision: Date.now(),
+ });
+ },
+ [navigate],
+ );
+ const changeView = useCallback((view: Route['view']) => {
+ const latest = current.current;
+ const next = { ...latest.route, view };
+ writeRoute(
+ next,
+ latest.overview?.cam ?? latest.route.cam,
+ latest.selection?.segment.start ?? null,
+ true,
+ playbackPosition.current.offset,
+ );
setRoute(next);
window.dispatchEvent(new Event('skyvision:navigate'));
- };
+ }, []);
const refresh = useCallback(async () => {
if (busy.current) return;
busy.current = true;
@@ -203,8 +222,9 @@ export function useArchive() {
setSelection(previous => {
const kept = previous && rows.find(s => s.start === previous.segment.start);
if (kept && previous) return { ...previous, segment: kept };
- const chosen = rows.at(-1);
- return chosen ? { segment: chosen, id: ++sequence.current, offset: 0, play: false } : null;
+ // Retry after a failed first load still opens the linked moment.
+ const { chosen, offset } = linked(rows, latest.route);
+ return chosen ? { segment: chosen, id: ++sequence.current, offset, play: false } : null;
});
} catch (e) {
if (token === generation.current) setError(e instanceof Error ? e : new Error('Refresh failed.'));
diff --git a/package/dashboard/ui/public/agent/guides/workflows.md b/package/dashboard/ui/public/agent/guides/workflows.md
index f10ad18..8a8e1de 100644
--- a/package/dashboard/ui/public/agent/guides/workflows.md
+++ b/package/dashboard/ui/public/agent/guides/workflows.md
@@ -50,7 +50,7 @@ Class, confidence, count, and motion rules read the predictions of the model ups
- A camera-archive input needs exactly one footage step, before any model. A labeled-bucket input has none.
- Every path into an output passes a model. A run cannot write into its own input bucket.
- 1–4 models, all `workflow_ready`. A run covers at most 100 recordings and 2,000 sampled frames, or 2,000 labeled images. Use a shorter window or a larger interval to fit.
-- A schedule needs a moving footage day (`run-day` or `previous-day`). A `time` trigger needs days and `at`. With `run-day`, `at` must be at least 10 minutes after the window's `until`, so the footage has uploaded.
+- A schedule needs a moving footage day (`run-day` or `previous-day`). A `time` trigger needs days and `at`. With `run-day`, `at` must be at least 10 minutes after the window's `until`, so the footage has uploaded. An `arrival` trigger needs `run-day` and a window that ends by 23:35 UTC, so the day's last check, at 23:45, sees its last clips.
## Editing
diff --git a/package/dashboard/ui/public/agent/skills/build-a-workflow.md b/package/dashboard/ui/public/agent/skills/build-a-workflow.md
index d2d417e..a44596f 100644
--- a/package/dashboard/ui/public/agent/skills/build-a-workflow.md
+++ b/package/dashboard/ui/public/agent/skills/build-a-workflow.md
@@ -26,4 +26,4 @@ Goal: a workflow that samples a footage window, detects objects, keeps the frame
6. **Run.** `runs start`, then loop `runs watch` until finished. Report progress to the user as it moves.
7. **Verify** (skill `verify-changes`): `runs status` shows `completed` with no `errors`, and `images search` with the `run_id` returns the saved frames.
-To run it automatically, add a schedule step linked into the input (guide `workflows`, Schedules). Use `run-day` or `previous-day` as the footage day.
+To run it automatically, add a schedule step linked into the input (guide `workflows`, Schedules). Use `run-day` or `previous-day` as the footage day; When new footage arrives (`arrival`) needs `run-day` and a window that ends by 23:35 UTC.
diff --git a/package/shared/architecture.check.mjs b/package/shared/architecture.check.mjs
index 4390150..69a1be9 100644
--- a/package/shared/architecture.check.mjs
+++ b/package/shared/architecture.check.mjs
@@ -56,3 +56,10 @@ test('routes register named handlers without execution or business logic', () =>
),
);
});
+test('reads the layer below src/, not from folders the checkout sits in', () => {
+ const index = "import { routes } from './routes/statusStorageRoute'; export default { fetch(r: Request) {} };";
+ assert.deepEqual(inspectSource('/home/me/service/openskycam/package/worker/src/index.ts', index), []);
+ assert.ok(
+ inspectSource('/home/me/service/openskycam/package/worker/src/db/x.ts', '').some(s => s.includes('filename')),
+ );
+});
diff --git a/package/shared/check-architecture.mjs b/package/shared/check-architecture.mjs
index 844f164..89c9b0c 100644
--- a/package/shared/check-architecture.mjs
+++ b/package/shared/check-architecture.mjs
@@ -22,8 +22,10 @@ const suffix = {
utils: 'Util',
models: 'model',
};
+// Only folders below src/ name a layer, so a checkout inside, say, ~/service/ is not read as one.
function layerOf(file) {
- return file.split(/[\\/]/).findLast(part => Object.hasOwn(layers, part));
+ const parts = file.split(/[\\/]/);
+ return parts.slice(parts.lastIndexOf('src') + 1).findLast(part => Object.hasOwn(layers, part));
}
function persistent(type) {
return /\b(?:D1Database(?:Session)?|D1PreparedStatement|R2Bucket)\b/.test(type);
diff --git a/package/worker/src/service/cameraStorageService.ts b/package/worker/src/service/cameraStorageService.ts
index 80a8e0b..69699c5 100644
--- a/package/worker/src/service/cameraStorageService.ts
+++ b/package/worker/src/service/cameraStorageService.ts
@@ -8,8 +8,9 @@ export function cameraIds(env: Env): Set<string> {
);
}
+/** Forbidden, not invalid: a camera missing from CAMERAS is a deployment mismatch, and the relay keeps its footage. */
export function requireKnownCamera(env: Env, cam: string): void {
if (!cameraIds(env).has(cam)) {
- throw new ApplicationError('invalid_input', `unknown camera: ${cam}`);
+ throw new ApplicationError('forbidden', `unknown camera: ${cam}`);
}
}
diff --git a/package/worker/test/segments.test.ts b/package/worker/test/segments.test.ts
index b99c2f3..e1cc8f9 100644
--- a/package/worker/test/segments.test.ts
+++ b/package/worker/test/segments.test.ts
@@ -77,7 +77,7 @@ describe('PUT /v1/segments/:cam/:start', () => {
it.each([
['an unknown token', startMinutesAgo(20), { token: 'wrong' }, 401],
- ['an unknown camera', startMinutesAgo(21), { cam: 'nope' }, 400],
+ ['an unknown camera', startMinutesAgo(21), { cam: 'nope' }, 403],
['an impossible start time', '20261340T256199Z', {}, 400],
['a start older than the retention window', startMinutesAgo(41 * 24 * 60), {}, 400],
['a start in the future', startMinutesAgo(-10), {}, 400],
diff --git a/processor/install-model-sync.sh b/processor/install-model-sync.sh
index d27fd81..f0f33b1 100644
--- a/processor/install-model-sync.sh
+++ b/processor/install-model-sync.sh
@@ -76,9 +76,17 @@ case $processor_plist in
;;
*)
systemctl --user daemon-reload
+ # systemd keeps a unit it cannot parse and the timer then fails every minute; stop here instead of reporting success.
+ for processor_unit in "skyvision-models-$processor_device.service" "skyvision-models-$processor_device.timer"; do
+ processor_state=$(systemctl --user show --property=LoadState --value "$processor_unit")
+ if [ "$processor_state" != loaded ]; then
+ echo "systemd cannot load $processor_unit ($processor_state). See: systemctl --user status $processor_unit" >&2
+ exit 1
+ fi
+ done
systemctl --user enable "skyvision-models-$processor_device.timer"
systemctl --user restart "skyvision-models-$processor_device.timer"
- echo "Run 'loginctl enable-linger $USER' once to keep model sync running after logout."
+ echo "Run 'loginctl enable-linger $(id -un)' once to keep model sync running after logout."
;;
esac
echo "Model sync installed: $processor_device (every 60 seconds). Status is visible under Models → Devices."
diff --git a/processor/install-workshop.sh b/processor/install-workshop.sh
index 4c4d95d..e155506 100755
--- a/processor/install-workshop.sh
+++ b/processor/install-workshop.sh
@@ -13,6 +13,9 @@ workshop_runtime=$3
workshop_origin=$4
workshop_dependencies=${5:-}
workshop_source=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+# The runner imports these before it claims a job; without them the service would restart every 30 seconds.
+PYTHONPATH="$workshop_source${workshop_dependencies:+:$workshop_dependencies}" "$workshop_python" -c 'import av, cv2, torch, ultralytics' \
+ || { echo "$workshop_python cannot import av, cv2, torch and ultralytics: install requirements.txt into it first" >&2; exit 1; }
workshop_plist=$("$workshop_python" - "$workshop_python" "$workshop_key" "$workshop_runtime" "$workshop_origin" "$workshop_source" "$workshop_dependencies" <<'PY'
import os, plistlib, re, sys
from pathlib import Path
@@ -20,8 +23,11 @@ from urllib.parse import urlsplit
python, key, runtime, origin, source, dependencies = sys.argv[1:]
if not re.fullmatch(r'workshop-[a-f0-9]{32}', runtime):
raise SystemExit('A registered immutable Workshop runtime ID is required')
-if urlsplit(origin).scheme != 'https':
+parsed = urlsplit(origin.rstrip('/'))
+if parsed.scheme != 'https':
raise SystemExit('The persistent runner requires HTTPS')
+if parsed.path or parsed.query or parsed.fragment or parsed.username or not parsed.hostname:
+ raise SystemExit('The API origin is https://<api-host>, with no path')
python, key, source = (str(Path(p).absolute()) for p in (python, key, source))
if not Path(key).is_file() or Path(key).stat().st_mode & 0o077:
raise SystemExit('Credential file must exist with mode 600')
@@ -71,9 +77,15 @@ case $workshop_plist in
;;
*)
systemctl --user daemon-reload
+ # systemd keeps a unit it cannot parse and only fails when it starts; stop here instead of reporting success.
+ workshop_state=$(systemctl --user show --property=LoadState --value skyvision-workshop.service)
+ if [ "$workshop_state" != loaded ]; then
+ echo "systemd cannot load $workshop_plist ($workshop_state). See: systemctl --user status skyvision-workshop" >&2
+ exit 1
+ fi
systemctl --user enable skyvision-workshop.service
systemctl --user restart skyvision-workshop.service
- echo "Status: systemctl --user status skyvision-workshop. Run 'loginctl enable-linger $USER' once to keep it running after logout."
+ echo "Status: systemctl --user status skyvision-workshop. Run 'loginctl enable-linger $(id -un)' once to keep it running after logout."
;;
esac
echo "Workshop runner installed. Verify its heartbeat under Workshop before submitting work."
diff --git a/processor/skyvision_processor/db/WorkshopInferenceOps.py b/processor/skyvision_processor/db/WorkshopInferenceOps.py
index b8e678d..11b6f23 100644
--- a/processor/skyvision_processor/db/WorkshopInferenceOps.py
+++ b/processor/skyvision_processor/db/WorkshopInferenceOps.py
@@ -3,7 +3,7 @@ import hashlib
import json
import tempfile
from pathlib import Path
-from ..utils.httpSharedUtil import Client
+from ..utils.httpSharedUtil import Client, retrying
from ..models.workshopInferencemodel import WorkshopJob, WorkshopResult, WorkshopProgress
@@ -15,20 +15,27 @@ def headers(job: WorkshopJob):
return {'X-Lease-Generation': str(job['generation']), 'X-Lease-Token': job['lease_token']}
+# Heartbeats, downloads and uploads are repeatable, so a dropped connection or a 5xx is tried again
+# instead of failing the job for good.
def heartbeat(client, job):
- with client.open('/v1/workshop/jobs/' + job['id'] + '/heartbeat', b'{}', {**headers(job), 'Content-Type': 'application/json'}, 'POST') as response:
- return json.load(response)
+ def renew():
+ with client.open('/v1/workshop/jobs/' + job['id'] + '/heartbeat', b'{}', {**headers(job), 'Content-Type': 'application/json'}, 'POST') as response:
+ return json.load(response)
+ return retrying(renew)
def download(client, job, suffix, target: Path, expected: str, size=None):
- digest = hashlib.sha256(); count = 0
- with client.open('/v1/workshop/jobs/' + job['id'] + suffix, headers=headers(job)) as response, target.open('wb') as output:
- for chunk in iter(lambda: response.read(1024 * 1024), b''):
- count += len(chunk)
- if count > (size if size is not None else 8 * 1024**2):
- raise ValueError('Download exceeds pinned size')
- digest.update(chunk); output.write(chunk)
- if digest.hexdigest() != expected or (size is not None and count != size):
+ def fetch():
+ digest = hashlib.sha256(); count = 0
+ with client.open('/v1/workshop/jobs/' + job['id'] + suffix, headers=headers(job)) as response, target.open('wb') as output:
+ for chunk in iter(lambda: response.read(1024 * 1024), b''):
+ count += len(chunk)
+ if count > (size if size is not None else 8 * 1024**2):
+ raise ValueError('Download exceeds pinned size')
+ digest.update(chunk); output.write(chunk)
+ return digest.hexdigest(), count
+ digest, count = retrying(fetch)
+ if digest != expected or (size is not None and count != size):
raise ValueError('Pinned media checksum mismatch')
@@ -38,8 +45,10 @@ def temporary_attempt():
def upload(client, job, data: bytes):
digest = hashlib.sha256(data).hexdigest()
- with client.open('/v1/workshop/jobs/' + job['id'] + '/images/' + digest, data, {**headers(job), 'Content-Type': 'image/jpeg'}, 'PUT') as response:
- json.load(response)
+ def send():
+ with client.open('/v1/workshop/jobs/' + job['id'] + '/images/' + digest, data, {**headers(job), 'Content-Type': 'image/jpeg'}, 'PUT') as response:
+ json.load(response)
+ retrying(send)
return digest
diff --git a/processor/skyvision_processor/service/deploymentInferenceService.py b/processor/skyvision_processor/service/deploymentInferenceService.py
index 65c2a46..24e3714 100644
--- a/processor/skyvision_processor/service/deploymentInferenceService.py
+++ b/processor/skyvision_processor/service/deploymentInferenceService.py
@@ -70,6 +70,7 @@ def sync_models(client: Client, device: str, root: Path) -> SyncResult:
active = ops.read_active(root)
current = active['deployment'] if active else None
keep = {m['artifact']['sha256'] for m in current['models']} if current else set()
+ verified = set() # checkpoints on disk that match their checksum
desired = None
try:
desired = ops.get_desired(client, device)
@@ -86,6 +87,7 @@ def sync_models(client: Client, device: str, root: Path) -> SyncResult:
valid = path.exists() and path.stat().st_size == artifact['bytes'] and file_hash(path) == artifact['sha256']
if not valid:
ops.download_artifact(client, artifact, path); changed = True
+ verified.add(artifact['sha256'])
if changed:
smoke_model(path, binding)
if (ops.get_desired(client, device) or {}).get('revision') != desired['revision']:
@@ -100,8 +102,9 @@ def sync_models(client: Client, device: str, root: Path) -> SyncResult:
keep = {m['artifact']['sha256'] for m in desired['models']}
ops.prune(root, keep)
except Exception:
- # A failed candidate is disposable; the active version and parameters survive.
- ops.prune(root, keep)
+ # A failed candidate is disposable; the active version and parameters survive. Checkpoints that
+ # matched their checksum stay, so a sync that keeps failing does not download them every minute.
+ ops.prune(root, keep | verified)
try: ops.report(client, device, desired['revision'] if desired else None, current['revision'] if current else None, 'error', 'model-sync-failed')
except Exception: pass
raise
diff --git a/processor/skyvision_processor/service/modelInferenceService.py b/processor/skyvision_processor/service/modelInferenceService.py
index cfc8f42..aaa76f0 100644
--- a/processor/skyvision_processor/service/modelInferenceService.py
+++ b/processor/skyvision_processor/service/modelInferenceService.py
@@ -64,6 +64,8 @@ def training_callback(client: Client, metadata: dict):
def train_and_publish(client: Client, metadata: dict, config: dict):
from ultralytics import YOLO
settings = dict(config)
+ # Ultralytics puts a relative project under its own runs directory; resolve it from here instead.
+ if 'project' in settings: settings['project'] = str(Path(settings['project']).resolve())
source = settings.pop('model')
model = YOLO(source)
model.add_callback('on_train_end', training_callback(client, metadata))
diff --git a/processor/skyvision_processor/service/workshopInferenceService.py b/processor/skyvision_processor/service/workshopInferenceService.py
index 0d7affd..486f16a 100644
--- a/processor/skyvision_processor/service/workshopInferenceService.py
+++ b/processor/skyvision_processor/service/workshopInferenceService.py
@@ -425,6 +425,8 @@ def run_workshop(client, runtime, fingerprint=None):
def process_workshop(client, runtime, watch=False):
"""Keep WORKERS recordings in flight so one's download, decode and server calls overlap another's model pass."""
+ # A runner that claimed jobs it cannot import the libraries for would fail every one of them for good.
+ import av, cv2, torch, ultralytics # noqa: F401
fingerprint = code_hash()
if not watch:
yield run_workshop(client, runtime, fingerprint)
diff --git a/processor/skyvision_processor/utils/httpSharedUtil.py b/processor/skyvision_processor/utils/httpSharedUtil.py
index 79c2969..f1de86c 100644
--- a/processor/skyvision_processor/utils/httpSharedUtil.py
+++ b/processor/skyvision_processor/utils/httpSharedUtil.py
@@ -1,4 +1,5 @@
"""Stateful outbound transport: credentials are restricted to one origin, without redirects."""
+import io
import json
import time
import urllib.error
@@ -27,19 +28,30 @@ class Client:
parsed = urllib.parse.urlsplit(url)
if (parsed.scheme, parsed.netloc) != self.origin or parsed.username:
raise ProcessingError("download_failed")
+ if isinstance(data, bytes):
+ # Sent in blocks, so the timeout bounds each block rather than a whole 8 MiB chunk on a slow uplink.
+ headers = {**(headers or {}), "Content-Length": str(len(data))}
+ data = io.BytesIO(data)
return self.opener.open(urllib.request.Request(url, data=data,
headers={"Authorization": "Bearer " + self.token, **(headers or {})}, method=method), timeout=45)
def post(self, path, body, retry=False):
data = json.dumps(body, separators=(",", ":"), allow_nan=False).encode()
- for attempt in range(3 if retry else 1):
- try:
- with self.open(path, data, {"Content-Type": "application/json"}, "POST") as response:
- return json.load(response)
- except urllib.error.HTTPError as error:
- if error.code < 500 or not retry or attempt == 2:
- raise
- except (OSError, TimeoutError):
- if not retry or attempt == 2:
- raise
- time.sleep(attempt + 1)
+ def send():
+ with self.open(path, data, {"Content-Type": "application/json"}, "POST") as response:
+ return json.load(response)
+ return retrying(send, 3 if retry else 1)
+
+
+def retrying(action, attempts=3):
+ """Run action again after a 5xx answer or a network error; any other failure is final."""
+ for attempt in range(attempts):
+ try:
+ return action()
+ except urllib.error.HTTPError as error:
+ if error.code < 500 or attempt == attempts - 1:
+ raise
+ except (OSError, TimeoutError):
+ if attempt == attempts - 1:
+ raise
+ time.sleep(attempt + 1)
diff --git a/processor/tests/test_installers.py b/processor/tests/test_installers.py
index 2e4787a..b50198b 100644
--- a/processor/tests/test_installers.py
+++ b/processor/tests/test_installers.py
@@ -1,6 +1,7 @@
"""The GPU choice, and the systemd units the install scripts write on Linux."""
import os
import re
+import shutil
import subprocess
import sys
import tempfile
@@ -67,7 +68,7 @@ class LinuxUnits(unittest.TestCase):
def test_workshop_runner_keeps_its_checks(self):
args = ['/py', str(self.key), 'workshop-' + 'a' * 32, 'https://api.example.com', '/src', '']
- for index, bad in ((2, 'workshop-latest'), (3, 'http://api.example.com')):
+ for index, bad in ((2, 'workshop-latest'), (3, 'http://api.example.com'), (3, 'https://api.example.com/v1')):
self.assertNotEqual(0, install('install-workshop.sh', *args[:index], bad, *args[index + 1:], home=self.home).returncode)
self.key.chmod(0o644)
self.assertNotEqual(0, install('install-workshop.sh', *args, home=self.home).returncode)
@@ -87,6 +88,35 @@ class LinuxUnits(unittest.TestCase):
self.assertNotEqual(0, install('install-model-sync.sh', '/py', 'Bad Device', str(self.key), 'https://x', '/m', '/s',
home=self.home).returncode)
+ @unittest.skipUnless(shutil.which('systemd-analyze'), 'needs systemd-analyze')
+ def test_systemd_loads_the_units(self):
+ # The string checks above passed while systemd rejected a quoted WorkingDirectory=, so ask systemd's parser too.
+ source = Path(self.home) / 'src 100%'
+ source.mkdir()
+ workshop = install('install-workshop.sh', sys.executable, str(self.key), 'workshop-' + 'a' * 32,
+ 'https://api.example.com', str(source), '', home=self.home)
+ sync = install('install-model-sync.sh', sys.executable, 'gpu-box', str(self.key), 'https://api.example.com',
+ '/models', str(source), home=self.home)
+ units = [workshop.stdout.strip(), str(Path(sync.stdout.strip()).with_suffix('.service'))]
+ result = subprocess.run(['systemd-analyze', 'verify', '--man=no', *units], capture_output=True, text=True)
+ self.assertEqual(0, result.returncode, result.stderr)
+
+ @unittest.skipUnless(sys.platform.startswith('linux'), 'installs a systemd unit')
+ def test_workshop_installer_stops_when_systemd_cannot_load_the_unit(self):
+ stubs, log = Path(self.home) / 'bin', Path(self.home) / 'systemctl.log'
+ stubs.mkdir()
+ for module in ('av', 'cv2', 'torch', 'ultralytics'): # the runner's libraries, as the dependency directory
+ (stubs / (module + '.py')).write_text('')
+ (stubs / 'systemctl').write_text(f'#!/bin/sh\necho "$*" >> "{log}"\n[ "$2" = show ] && echo "$STATE"\nexit 0\n')
+ (stubs / 'systemctl').chmod(0o755)
+ for state, code in (('bad-setting', 1), ('loaded', 0)):
+ log.write_text('')
+ result = subprocess.run(['bash', str(PROCESSOR / 'install-workshop.sh'), sys.executable, str(self.key),
+ 'workshop-' + 'a' * 32, 'https://api.example.com', str(stubs)], capture_output=True, text=True,
+ env={**os.environ, 'HOME': self.home, 'STATE': state, 'PATH': f'{stubs}:{os.environ["PATH"]}'})
+ self.assertEqual(code, result.returncode, result.stderr)
+ self.assertEqual(state == 'loaded', 'enable skyvision-workshop.service' in log.read_text())
+
def test_scripts_require_the_api_origin(self):
for script, args in (('install-workshop.sh', ['/py', str(self.key), 'workshop-' + 'a' * 32]),
('install-model-sync.sh', ['/py', 'gpu-box', str(self.key)])):
diff --git a/processor/tests/test_model_deployment.py b/processor/tests/test_model_deployment.py
index d261ed9..3cad820 100644
--- a/processor/tests/test_model_deployment.py
+++ b/processor/tests/test_model_deployment.py
@@ -44,11 +44,14 @@ class DeploymentTests(unittest.TestCase):
def test_bad_checksum_or_runtime_keeps_previous_weights_and_parameters(self):
old = deployment(); self.sync(old); before = (self.root / 'active.json').read_bytes()
- for data, fail in [(b'truncated', False), (b'new weights', True)]:
+ # A truncated download is dropped; a checkpoint that matched its checksum stays for the next try.
+ for data, fail, files in [(b'truncated', False, 1), (b'new weights', True, 2)]:
with self.assertRaises(ValueError): self.sync(deployment(b'new weights', 'v2'), data, fail)
self.assertEqual((self.root / 'active.json').read_bytes(), before)
- self.assertEqual(len(list((self.root / 'artifacts').iterdir())), 1)
+ self.assertEqual(len(list((self.root / 'artifacts').iterdir())), files)
self.assertEqual(ops.artifact_path(self.root, old['models'][0]['artifact']).read_bytes(), b'weights')
+ # The next minute's sync does not download it again: these bytes would fail the checksum.
+ with self.assertRaisesRegex(ValueError, 'bad runtime'): self.sync(deployment(b'new weights', 'v2'), b'other', True)
def test_empty_assignment_removes_weights_but_not_cloud_history_or_unrelated_files(self):
old = deployment(); self.sync(old)
diff --git a/processor/tests/test_workshop.py b/processor/tests/test_workshop.py
index be29761..6718824 100644
--- a/processor/tests/test_workshop.py
+++ b/processor/tests/test_workshop.py
@@ -194,3 +194,35 @@ class WorkshopSampling(unittest.TestCase):
with patch.object(w, 'samples', side_effect=broken), self.assertRaises(ValueError):
w.run_workshop(None,'runtime','f')
self.assertEqual(events, ['acquire', 'decode', 'complete', 'release', 'acquire', 'decode', 'release'])
+
+
+class WorkshopTransport(unittest.TestCase):
+ def test_a_dropped_connection_or_5xx_is_tried_again_and_a_refusal_is_not(self):
+ import io
+ import hashlib
+ import tempfile
+ import urllib.error
+ from pathlib import Path
+ from unittest.mock import patch
+ from skyvision_processor.db import WorkshopInferenceOps as ops
+ data = b'recording bytes'
+ job = {'id': 'job_1', 'generation': 1, 'lease_token': 'token'}
+
+ class Client:
+ def __init__(self, *failures):
+ self.failures, self.calls = list(failures), 0
+ def open(self, *_args, **_kwargs):
+ self.calls += 1
+ if self.failures: raise self.failures.pop(0)
+ return io.BytesIO(data)
+
+ target = Path(tempfile.mkdtemp()) / 'input.mp4'
+ busy = urllib.error.HTTPError('u', 502, 'Bad Gateway', {}, None)
+ with patch('skyvision_processor.utils.httpSharedUtil.time.sleep'):
+ client = Client(ConnectionResetError(), busy)
+ ops.download(client, job, '/input', target, hashlib.sha256(data).hexdigest(), len(data))
+ self.assertEqual((3, data), (client.calls, target.read_bytes()))
+ lost = Client(urllib.error.HTTPError('u', 409, 'Conflict', {}, None))
+ with self.assertRaises(urllib.error.HTTPError):
+ ops.download(lost, job, '/input', target, hashlib.sha256(data).hexdigest(), len(data))
+ self.assertEqual(1, lost.calls)
diff --git a/setup/lib.check.mjs b/setup/lib.check.mjs
index d12598c..db262d5 100644
--- a/setup/lib.check.mjs
+++ b/setup/lib.check.mjs
@@ -56,7 +56,8 @@ test('team domains are normalized, and bad input is reported', () => {
for (const v of ['myteam', 'myteam.cloudflareaccess.com', 'https://MyTeam.cloudflareaccess.com/'])
assert.equal(lib.teamDomain(v), 'https://myteam.cloudflareaccess.com');
assert.equal(lib.teamDomain('my team'), undefined);
- assert.equal(lib.problems.host('sky.example.com'), undefined);
+ assert.equal(lib.problems.host('sky.openskycam.org'), undefined);
+ assert.match(lib.problems.host('api.sky.example.com'), /placeholder/); // .env.example's values are not answers
assert.ok(lib.problems.host('https://sky.example.com'));
assert.ok(lib.problems.aud('abc'));
assert.equal(lib.problems.aud('a'.repeat(64)), undefined);
diff --git a/setup/lib.mjs b/setup/lib.mjs
index 07d1431..97fd02e 100644
--- a/setup/lib.mjs
+++ b/setup/lib.mjs
@@ -81,9 +81,11 @@ export function teamDomain(value) {
/** Each returns a problem to show, or undefined when the value is fine. */
export const problems = {
host: v =>
- /^(?=.{1,253}$)([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}$/.test(v)
- ? undefined
- : 'Enter a hostname such as sky.example.com, without https://.',
+ !/^(?=.{1,253}$)([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}$/.test(v)
+ ? 'Enter a hostname such as sky.example.com, without https://.'
+ : /(^|\.)example\.(com|net|org)$/.test(v)
+ ? 'Enter a hostname on your own domain; example.com is the placeholder from .env.example.'
+ : undefined,
ssh: v => (/^[A-Za-z0-9._@-]+$/.test(v) ? undefined : 'Enter an SSH host such as [email protected] or a Host alias.'),
folder: v => (/^\/[^'"\n]*$/.test(v) ? undefined : 'Enter an absolute path, such as /mnt/footage.'),
camera: v =>
diff --git a/setup/setup.mjs b/setup/setup.mjs
index 3dc1a5b..da6e28e 100644
--- a/setup/setup.mjs
+++ b/setup/setup.mjs
@@ -218,8 +218,13 @@ async function linger(host) {
if (ssh(host, 'uname').out.trim() !== 'Linux') return;
if (ssh(host, 'loginctl show-user "$USER" --property=Linger').out.includes('Linger=yes')) return;
say(`${host} must keep its services running when nobody is logged in (loginctl enable-linger).`);
- if (await confirm('Turn that on now? It may ask for your password on that host.'))
- sshTerminal(host, `sudo ${assumeYes ? '-n ' : ''}loginctl enable-linger "$USER"`);
+ if (!(await confirm('Turn that on now? It may ask for your password on that host.'))) {
+ say(yellow(`Without it, ${host} stops openskycam's services when you log out.`));
+ return;
+ }
+ sshTerminal(host, `sudo ${assumeYes ? '-n ' : ''}loginctl enable-linger "$USER"`);
+ if (!ssh(host, 'loginctl show-user "$USER" --property=Linger').out.includes('Linger=yes'))
+ throw new Error(`Lingering is still off on ${host}. Run sudo loginctl enable-linger "$USER" there, then run setup again.`);
}
const storeToken = () => {
@@ -445,7 +450,14 @@ const steps = [
async check() {
if (!env.INGEST_URL || !env.INGEST_TOKEN) return 'The ingest Worker is not deployed yet.';
const status = await http(`${env.INGEST_URL}/v1/status`, { headers: { Authorization: `Bearer ${env.INGEST_TOKEN}` } });
- return [200, 503].includes(status.status) && json(status.text)?.cameras ? true : `${env.INGEST_URL} answers ${status.status || status.text}.`;
+ const listed = json(status.text)?.cameras;
+ if (![200, 503].includes(status.status) || !listed) return `${env.INGEST_URL} answers ${status.status || status.text}.`;
+ // A camera added to CAMERAS after the deploy is refused until the Worker is deployed again.
+ const missing = (env.CAMERAS ?? '')
+ .split(',')
+ .map(c => c.trim())
+ .filter(c => c && !listed.some(l => l.cam === c));
+ return missing.length ? `The ingest Worker does not accept ${missing.join(', ')} yet.` : true;
},
async run() {
if (!/^[a-f0-9]{64}$/.test(env.INGEST_TOKEN ?? '')) save({ INGEST_TOKEN: crypto.randomBytes(32).toString('hex') });
@@ -471,7 +483,7 @@ const steps = [
async check() {
if (!env.INGEST_URL) return 'The ingest Worker is not deployed yet.';
const status = json((await http(`${env.INGEST_URL}/v1/status`, { headers: { Authorization: `Bearer ${env.INGEST_TOKEN}` } })).text);
- const camera = status?.cameras?.find(c => c.cam === env.CAM_ID) ?? status?.cameras?.[0];
+ const camera = status?.cameras?.find(c => c.cam === env.CAM_ID);
if (camera?.age_seconds == null) return 'No recording has arrived yet.';
return camera.age_seconds <= 180 ? true : `The newest recording is ${camera.age_seconds} seconds old.`;
},
@@ -484,6 +496,8 @@ const steps = [
if (missing.length) {
say(yellow(`${host} is missing: ${missing.join(', ')}.`));
if (await confirm('Install ffmpeg and curl with apt now? It asks for your password on that host.')) sshTerminal(host, `sudo ${assumeYes ? '-n ' : ''}sh -c 'apt-get update && apt-get install -y ffmpeg curl'`);
+ const still = ssh(host, 'for t in ffmpeg ffprobe curl; do command -v $t >/dev/null || echo $t; done').out.split('\n').filter(Boolean);
+ if (still.length) throw new Error(`${host} is still missing ${still.join(', ')}. Install them there, then run setup again.`);
}
await linger(host);
for (;;) {