Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

feat(pro): diagnosable crashes and a later OTA confirmation (#1423)

0xdiego2564dba2d67a30
diff --git a/devices/harness-pro/README.md b/devices/harness-pro/README.md
index 5ce1704d9..ffee5eb47 100644
--- a/devices/harness-pro/README.md
+++ b/devices/harness-pro/README.md
@@ -68,6 +68,20 @@ No Wi-Fi: the Pro talks over the cable only, and the module's ESP32-C5 is held i
only for `# > _ ?`, and font 32 only for `> _` (`test_fonts.py` fails on any other character drawn at those sizes). The image is about 1.87 MB;
`docs/size-baseline.txt` has the before and after.
+## Crashes and updates
+
+- A panic always reboots (JTAG-aware halting is off). Its backtrace, the phase each task was in (`ota.write`,
+ `render`, `cable.read`, `voice.*`), the last cable byte and the internal heap are kept in RTC memory
+ (`sys/last_words.c`, `sys/crash_record.c`). After a panic or watchdog reset the next session logs them as
+ `crumbs:` lines with the reset reason and the last log lines. Decode the backtrace with `addr2line` against the ELF of the
+ version the unit ran. A power cut clears RTC memory, so it leaves nothing.
+- The task watchdog (10 s) covers the render task and the cable reader task; the interrupt watchdog is on. A hang
+ becomes a TASK_WDT or INT_WDT panic with a backtrace. A bare `WATCHDOG` reason with no `panic backtrace` is a
+ hardware watchdog: read the `crumbs:` task lines, the stale one is the stuck task.
+- A new OTA image is confirmed late: after 60 s up with a cable session that has stayed up 30 s, or after 180 s up
+ with no cable. A reset before then rolls back to the previous image (the daemon does not offer a written version
+ again).
+
## Layout
- `docs/`: the cable contract and the port inventory.
@@ -77,7 +91,7 @@ only for `# > _ ?`, and font 32 only for `> _` (`test_fonts.py` fails on any oth
- `firmware/main/cable/`: cable link, frame parsing, JSON guard, features, machines, scroll.
- `firmware/main/display/`: panel and bus, GT911 touch, compositor, canvas, terminal, fonts.
- `firmware/main/pod/`: the Pod model, navigation, views, pets and engine logos.
-- `firmware/main/sys/`: config store, device MAC, last words, RAM telemetry.
+- `firmware/main/sys/`: config store, device MAC, last words and crash record, RAM telemetry.
- `firmware/fonts/`: bundled Liberation Sans for tests and CI.
- `firmware/scripts/`: font generator, flash, pet import, logo generator.
- `firmware/test/`: host tests, host stubs and the tree-isolation check.
diff --git a/devices/harness-pro/firmware/main/CMakeLists.txt b/devices/harness-pro/firmware/main/CMakeLists.txt
index 431f52c47..473c7f273 100644
--- a/devices/harness-pro/firmware/main/CMakeLists.txt
+++ b/devices/harness-pro/firmware/main/CMakeLists.txt
@@ -54,6 +54,7 @@ idf_component_register(
"pod/pod_shared.c"
"pod/pod_view.c"
"sys/config_store.c"
+ "sys/crash_record.c"
"sys/device_mac.c"
"sys/fw_update.c"
"sys/last_words.c"
diff --git a/devices/harness-pro/firmware/main/app/pro_app.c b/devices/harness-pro/firmware/main/app/pro_app.c
index 0bf1a65fb..565b72146 100644
--- a/devices/harness-pro/firmware/main/app/pro_app.c
+++ b/devices/harness-pro/firmware/main/app/pro_app.c
@@ -24,6 +24,7 @@
#include "pro_frame.h"
#include "sys/config_store.h"
#include "sys/fw_update.h"
+#include "sys/last_words.h"
#include "sys/ram_telemetry.h"
#include "task_stacks.h"
#include "theme.h"
@@ -869,6 +870,7 @@ static void render_task(void *arg)
uint32_t frames = 0, raster_max_us = 0, frame_max_us = 0;
for (;;) {
ESP_ERROR_CHECK(esp_task_wdt_reset());
+ last_words_phase(LW_RENDER, "render");
pro_app_tick();
const bool fresh = pro_app_take_scene(&s_scene[front ^ 1]);
uint32_t wake_ms = pro_app_next_wake_ms();
diff --git a/devices/harness-pro/firmware/main/app_main.c b/devices/harness-pro/firmware/main/app_main.c
index afe3cb409..5f83a4f38 100644
--- a/devices/harness-pro/firmware/main/app_main.c
+++ b/devices/harness-pro/firmware/main/app_main.c
@@ -64,10 +64,10 @@ void app_main(void)
pro_cable_start();
ram_telemetry_checkpoint("app_ready");
- // A freshly installed image is confirmed by fw_confirm_* (sys/fw_update.h): never before the first frame, then at the first
- // cable session (pro_cable) or at 60 s up, whichever comes first. One that never paints is reverted at once.
+ // A freshly installed image is confirmed by fw_confirm_* (sys/fw_update.h): never before the first frame, then after
+ // 60 s up with a 30 s cable session (or 180 s up without). One that never paints is reverted at once.
if (pro_app_wait_first_frame(10000)) {
- if (fw_confirm_frame_painted()) pro_app_note_update();
+ fw_confirm_frame_painted();
while (fw_confirm_pending()) {
vTaskDelay(pdMS_TO_TICKS(1000));
if (fw_confirm_tick((uint32_t)(esp_timer_get_time() / 1000))) pro_app_note_update();
diff --git a/devices/harness-pro/firmware/main/audio/pro_voice.c b/devices/harness-pro/firmware/main/audio/pro_voice.c
index b9a04b35e..a355b325d 100644
--- a/devices/harness-pro/firmware/main/audio/pro_voice.c
+++ b/devices/harness-pro/firmware/main/audio/pro_voice.c
@@ -23,6 +23,7 @@
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "sys/config_store.h"
+#include "sys/last_words.h"
#define CAP_MS 600000u // the 600 s duration cap (UH:5265)
#define REPLY_WAIT_MS 65000u // how long a sent clip waits for the daemon's reply
@@ -66,6 +67,7 @@ void pro_voice_begin(const char *agent_id)
{
bool expected = false;
if (!atomic_compare_exchange_strong(&s_starting, &expected, true)) return; // another begin is mid-flight
+ last_words_phase(LW_VOICE, "voice.begin");
TEST_STEP(1);
expected = false;
// Open at once so the Talk frame is never judged "no recording behind it"; STARTING keeps the tick away.
@@ -111,6 +113,7 @@ static void discard(void)
void pro_voice_end(bool send)
{
if (!atomic_load(&s_open)) return;
+ last_words_phase(LW_VOICE, send ? "voice.send" : "voice.end");
if (send) {
if (atomic_load(&s_waiting)) return; // the clip already went; its reply is awaited
if (!pro_voice_start_pending() && pro_rec_recording()) {
diff --git a/devices/harness-pro/firmware/main/cable/pro_cable.c b/devices/harness-pro/firmware/main/cable/pro_cable.c
index 863dfd563..4d2f32636 100644
--- a/devices/harness-pro/firmware/main/cable/pro_cable.c
+++ b/devices/harness-pro/firmware/main/cable/pro_cable.c
@@ -28,6 +28,7 @@
#include "sys/device_mac.h"
#include "sys/fw_update.h"
#include "sys/last_words.h"
+#include "esp_task_wdt.h"
#include "task_stacks.h"
static const char *TAG = "pro_cable";
@@ -793,8 +794,7 @@ static void session_up(const cJSON *p)
// console and `idf.py monitor` behaves as it always has.
cable_link_set_log_framing(true);
ESP_LOGI(TAG, "session up: %s", s_machine_name);
- // The welcome parsed: this image's cable path works, so a fresh install may be confirmed now (fw_update.h).
- if (fw_confirm_session_up()) pro_app_note_update();
+ fw_confirm_session_up((uint32_t)(esp_timer_get_time() / 1000)); // starts the clock a fresh image is confirmed on (fw_update.h)
}
if (was && !moved) return;
// Anything held belonged to another host. Drop a half-streamed roster before its end can make it current.
@@ -828,6 +828,7 @@ static void session_down(const char *why)
cable_link_reset_decoder(); // runs on the tick: whatever is half-received belonged to the session that ended
cable_link_set_log_framing(false); // first: what follows should be readable on a plain monitor
ESP_LOGI(TAG, "session down (%s)", why);
+ fw_confirm_session_down();
fw_update_abort("session down");
pro_app_machines(s_machines.rows, 0, "");
pro_app_link(false, NULL);
@@ -962,6 +963,13 @@ static void link_resync(void)
void pro_cable_tick(void *ctx)
{
(void)ctx;
+ // Under the task watchdog like the render task, so a hang here is a TASK_WDT panic with a backtrace, not a silent
+ // hardware reset. Joined on the first tick (this runs on the link's reader task, which this file does not create).
+ static bool watched;
+ if (!watched) { watched = esp_task_wdt_add(NULL) == ESP_OK; }
+ else esp_task_wdt_reset();
+ last_words_phase(LW_CABLE, fw_update_active() ? "ota" : "cable.read");
+ last_words_rx((uint32_t)(s_last_rx_us / 1000));
fw_update_tick((uint32_t)(esp_timer_get_time() / 1000));
link_resync();
if (atomic_load(&s_session) && esp_timer_get_time() - s_last_rx_us > (int64_t)SILENCE_MS * 1000)
diff --git a/devices/harness-pro/firmware/main/sys/crash_record.c b/devices/harness-pro/firmware/main/sys/crash_record.c
new file mode 100644
index 000000000..f90021319
--- /dev/null
+++ b/devices/harness-pro/firmware/main/sys/crash_record.c
@@ -0,0 +1,49 @@
+#include "crash_record.h"
+
+#include <stdio.h>
+#include <string.h>
+
+static const char *const SLOT_NAME[CR_SLOTS] = { "cable", "render", "voice" };
+
+void cr_init(crash_record_t *r)
+{
+ memset(r, 0, sizeof *r);
+ r->magic = CR_MAGIC;
+}
+
+bool cr_valid(const crash_record_t *r) { return r->magic == CR_MAGIC && r->npc <= CR_PCS; }
+
+void cr_phase(crash_record_t *r, unsigned slot, const char *phase, uint32_t now_ms)
+{
+ if (slot >= CR_SLOTS) return;
+ if (strncmp(r->slot[slot].name, phase, CR_NAME - 1) != 0) { // same phase again: only the clock moves
+ strncpy(r->slot[slot].name, phase, CR_NAME - 1);
+ r->slot[slot].name[CR_NAME - 1] = '\0';
+ }
+ r->slot[slot].ms = now_ms;
+ r->up_ms = now_ms;
+}
+
+size_t cr_format(const crash_record_t *r, char *out, size_t cap)
+{
+ if (!cap) return 0;
+ size_t n = 0;
+ out[0] = '\0';
+#define ADD(...) do { if (n < cap) { int w = snprintf(out + n, cap - n, __VA_ARGS__); if (w > 0) n = n + (size_t)w < cap ? n + (size_t)w : cap - 1; } } while (0)
+ ADD("crumbs: alive until %u ms, last cable byte %u ms, internal heap %u free / %u lowest\n",
+ (unsigned)r->up_ms, (unsigned)r->rx_ms, (unsigned)r->free_int, (unsigned)r->min_int);
+ for (unsigned i = 0; i < CR_SLOTS; i++) {
+ if (!r->slot[i].name[0]) continue;
+ char name[CR_NAME]; // a torn write must not run the %s past the record
+ memcpy(name, r->slot[i].name, sizeof name);
+ name[CR_NAME - 1] = '\0';
+ ADD("crumbs: %s task: %s at %u ms\n", SLOT_NAME[i], name, (unsigned)r->slot[i].ms);
+ }
+ if (r->panicked) {
+ ADD("crumbs: panic backtrace (addr2line against this build's ELF):");
+ for (uint32_t i = 0; i < r->npc && i < CR_PCS; i++) ADD(" 0x%08x", (unsigned)r->pc[i]);
+ ADD("\n");
+ }
+#undef ADD
+ return n;
+}
diff --git a/devices/harness-pro/firmware/main/sys/crash_record.h b/devices/harness-pro/firmware/main/sys/crash_record.h
new file mode 100644
index 000000000..9d448a097
--- /dev/null
+++ b/devices/harness-pro/firmware/main/sys/crash_record.h
@@ -0,0 +1,35 @@
+// The breadcrumb record kept in RTC memory (last_words.c owns the memory): what each task was doing, when,
+// and the panic's backtrace. Pure data and text, so it is host-tested; nothing here touches the IDF.
+#pragma once
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#define CR_MAGIC 0x43524244u // "CRBD"
+#define CR_PCS 12 // backtrace entries kept
+#define CR_SLOTS 3 // one phase per task that can hang on its own
+#define CR_NAME 12
+
+typedef struct {
+ char name[CR_NAME]; // the phase, e.g. "ota.write"; empty = never set
+ uint32_t ms; // uptime when it was set
+} cr_slot_t;
+
+typedef struct {
+ uint32_t magic;
+ uint32_t up_ms; // uptime at the latest breadcrumb: the device was alive at least this long
+ uint32_t rx_ms; // uptime of the last byte from the cable
+ uint32_t free_int; // internal heap free / lowest-ever, sampled with the breadcrumbs
+ uint32_t min_int;
+ uint32_t panicked; // the panic handler ran (and filled pc[])
+ uint32_t npc;
+ uint32_t pc[CR_PCS];
+ cr_slot_t slot[CR_SLOTS];
+} crash_record_t;
+
+void cr_init(crash_record_t *r);
+bool cr_valid(const crash_record_t *r);
+void cr_phase(crash_record_t *r, unsigned slot, const char *phase, uint32_t now_ms);
+// Text for the daemon's log, one '\n'-separated line per fact. Returns the length (always NUL-terminated).
+size_t cr_format(const crash_record_t *r, char *out, size_t cap);
diff --git a/devices/harness-pro/firmware/main/sys/fw_update.c b/devices/harness-pro/firmware/main/sys/fw_update.c
index a6173c776..c734af533 100644
--- a/devices/harness-pro/firmware/main/sys/fw_update.c
+++ b/devices/harness-pro/firmware/main/sys/fw_update.c
@@ -15,6 +15,7 @@
#include "freertos/task.h"
#include "hal/efuse_hal.h"
#include "mbedtls/sha256.h"
+#include "last_words.h"
static const char *TAG = "fw";
@@ -100,32 +101,30 @@ void fw_mark_valid(void)
}
// ---- when a fresh image is confirmed ---------------------------------------------------------------------------
-// Never before the first frame is painted, then at the first cable session (the welcome: parsed, the roster asked
-// for) or, failing that, once the uptime reaches CONFIRM_MS. An image whose cable path crashes would otherwise be
-// confirmed and then reboot-loop with no rollback, and the cable is the only way to fix it.
+// Late, so the rollback protects against what actually went wrong in the field (resets 9 to 37 s after boot): the
+// image must have been up for CONFIRM_MS with a cable session that has stayed up for SESSION_MS, or, with no cable
+// at all, for IDLE_MS. Never before the first frame is painted. A reset before that rolls back to the previous image.
#define CONFIRM_MS 60000u
-static atomic_bool s_painted, s_session_seen, s_confirmed;
+#define SESSION_MS 30000u
+#define IDLE_MS 180000u
+static atomic_bool s_painted, s_confirmed;
+static atomic_uint s_session_since; // uptime ms of the current session's welcome (0: no session)
-static bool confirm_now(void)
+bool fw_confirm_due(uint32_t uptime_ms, uint32_t session_since_ms)
{
- if (atomic_exchange(&s_confirmed, true)) return false;
- fw_mark_valid();
- return true;
-}
-// Each side stores its own flag, then reads the other's: whichever comes second confirms (once).
-bool fw_confirm_frame_painted(void)
-{
- atomic_store(&s_painted, true);
- return atomic_load(&s_session_seen) && confirm_now();
-}
-bool fw_confirm_session_up(void)
-{
- atomic_store(&s_session_seen, true);
- return atomic_load(&s_painted) && confirm_now();
+ if (uptime_ms >= IDLE_MS) return true;
+ return session_since_ms && uptime_ms >= CONFIRM_MS && (uint32_t)(uptime_ms - session_since_ms) >= SESSION_MS;
}
+
+void fw_confirm_frame_painted(void) { atomic_store(&s_painted, true); }
+void fw_confirm_session_up(uint32_t uptime_ms) { atomic_store(&s_session_since, uptime_ms ? uptime_ms : 1); }
+void fw_confirm_session_down(void) { atomic_store(&s_session_since, 0); } // "continuously": a drop restarts the 30 s
bool fw_confirm_tick(uint32_t uptime_ms)
{
- return atomic_load(&s_painted) && uptime_ms >= CONFIRM_MS && confirm_now();
+ if (!atomic_load(&s_painted) || !fw_confirm_due(uptime_ms, atomic_load(&s_session_since))) return false;
+ if (atomic_exchange(&s_confirmed, true)) return false;
+ fw_mark_valid();
+ return true;
}
bool fw_confirm_pending(void) { return !atomic_load(&s_confirmed); }
@@ -142,7 +141,7 @@ void fw_no_frame(void)
#ifdef FW_UPDATE_HOST_TEST
void fw_confirm_test_reset(void)
{
- atomic_store(&s_painted, false); atomic_store(&s_session_seen, false); atomic_store(&s_confirmed, false);
+ atomic_store(&s_painted, false); atomic_store(&s_session_since, 0); atomic_store(&s_confirmed, false);
}
#endif
@@ -220,13 +219,14 @@ bool fw_update_offer(const char *version, int size, const char *sha256_hex, bool
s_sha_hex[64] = 0;
snprintf(s_version, sizeof s_version, "%s", version);
- // esp_ota_begin ERASES the slot, which takes seconds; the screen goes up first so the device does not look
- // frozen at exactly the moment a user must not pull the cable.
+ // The slot is erased as the writes reach it (OTA_WITH_SEQUENTIAL_WRITES), a sector or two per slice: erasing the
+ // whole image up front blocked this task for seconds, long enough to starve the task watchdog it is now under.
const esp_app_desc_t *me = esp_app_get_description();
+ last_words_phase(LW_CABLE, "ota.accept");
ESP_LOGI(TAG, "accepting %s (%d B) into %s", version, size, target->label);
fw_update_ui_begin(me ? me->version : "", version);
s_handle = 0;
- if (esp_ota_begin(target, (size_t)size, &s_handle) != ESP_OK) {
+ if (esp_ota_begin(target, OTA_WITH_SEQUENTIAL_WRITES, &s_handle) != ESP_OK) {
if (s_handle) esp_ota_abort(s_handle);
reset_state();
fw_update_send_error("could not start the update");
@@ -299,6 +299,7 @@ void fw_update_slice(const uint8_t *payload, size_t plen)
}
if (len > s_expected - s_written) { fail("more bytes than the offer promised"); return; }
+ last_words_phase(LW_CABLE, "ota.write");
if (esp_ota_write(s_handle, data, len) != ESP_OK) { fail("flash write failed"); return; }
s_written += (uint32_t)len;
s_last_slice_ms = now_ms();
@@ -310,7 +311,9 @@ void fw_update_slice(const uint8_t *payload, size_t plen)
if (s_written < s_expected) return;
// ---- the image is complete: five checks, in order, ALL before esp_ota_set_boot_partition ------------
+ last_words_phase(LW_CABLE, "ota.end");
if (esp_ota_end(s_handle) != ESP_OK) { fail("image rejected by esp_ota_end"); return; } // 1
+ last_words_phase(LW_CABLE, "ota.verify");
if (!written_hash_matches()) { fail("sha256 mismatch"); return; } // 2
const char *why = image_refusal(); // 3, 4, 5
if (why) { fail(why); return; }
diff --git a/devices/harness-pro/firmware/main/sys/fw_update.h b/devices/harness-pro/firmware/main/sys/fw_update.h
index 5531d1349..2e9842aa3 100644
--- a/devices/harness-pro/firmware/main/sys/fw_update.h
+++ b/devices/harness-pro/firmware/main/sys/fw_update.h
@@ -16,15 +16,18 @@
// Confirm THIS image if this boot is a freshly installed one. With CONFIG_BOOTLOADER_APP_ROLLBACK_ENABLE an
// app booted from an OTA slot starts PENDING_VERIFY and the bootloader REVERTS it at the next reset unless told
// otherwise (an unconfirmed image keeps running until then). Do not call it directly: use the fw_confirm_*
-// triggers below, which hold it back until the image has proved its cable path.
+// triggers below, which hold it back until the image has proved itself.
void fw_mark_valid(void);
-// When to confirm: never before the first frame is painted, then at the FIRST cable session (pro_cable's
-// session_up on `welcome`) or after 60 s of uptime, whichever comes first. A crash on the cable path then
-// still rolls back, and an idle cable cannot cost a good image (it is merely confirmed at 60 s). Each returns
-// true only on the call that did the confirming, so the caller can announce the update once.
-bool fw_confirm_frame_painted(void);
-bool fw_confirm_session_up(void);
+// When to confirm: never before the first frame is painted; then once the image has run 60 s AND its cable session
+// has been up for 30 s without a break, or, if no such session happens, once it has run 180 s. A reset before that
+// rolls back to the previous image, so a crash in the first minute of a new image (render, cable, voice) is undone.
+// fw_confirm_due is that rule as a pure function (session_since_ms 0: no session); fw_confirm_tick applies it and
+// returns true only on the call that did the confirming, so the caller can announce the update once.
+bool fw_confirm_due(uint32_t uptime_ms, uint32_t session_since_ms);
+void fw_confirm_frame_painted(void);
+void fw_confirm_session_up(uint32_t uptime_ms); // pro_cable's welcome: the session starts now
+void fw_confirm_session_down(void); // the session ended: the 30 s count starts over
bool fw_confirm_tick(uint32_t uptime_ms);
bool fw_confirm_pending(void); // not confirmed (or decided) yet: keep ticking
diff --git a/devices/harness-pro/firmware/main/sys/last_words.c b/devices/harness-pro/firmware/main/sys/last_words.c
index c45d6fe08..33fc76ee1 100644
--- a/devices/harness-pro/firmware/main/sys/last_words.c
+++ b/devices/harness-pro/firmware/main/sys/last_words.c
@@ -2,8 +2,12 @@
#include <string.h>
+#include "crash_record.h"
#include "esp_attr.h"
+#include "esp_heap_caps.h"
#include "esp_log.h"
+#include "esp_private/panic_internal.h"
+#include "esp_timer.h"
#include "esp_system.h"
#include "freertos/FreeRTOS.h"
@@ -24,6 +28,12 @@ typedef struct {
} ring_t;
static RTC_NOINIT_ATTR ring_t s_ring;
+// The breadcrumbs: a second small RTC record for the resets the log ring cannot explain (hardware watchdog,
+// brownout) and the panic's backtrace. No checksum: tasks and the panic handler write it without a lock, and a
+// torn field is worth more than a dropped record; only a valid magic and an abnormal reset make it speak.
+static RTC_NOINIT_ATTR crash_record_t s_rec;
+static crash_record_t s_prev_rec;
+static bool s_have_rec;
static portMUX_TYPE s_mux = portMUX_INITIALIZER_UNLOCKED;
static esp_reset_reason_t s_reason;
@@ -73,12 +83,45 @@ void last_words_boot(void)
s_previous_len = s_ring.len;
s_previous[s_previous_len] = '\0';
}
+ if (s_abnormal && cr_valid(&s_rec)) { s_prev_rec = s_rec; s_have_rec = true; }
+ cr_init(&s_rec);
s_ring.magic = RING_MAGIC;
s_ring.head = 0;
s_ring.len = 0;
ring_seal();
}
+static uint32_t now_ms(void) { return (uint32_t)(esp_timer_get_time() / 1000); }
+
+void last_words_phase(lw_task_t task, const char *phase)
+{
+ const uint32_t now = now_ms();
+ cr_phase(&s_rec, (unsigned)task, phase, now);
+ static uint32_t s_heap_ms; // heap numbers are O(1) but not free: once a second is plenty
+ if (now - s_heap_ms >= 1000) {
+ s_heap_ms = now;
+ s_rec.free_int = (uint32_t)heap_caps_get_free_size(MALLOC_CAP_INTERNAL);
+ s_rec.min_int = (uint32_t)heap_caps_get_minimum_free_size(MALLOC_CAP_INTERNAL);
+ }
+}
+
+void last_words_rx(uint32_t ms) { s_rec.rx_ms = ms; }
+
+// The panic's backtrace. IDF's panic handler prints it through this weak hook, one (pc, sp) per frame, from
+// the frame-pointer unwinder (CONFIG_ESP_SYSTEM_USE_FRAME_POINTER), and it runs after the panic text is on its
+// way and before the reboot — on an exception, a failed assert, abort(), the task watchdog and the interrupt
+// watchdog alike. Kept: still prints as IDF does, plus a copy into RTC. IRAM and no calls out of it: it can
+// run with the flash cache off.
+void IRAM_ATTR esp_fp_generated_step(uint32_t pc, uint32_t sp)
+{
+ panic_print_str(" 0x");
+ panic_print_hex((int)pc);
+ panic_print_str(":0x");
+ panic_print_hex((int)sp);
+ if (s_rec.magic == CR_MAGIC && s_rec.npc < CR_PCS) s_rec.pc[s_rec.npc++] = pc;
+ s_rec.panicked = 1;
+}
+
void last_words_add(const char *line, size_t len)
{
if (!len) return;
@@ -102,6 +145,15 @@ void last_words_report(void)
s_reported = true;
if (s_abnormal) ESP_LOGW(TAG, "reset reason: %s", reason_name(s_reason));
else ESP_LOGI(TAG, "reset reason: %s", reason_name(s_reason));
+ if (s_have_rec) {
+ static char text[512];
+ cr_format(&s_prev_rec, text, sizeof text);
+ for (char *p = text, *nl; p && *p; p = nl ? nl + 1 : NULL) {
+ if ((nl = strchr(p, '\n'))) *nl = '\0';
+ ESP_LOGW(TAG, "%s", p);
+ }
+ s_have_rec = false;
+ }
if (!s_previous_len) return;
ESP_LOGW(TAG, "last %u bytes of log before that reset follow", (unsigned)s_previous_len);
// The first line is very likely a fragment (the ring wrapped through it); everything after is whole.
diff --git a/devices/harness-pro/firmware/main/sys/last_words.h b/devices/harness-pro/firmware/main/sys/last_words.h
index 50b9cefdb..457f40e5e 100644
--- a/devices/harness-pro/firmware/main/sys/last_words.h
+++ b/devices/harness-pro/firmware/main/sys/last_words.h
@@ -8,6 +8,7 @@
#include <stdbool.h>
#include <stddef.h>
+#include <stdint.h>
// At boot, before anything logs: read the reset reason, keep the previous boot's ring if the reset was
// abnormal, then arm the ring for this boot.
@@ -19,3 +20,9 @@ void last_words_add(const char *line, size_t len);
// Once a daemon is listening (session up): log the reset reason, and the previous boot's last lines if
// that reset was a crash. Logs once per boot.
void last_words_report(void);
+
+// Breadcrumbs: what each task is doing, for a reset that leaves no log line (a hardware watchdog, a brownout).
+// Cheap enough for the render loop: a few stores and a clock read. The text is replayed with the reset reason.
+typedef enum { LW_CABLE, LW_RENDER, LW_VOICE } lw_task_t;
+void last_words_phase(lw_task_t task, const char *phase);
+void last_words_rx(uint32_t ms); // uptime of the latest byte from the cable
diff --git a/devices/harness-pro/firmware/sdkconfig.defaults b/devices/harness-pro/firmware/sdkconfig.defaults
index 2577e0db0..b24cb2596 100644
--- a/devices/harness-pro/firmware/sdkconfig.defaults
+++ b/devices/harness-pro/firmware/sdkconfig.defaults
@@ -82,3 +82,23 @@ CONFIG_ESP_CONSOLE_SECONDARY_NONE=y
# over the cable (last_words.c). 10 s, so a full-face redraw is never mistaken for a hang.
CONFIG_ESP_TASK_WDT_PANIC=y
CONFIG_ESP_TASK_WDT_TIMEOUT_S=10
+
+# --- Crashes must be diagnosable and must reboot ---
+# OCDAWARE makes the panic handler HALT instead of rebooting whenever the JTAG block looks attached, which the
+# USB-Serial-JTAG port can look like to it; a halted unit then falls to the hardware watchdog with no trace. Off:
+# a panic always prints, records itself in RTC memory (last_words.c) and reboots.
+CONFIG_ESP_DEBUG_OCDAWARE=n
+CONFIG_ESP_SYSTEM_PANIC_PRINT_REBOOT=y
+CONFIG_ESP_SYSTEM_PANIC_REBOOT_DELAY_SECONDS=0
+# The unwinder last_words.c's backtrace hook rides on (esp_fp_generated_step): a few percent of code size and speed.
+CONFIG_ESP_SYSTEM_USE_FRAME_POINTER=y
+# Both watchdogs are the IDF defaults, stated so a changed default cannot switch them off. The task watchdog (above)
+# watches the idle tasks, the render task and the cable reader task; the interrupt watchdog catches a task that
+# keeps interrupts off or an ISR that never returns, as a panic with a backtrace instead of a hardware reset.
+CONFIG_ESP_INT_WDT=y
+CONFIG_ESP_INT_WDT_TIMEOUT_MS=300
+CONFIG_ESP_INT_WDT_CHECK_CPU1=y
+CONFIG_ESP_TASK_WDT_EN=y
+CONFIG_ESP_TASK_WDT_INIT=y
+CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU0=y
+CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU1=y
diff --git a/devices/harness-pro/firmware/test/host_esp/esp_ota_ops.h b/devices/harness-pro/firmware/test/host_esp/esp_ota_ops.h
index 36cdbd783..eb2827dbd 100644
--- a/devices/harness-pro/firmware/test/host_esp/esp_ota_ops.h
+++ b/devices/harness-pro/firmware/test/host_esp/esp_ota_ops.h
@@ -14,6 +14,7 @@ const esp_partition_t *esp_ota_get_next_update_partition(const esp_partition_t *
esp_err_t esp_ota_get_state_partition(const esp_partition_t *p, esp_ota_img_states_t *s);
esp_err_t esp_ota_mark_app_valid_cancel_rollback(void);
esp_err_t esp_ota_mark_app_invalid_rollback_and_reboot(void);
+#define OTA_WITH_SEQUENTIAL_WRITES 0xfffffffe
esp_err_t esp_ota_begin(const esp_partition_t *p, size_t image_size, esp_ota_handle_t *h);
esp_err_t esp_ota_write(esp_ota_handle_t h, const void *data, size_t n);
esp_err_t esp_ota_end(esp_ota_handle_t h);
diff --git a/devices/harness-pro/firmware/test/run.sh b/devices/harness-pro/firmware/test/run.sh
index 45c817048..840c24924 100755
--- a/devices/harness-pro/firmware/test/run.sh
+++ b/devices/harness-pro/firmware/test/run.sh
@@ -123,6 +123,11 @@ cc "${cc_flags[@]}" -DFW_UPDATE_HOST_TEST -Wframe-larger-than=1024 -c -o "$out/f
cc "${cc_flags[@]}" -I"$fw/main/sys" -o "$out/test_fw_update" "$here/test_fw_update.c" "$out/fw_update.o"
"$out/test_fw_update"
+# crash_record: the RTC breadcrumb record's phases and text (the pure half of last_words.c).
+cc "${cc_flags[@]}" -Wframe-larger-than=1024 -c -o "$out/crash_record.o" "$fw/main/sys/crash_record.c"
+cc "${cc_flags[@]}" -I"$fw/main/sys" -o "$out/test_crash_record" "$here/test_crash_record.c" "$out/crash_record.o"
+"$out/test_crash_record"
+
# The app layer: pro_app.c, pod_glue.c and pro_touch.c with the cable, the recorder and the clock as host stubs,
# replayed. It builds its own fonts and objects.
python3 "$here/test_pro_app.py"
diff --git a/devices/harness-pro/firmware/test/stress_pro_app.c b/devices/harness-pro/firmware/test/stress_pro_app.c
index 070d5eceb..aec12764a 100644
--- a/devices/harness-pro/firmware/test/stress_pro_app.c
+++ b/devices/harness-pro/firmware/test/stress_pro_app.c
@@ -23,6 +23,7 @@
#include "esp_heap_caps.h"
#include "esp_task_wdt.h"
+#include "sys/last_words.h"
#include "esp_timer.h"
#include "freertos/FreeRTOS.h"
#include "freertos/semphr.h"
@@ -44,6 +45,8 @@ void vTaskDelay(TickType_t t) { (void)t; }
uint32_t uxTaskGetStackHighWaterMark(TaskHandle_t t) { (void)t; return 0; }
esp_err_t esp_task_wdt_add(void *t) { (void)t; return ESP_OK; }
esp_err_t esp_task_wdt_reset(void) { return ESP_OK; }
+void last_words_phase(lw_task_t t, const char *p) { (void)t; (void)p; }
+void last_words_rx(uint32_t ms) { (void)ms; }
size_t heap_caps_get_free_size(unsigned c) { (void)c; return 0; }
size_t heap_caps_get_largest_free_block(unsigned c) { (void)c; return 0; }
uint8_t config_load_brightness(void) { return 220; }
diff --git a/devices/harness-pro/firmware/test/test_crash_record.c b/devices/harness-pro/firmware/test/test_crash_record.c
new file mode 100644
index 000000000..1351f8f32
--- /dev/null
+++ b/devices/harness-pro/firmware/test/test_crash_record.c
@@ -0,0 +1,50 @@
+// crash_record.c: the breadcrumb record's phases and its text for the daemon. Host-only, pure.
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "crash_record.h"
+
+int main(void)
+{
+ crash_record_t r;
+ memset(&r, 0xA5, sizeof r);
+ assert(!cr_valid(&r)); // cold RTC garbage is not a record
+ cr_init(&r);
+ assert(cr_valid(&r));
+
+ char out[512];
+ cr_phase(&r, 0, "ota.write", 1000);
+ cr_phase(&r, 1, "render", 1200);
+ cr_phase(&r, 0, "ota.verify", 1500); // a newer phase replaces the older one in its slot
+ cr_phase(&r, 7, "bogus", 9999); // an unknown slot is ignored
+ assert(r.up_ms == 1500 && !strcmp(r.slot[0].name, "ota.verify") && r.slot[0].ms == 1500 && r.slot[1].ms == 1200);
+ r.rx_ms = 1400; r.free_int = 123456; r.min_int = 100000;
+ cr_format(&r, out, sizeof out);
+ assert(strstr(out, "alive until 1500 ms, last cable byte 1400 ms, internal heap 123456 free / 100000 lowest\n"));
+ assert(strstr(out, "cable task: ota.verify at 1500 ms\n") && strstr(out, "render task: render at 1200 ms\n"));
+ assert(!strstr(out, "voice") && !strstr(out, "panic"));
+
+ // A long phase is cut, never overrun; a record from a torn write cannot run %s off the end.
+ cr_phase(&r, 2, "a-very-long-phase-name", 1600);
+ assert(strlen(r.slot[2].name) == CR_NAME - 1);
+ memset(r.slot[2].name, 'x', CR_NAME);
+ cr_format(&r, out, sizeof out);
+ assert(strstr(out, "voice task: xxxxxxxxxxx at 1600 ms\n"));
+
+ // The panic backtrace, most recent frame first; more frames than fit are dropped, not wrapped.
+ r.panicked = 1; r.npc = 2; r.pc[0] = 0x4ff00123u; r.pc[1] = 0x40001000u;
+ cr_format(&r, out, sizeof out);
+ assert(strstr(out, "panic backtrace (addr2line against this build's ELF): 0x4ff00123 0x40001000\n"));
+ r.npc = CR_PCS + 1;
+ assert(!cr_valid(&r));
+
+ // Too small an output buffer: truncated, terminated, no overrun.
+ char tiny[16];
+ memset(tiny, 'Z', sizeof tiny);
+ size_t n = cr_format(&r, tiny, sizeof tiny);
+ assert(n < sizeof tiny && tiny[n] == '\0');
+ assert(cr_format(&r, tiny, 0) == 0);
+ puts("test_crash_record: ok");
+ return 0;
+}
diff --git a/devices/harness-pro/firmware/test/test_fw_update.c b/devices/harness-pro/firmware/test/test_fw_update.c
index 24435f6cf..d629ee579 100644
--- a/devices/harness-pro/firmware/test/test_fw_update.c
+++ b/devices/harness-pro/firmware/test/test_fw_update.c
@@ -15,6 +15,7 @@
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "fw_update.h"
+#include "last_words.h"
#include "hal/efuse_hal.h"
#include "mbedtls/sha256.h"
@@ -69,6 +70,7 @@ void fw_update_ui_pct(int p) { if (h_pcts < 64) pcts[h_pcts++] = p; }
void fw_update_ui_restarting(const char *t) { h_restarting++; snprintf(restarting_to, 32, "%s", t); }
void fw_update_ui_failed(void) { h_failed++; }
void fw_update_restart(void) { h_restart++; }
+void last_words_phase(lw_task_t t, const char *p) { (void)t; (void)p; }
// ---- fixtures --------------------------------------------------------------------------------------------
static uint8_t img[IMG_SIZE];
@@ -293,29 +295,41 @@ static void mark_valid_only_when_pending(void)
assert(c_cancel == 0 && !fw_was_updated(v, sizeof v));
}
-// When a fresh image is confirmed: never before the first frame; then at the first cable session or at 60 s up.
+// When a fresh image is confirmed: after the first frame, 60 s up with a 30 s unbroken session, or 180 s up alone.
+static void confirm_policy(void)
+{
+ assert(!fw_confirm_due(0, 0) && !fw_confirm_due(179999, 0) && fw_confirm_due(180000, 0)); // no cable: 180 s
+ assert(!fw_confirm_due(2000, 1500) && !fw_confirm_due(59999, 1000)); // the old welcome-time rule is gone
+ assert(!fw_confirm_due(60000, 40000)); // session only 20 s old
+ assert(!fw_confirm_due(60000, 30001) && fw_confirm_due(60000, 30000));
+ assert(!fw_confirm_due(59999, 1000)); // 58 s of session, but the image is under 60 s
+ assert(fw_confirm_due(90000, 60000) && !fw_confirm_due(89999, 60000)); // late welcome: 30 s after it
+ assert(fw_confirm_due(0xFFFFFF00u, 1)); // far past every limit
+}
+
static void confirm_timing(void)
{
char v[32];
- // welcome before the first frame: nothing yet; the frame then confirms (once)
+ // a clean run with a cable: nothing before the frame, nothing at 60 s, confirmed 30 s after the welcome (once)
reset(); fw_confirm_test_reset(); img_state = ESP_OTA_IMG_PENDING_VERIFY;
- assert(fw_confirm_pending());
- assert(!fw_confirm_session_up() && c_cancel == 0);
- assert(!fw_confirm_tick(61000) && c_cancel == 0); // 60 s up but no frame: still not
- assert(fw_confirm_frame_painted() && c_cancel == 1 && !fw_confirm_pending());
- assert(!fw_confirm_session_up() && !fw_confirm_tick(90000) && c_cancel == 1);
- assert(fw_was_updated(v, sizeof v));
- // frame first, no welcome: not before 60 s, once at 60 s
+ fw_confirm_session_up(2000);
+ assert(!fw_confirm_tick(90000) && c_cancel == 0 && fw_confirm_pending()); // no frame yet
+ fw_confirm_frame_painted();
+ assert(!fw_confirm_tick(31999) && !fw_confirm_tick(59999) && c_cancel == 0);
+ assert(fw_confirm_tick(60000) && c_cancel == 1 && !fw_confirm_pending());
+ assert(!fw_confirm_tick(61000) && c_cancel == 1 && fw_was_updated(v, sizeof v));
+ // the session drops and comes back: the 30 s start over
reset(); fw_confirm_test_reset(); img_state = ESP_OTA_IMG_PENDING_VERIFY;
- assert(!fw_confirm_frame_painted() && c_cancel == 0);
- assert(!fw_confirm_tick(1000) && !fw_confirm_tick(59999) && c_cancel == 0 && fw_confirm_pending());
- assert(fw_confirm_tick(60000) && c_cancel == 1);
- assert(!fw_confirm_tick(61000) && !fw_confirm_session_up() && c_cancel == 1);
- // frame, then the welcome: confirmed on the welcome, long before 60 s, once
+ fw_confirm_frame_painted(); fw_confirm_session_up(2000);
+ fw_confirm_session_down();
+ assert(!fw_confirm_tick(100000) && c_cancel == 0);
+ fw_confirm_session_up(100000);
+ assert(!fw_confirm_tick(129999) && fw_confirm_tick(130000) && c_cancel == 1);
+ // no cable ever: 180 s
reset(); fw_confirm_test_reset(); img_state = ESP_OTA_IMG_PENDING_VERIFY;
- assert(!fw_confirm_frame_painted() && !fw_confirm_tick(5000) && c_cancel == 0);
- assert(fw_confirm_session_up() && c_cancel == 1);
- assert(!fw_confirm_session_up() && !fw_confirm_tick(60000) && c_cancel == 1);
+ fw_confirm_frame_painted();
+ assert(!fw_confirm_tick(60000) && !fw_confirm_tick(179999) && c_cancel == 0);
+ assert(fw_confirm_tick(180000) && c_cancel == 1 && !fw_confirm_tick(181000));
}
// M6: no first frame on a trial image reverts it; on a confirmed image nothing happens.
@@ -330,7 +344,7 @@ int main(void)
make_image();
offer_validation(); busy_declined(); second_offer_declined(); slice_without_offer_is_ignored();
slice_past_size_aborts(); stall_aborts_after_10s(); offsets_and_resend(); silence_asks_for_resend(); short_payload_ignored(); end_checks(); happy_path(); mark_valid_only_when_pending();
- confirm_timing(); no_frame_rolls_back();
+ confirm_policy(); confirm_timing(); no_frame_rolls_back();
puts("test_fw_update: ok");
return 0;
}
diff --git a/devices/harness-pro/firmware/test/test_pro_app.py b/devices/harness-pro/firmware/test/test_pro_app.py
index 6bcdda223..4a270c138 100644
--- a/devices/harness-pro/firmware/test/test_pro_app.py
+++ b/devices/harness-pro/firmware/test/test_pro_app.py
@@ -43,6 +43,7 @@ code = r'''
#include "sys/config_store.h"
#include "sys/fw_update.h"
#include "sys/ram_telemetry.h"
+#include "sys/last_words.h"
#include "pro_display.h"
#include "pro_frame.h"
@@ -84,6 +85,8 @@ void vTaskDelay(TickType_t t) { (void)t; }
uint32_t uxTaskGetStackHighWaterMark(TaskHandle_t t) { (void)t; return 0; }
esp_err_t esp_task_wdt_add(void *t) { (void)t; return ESP_OK; }
esp_err_t esp_task_wdt_reset(void) { return ESP_OK; }
+void last_words_phase(lw_task_t t, const char *p) { (void)t; (void)p; }
+void last_words_rx(uint32_t ms) { (void)ms; }
size_t heap_caps_get_free_size(unsigned c) { (void)c; return 0; }
size_t heap_caps_get_largest_free_block(unsigned c) { (void)c; return 0; }
uint8_t config_load_brightness(void) { return 220; }
diff --git a/devices/harness-pro/firmware/test/test_pro_cable.c b/devices/harness-pro/firmware/test/test_pro_cable.c
index fdc5bf3ad..57675e90f 100644
--- a/devices/harness-pro/firmware/test/test_pro_cable.c
+++ b/devices/harness-pro/firmware/test/test_pro_cable.c
@@ -25,6 +25,7 @@
#include "sys/device_mac.h"
#include "task_stacks.h"
#include "sys/last_words.h"
+#include "esp_task_wdt.h"
#include "board/board.h"
#include "sys/config_store.h"
#include "sys/fw_update.h"
@@ -141,6 +142,10 @@ const esp_app_desc_t *esp_app_get_description(void) { static const esp_app_desc_
const board_t *board(void) { static const board_t b = { .name = "stub" }; return &b; }
bool device_mac_str(char *o, size_t n) { snprintf(o, n, "aa"); return true; }
void last_words_report(void) {}
+void last_words_phase(lw_task_t t, const char *p) { (void)t; (void)p; }
+void last_words_rx(uint32_t ms) { (void)ms; }
+esp_err_t esp_task_wdt_add(void *t) { (void)t; return ESP_OK; }
+esp_err_t esp_task_wdt_reset(void) { return ESP_OK; }
struct host_mutex { pthread_mutex_t m; };
SemaphoreHandle_t xSemaphoreCreateMutex(void) { struct host_mutex *m = malloc(sizeof *m); pthread_mutex_init(&m->m, NULL); return m; }
BaseType_t xSemaphoreTake(SemaphoreHandle_t m, TickType_t w) { (void)w; pthread_mutex_lock(&m->m); return pdTRUE; }
@@ -170,12 +175,14 @@ bool pro_voice_open(void) { return voice_is_open; }
static struct { int offers, slices, aborts, ticks; char version[32], sha[80], why[32]; int size; bool busy; size_t slice_len; uint8_t first; uint32_t now; } fw;
// What the gate says, how long the stubbed fw_update_* "block" (the slot erase, the read-back hash), what they answer.
static bool gate_ok = true, offer_accepts, offer_active;
-static int gate_begins, gate_ends, confirms, notes;
+static int gate_begins, gate_ends, confirms, confirm_downs, notes;
+static uint32_t confirm_at;
static int64_t block_us;
bool pro_app_begin_update_gate(void) { gate_begins++; return gate_ok; }
void pro_app_end_update_gate(void) { gate_ends++; }
void pro_app_note_update(void) { notes++; }
-bool fw_confirm_session_up(void) { confirms++; return confirms == 1; }
+void fw_confirm_session_up(uint32_t ms) { confirms++; confirm_at = ms; }
+void fw_confirm_session_down(void) { confirm_downs++; }
bool fw_update_active(void) { return offer_active; }
bool fw_update_offer(const char *v, int size, const char *sha, bool busy)
{
@@ -913,17 +920,18 @@ static void fw_offer_gate(void)
end_session();
}
-// H3: the first welcome asks fw to confirm the image (once per session-up), and announces the update only if it did.
+// H3: the welcome starts the clock a fresh image is confirmed on (once per session-up), and the end of the session stops it.
static void fw_confirms_on_welcome(void)
{
- confirms = notes = 0;
+ confirms = confirm_downs = notes = 0;
welcome();
- CHECK(confirms == 1 && notes == 1);
+ CHECK(confirms == 1 && confirm_at == (uint32_t)(now_us / 1000) && notes == 0);
welcome(); // a repeat welcome on a live session is not a new session
- CHECK(confirms == 1 && notes == 1);
+ CHECK(confirms == 1 && confirm_downs == 0);
end_session();
- welcome(); // a new session asks again, but nothing confirmed this time
- CHECK(confirms == 2 && notes == 1);
+ CHECK(confirm_downs == 1);
+ welcome(); // a new session starts the clock again
+ CHECK(confirms == 2);
end_session();
}
diff --git a/devices/harness-pro/firmware/test/test_pro_voice_gate.py b/devices/harness-pro/firmware/test/test_pro_voice_gate.py
index e09265cbb..c67f88c23 100644
--- a/devices/harness-pro/firmware/test/test_pro_voice_gate.py
+++ b/devices/harness-pro/firmware/test/test_pro_voice_gate.py
@@ -41,6 +41,7 @@ code = r'''
#include "pro_canvas.h"
#include "sys/config_store.h"
#include "sys/ram_telemetry.h"
+#include "sys/last_words.h"
#include "app/pro_touch.h"
#include "pro_display.h"
#include "pro_frame.h"
@@ -85,6 +86,8 @@ void vTaskDelay(TickType_t t) { (void)t; }
uint32_t uxTaskGetStackHighWaterMark(TaskHandle_t t) { (void)t; return 0; }
esp_err_t esp_task_wdt_add(void *t) { (void)t; return ESP_OK; }
esp_err_t esp_task_wdt_reset(void) { return ESP_OK; }
+void last_words_phase(lw_task_t t, const char *p) { (void)t; (void)p; }
+void last_words_rx(uint32_t ms) { (void)ms; }
size_t heap_caps_get_free_size(unsigned c) { (void)c; return 0; }
size_t heap_caps_get_largest_free_block(unsigned c) { (void)c; return 0; }
uint8_t config_load_brightness(void) { return 220; }