Skip to content

Free games for graphing calculators: NumWorks, TI-Nspire CX, Casio fx-CG50 and soon TI-84 Plus CE. Balatro, Celeste, Hollow Knight, Minecraft and more.

Undertale: the pictures packed with LZMA, unpacked as they are needed

Mason Chen2022ff11d41d
diff --git a/games/undertale/src/blk.c b/games/undertale/src/blk.c
new file mode 100644
index 00000000..23b74619
--- /dev/null
+++ b/games/undertale/src/blk.c
@@ -0,0 +1,107 @@
+/* data.bin's LZMA blocks and the RAM they're unpacked into (blk.h). */
+#include <string.h>
+#include "blk.h"
+#include "lzma/LzmaDec.h"
+#include "res.h"
+
+#define ARENA (60 * 1024) /* the cache: a screen of tiles is 20 KB, rarely 55; a battle's monsters 40 */
+#define MAX_IN 128 /* blocks in it at once */
+#define MAX_BLOCKS 4096 /* (tools/pack.py's) */
+#define NONE 0xFF
+
+static uint8_t arena[ARENA] __attribute__((aligned(4)));
+typedef struct {
+ uint16_t id, used; /* the block, the frame it was last used in */
+ uint32_t off, size; /* where it is in the arena */
+} In;
+static In in[MAX_IN]; /* by where they are */
+static int nin;
+static uint8_t where[MAX_BLOCKS]; /* each block's place in `in`, or NONE */
+static uint16_t frame_no = 1;
+static bool ready;
+uint32_t blk_unpacked, blk_most_in_frame;
+static uint32_t unpacked_now;
+
+/* the decoder's tables, in one piece (LzmaDecode asks once a block) */
+static uint16_t probs[2800];
+static void *probs_alloc(ISzAllocPtr p, size_t size) {
+ (void)p;
+ return size <= sizeof probs ? probs : NULL;
+}
+static void probs_free(ISzAllocPtr p, void *a) { (void)p, (void)a; }
+static const ISzAlloc alloc = {probs_alloc, probs_free};
+
+static void init(void) {
+ memset(where, NONE, sizeof where);
+ nin = 0, ready = true;
+}
+
+static void drop(int k) {
+ where[in[k].id] = NONE;
+ memmove(&in[k], &in[k + 1], (size_t)(nin - k - 1) * sizeof in[0]);
+ nin--;
+ for (int i = k; i < nin; i++) where[in[i].id] = (uint8_t)i;
+}
+
+/* room for n bytes: its place in `in` (the new entry, made), or -1 */
+static int make_room(uint32_t n) {
+ for (;;) {
+ if (nin < MAX_IN) {
+ uint32_t at = 0;
+ for (int i = 0; i <= nin; i++) {
+ uint32_t end = i < nin ? in[i].off : ARENA;
+ if (end - at >= n) {
+ memmove(&in[i + 1], &in[i], (size_t)(nin - i) * sizeof in[0]);
+ nin++;
+ for (int j = i + 1; j < nin; j++) where[in[j].id] = (uint8_t)j;
+ in[i].off = at, in[i].size = n;
+ return i;
+ }
+ if (i < nin) at = in[i].off + in[i].size;
+ }
+ }
+ if (!nin) return -1;
+ /* throw out the least recently used (those of this frame last) */
+ int old = 0;
+ for (int i = 1; i < nin; i++)
+ if ((uint16_t)(frame_no - in[i].used) > (uint16_t)(frame_no - in[old].used)) old = i;
+ drop(old);
+ }
+}
+
+const uint8_t *blk_get(int id) {
+ if (!ready) init();
+ const uint8_t *s = section(SEC_BLOCKS);
+ if (id < 0 || id >= (int)rd32(s)) return NULL;
+ if (where[id] != NONE) {
+ In *b = &in[where[id]];
+ b->used = frame_no;
+ return arena + b->off;
+ }
+ const uint8_t *e = s + 12 + 8 * id;
+ uint32_t off = rd32(e), size = rd32(e + 4);
+ uint32_t next = id + 1 < (int)rd32(s) ? rd32(e + 8) : 0;
+ int k = make_room(size);
+ if (k < 0) return NULL;
+ In *b = &in[k];
+ b->id = (uint16_t)id, b->used = frame_no;
+ where[id] = (uint8_t)k;
+ SizeT out = size, src = next ? next - off : 1u << 30;
+ ELzmaStatus st;
+ LzmaDecode(arena + b->off, &out, s + off, &src, s + 4, LZMA_PROPS_SIZE, LZMA_FINISH_ANY, &st, &alloc);
+ blk_unpacked++, unpacked_now++;
+ return arena + b->off;
+}
+
+void blk_frame(void) {
+ if (!ready) init();
+ /* the blocks packed together, so the free room is in one piece */
+ uint32_t at = 0;
+ for (int i = 0; i < nin; i++) {
+ if (in[i].off != at) memmove(arena + at, arena + in[i].off, in[i].size), in[i].off = at;
+ at += in[i].size;
+ }
+ if (unpacked_now > blk_most_in_frame) blk_most_in_frame = unpacked_now;
+ unpacked_now = 0;
+ frame_no++;
+}
diff --git a/games/undertale/src/blk.h b/games/undertale/src/blk.h
new file mode 100644
index 00000000..65b495d5
--- /dev/null
+++ b/games/undertale/src/blk.h
@@ -0,0 +1,16 @@
+/* Undertale: data.bin's BLOCKS, unpacked when needed (tools/pack.py packs them with LZMA).
+ *
+ * The frames and tiles live in blocks; blk_get unpacks a block into a RAM cache
+ * and gives its bytes. A pointer it gives holds until the next blk_get (which can
+ * throw a block out to make room), so callers use it right away, and nothing keeps
+ * one from a frame to the next: blk_frame, at the frame's start, packs the cache's
+ * blocks together. The blocks the frame used are thrown out last. */
+#ifndef UT_BLK_H
+#define UT_BLK_H
+#include <stdint.h>
+
+const uint8_t *blk_get(int id);
+void blk_frame(void);
+/* (for the tests: blocks unpacked so far, the most in a frame) */
+extern uint32_t blk_unpacked, blk_most_in_frame;
+#endif
diff --git a/games/undertale/src/data.bin b/games/undertale/src/data.bin
index a1f28878..2833d11a 100644
Binary files a/games/undertale/src/data.bin and b/games/undertale/src/data.bin differ
diff --git a/games/undertale/src/data.h b/games/undertale/src/data.h
index 8f703c29..3080f83f 100644
--- a/games/undertale/src/data.h
+++ b/games/undertale/src/data.h
@@ -10,6 +10,7 @@
#define SEC_FONTS 6
#define SEC_STRINGS 7
#define SEC_DOORS 8
+#define SEC_BLOCKS 9
#define SPR_FRISK_D 0
#define SPR_FRISK_U 1
#define SPR_FRISK_L 2
diff --git a/games/undertale/src/gfx.c b/games/undertale/src/gfx.c
index 10200547..08fd184d 100644
--- a/games/undertale/src/gfx.c
+++ b/games/undertale/src/gfx.c
@@ -5,6 +5,7 @@
* (skip, length, pixels), so a band only reads the rows it shows. Alpha has
* 33 steps (0..32), applied with the usual RGB565 trick. */
#include <string.h>
+#include "blk.h"
#include "gfx.h"
#include "res.h"
#include "world.h"
@@ -46,6 +47,7 @@ int gfx_dropped;
static int dropped_now;
void gfx_begin(void) {
+ blk_frame();
nitems = 0, npool = 0, dropped_now = 0;
gfx_clip_all();
}
diff --git a/games/undertale/src/res.c b/games/undertale/src/res.c
index e81f146b..d9a4d015 100644
--- a/games/undertale/src/res.c
+++ b/games/undertale/src/res.c
@@ -1,5 +1,6 @@
/* data.bin: sections, sprites, tiles, rooms and strings (see res.h). */
#include <stddef.h>
+#include "blk.h"
#include "res.h"
const uint8_t *ut_bin;
@@ -10,6 +11,13 @@ int spr_frames(uint16_t spr) {
return rd16(s + 4 + 4 * spr + 2);
}
+/* a frame's or tile's bytes: in its block, unpacked (blk.h) */
+static const uint8_t *in_block(int sec, int k) {
+ const uint8_t *t = section(sec) + 4 + 4 * k;
+ const uint8_t *b = blk_get(rd16(t));
+ return b ? b + rd16(t + 2) : NULL;
+}
+
bool frame_get(uint16_t spr, int i, Frame *f) {
const uint8_t *s = section(SEC_SPRITES);
if (spr >= rd32(s)) return false;
@@ -17,8 +25,8 @@ bool frame_get(uint16_t spr, int i, Frame *f) {
if (n <= 0) return false;
i %= n;
if (i < 0) i += n;
- const uint8_t *fs = section(SEC_FRAMES);
- const uint8_t *p = fs + rd32(fs + 4 + 4 * (first + i));
+ const uint8_t *p = in_block(SEC_FRAMES, first + i);
+ if (!p) return false;
f->w = rd16(p), f->h = rd16(p + 2);
f->ox = rds16(p + 4), f->oy = rds16(p + 6);
f->bpp = p[8], f->ncol = p[9];
@@ -39,8 +47,9 @@ int spr_h(uint16_t spr) {
}
void tile_get(uint16_t id, Tile *t) {
- const uint8_t *s = section(SEC_TILES);
- const uint8_t *p = s + rd32(s + 4 + 4 * id);
+ static const uint8_t black[4] = {0, 1, 0, 0};
+ const uint8_t *p = in_block(SEC_TILES, id);
+ if (!p) p = black;
t->bpp = p[0], t->ncol = p[1];
t->pal = p + 2;
t->bits = p + 2 + 2 * t->ncol;
diff --git a/games/undertale/src/res.h b/games/undertale/src/res.h
index 558cae8d..20217dce 100644
--- a/games/undertale/src/res.h
+++ b/games/undertale/src/res.h
@@ -1,7 +1,9 @@
/* Undertale: data.bin (tools/pack.py), read where it lies in flash.
*
* Its sections (SEC_* in data.h): sprites and their frames, the areas' tiles,
- * rooms (a tile grid and a collision map each), fonts and strings. */
+ * rooms (a tile grid and a collision map each), fonts and strings. Frames and
+ * tiles are packed in LZMA blocks (blk.h): their pointers hold until the next
+ * frame_get or tile_get. */
#ifndef UT_RES_H
#define UT_RES_H
#include <stdbool.h>
diff --git a/games/undertale/tests/play.c b/games/undertale/tests/play.c
index 6277afa7..80026538 100644
--- a/games/undertale/tests/play.c
+++ b/games/undertale/tests/play.c
@@ -15,6 +15,7 @@
#include <string.h>
#include <sys/stat.h>
#include "../src/battle.h"
+#include "../src/blk.h"
#include "../src/game.h"
#include "../src/gfx.h"
#include "../src/menu.h"
@@ -433,6 +434,7 @@ int main(int argc, char **argv) {
}
}
if (gfx_dropped) printf("dropped %d draw commands\n", gfx_dropped);
+ if (getenv("BLK_STATS")) fprintf(stderr, "blocks unpacked %u, at most %u in a frame\n", blk_unpacked, blk_most_in_frame);
printf("END mode %d room %d at %d,%d hp %d lv %d gold %d kills %d path %s\nFLAGS", g_mode, g_room, pl.x, pl.y, sv.hp,
sv.lv, sv.gold, sv.kills, path_at && *path_at ? "left" : "done");
for (int i = 0; i < FLAG_COUNT; i++)
diff --git a/games/undertale/tools/pack.py b/games/undertale/tools/pack.py
index 5a96062d..631376fb 100644
--- a/games/undertale/tools/pack.py
+++ b/games/undertale/tools/pack.py
@@ -16,7 +16,13 @@ data.bin: "UTD1", the number of sections, their offsets, then the sections (SEC_
bitmap offset u16) and the bitmaps (rows of whole bytes, highest bit first)
- STRINGS: the room names and other texts the data refers to
- DOORS: for each room its first door and how many (u16, u16), then the doors: 16 bytes each (x, y, w, h on
- the map, the room it leads to, where Frisk's top-left lands, the way he faces)"""
+ the map, the room it leads to, where Frisk's top-left lands, the way he faces)
+- BLOCKS: the frames and tiles themselves, in LZMA streams the game unpacks when it needs them (blk.c): the
+ count, the 5 LZMA properties bytes (3 of padding), then for each block its stream's offset (u32) and its size
+ unpacked (u32), then the streams. FRAMES and TILES only say where each frame or tile is: its block and its
+ offset in it (u16, u16). A sprite's frames share blocks (of 16 KB at most); a room's tiles go in 8 x 8 squares,
+ so the tiles a screen shows sit in few blocks (of 2 KB)."""
+import lzma
import os
import struct
import sys
@@ -32,7 +38,55 @@ import sprites as S # noqa: E402
OUT_BIN = os.path.join(HERE, "..", "src", "data.bin")
OUT_H = os.path.join(HERE, "..", "src", "data.h")
-SECTIONS = ["SPRITES", "FRAMES", "TILES", "ROOMS", "GRID", "COLL", "FONTS", "STRINGS", "DOORS"]
+SECTIONS = ["SPRITES", "FRAMES", "TILES", "ROOMS", "GRID", "COLL", "FONTS", "STRINGS", "DOORS", "BLOCKS"]
+BLOCK_TILES, BLOCK_FRAMES = 2048, 16384 # the most bytes a block holds, unpacked (a frame bigger: alone)
+MAX_BLOCKS = 4096 # (blk.c's table)
+LZMA_FILTERS = [{"id": lzma.FILTER_LZMA1, "dict_size": 1 << 16, "lc": 0, "lp": 0, "pb": 0}]
+
+
+class Blocks:
+ """the BLOCKS section, and the tables that point into it"""
+ def __init__(self):
+ self.raw = []
+
+ def place(self, entries, groups):
+ """entries (bytes) put in blocks, one per group of their indices -> u32 count, (block, offset) each"""
+ where = [None] * len(entries)
+ for g in groups:
+ data = bytearray()
+ for i in g:
+ where[i] = (len(self.raw), len(data))
+ data += entries[i]
+ assert len(data) < 65536
+ self.raw.append(bytes(data))
+ return struct.pack("<I", len(entries)) + b"".join(struct.pack("<HH", b, o) for b, o in where)
+
+ def section(self):
+ assert len(self.raw) <= MAX_BLOCKS
+ streams = [lzma.compress(r, format=lzma.FORMAT_RAW, filters=LZMA_FILTERS) for r in self.raw]
+ props = bytes([0]) + struct.pack("<I", 1 << 16) + bytes(3) # lc = lp = pb = 0, a 64 KB dictionary
+ head = struct.pack("<I", len(streams)) + props
+ base = len(head) + 8 * len(streams)
+ table, body = bytearray(), bytearray()
+ for r, c in zip(self.raw, streams):
+ table += struct.pack("<II", base + len(body), len(r))
+ body += c
+ return head + bytes(table) + bytes(body)
+
+
+def groups_by_size(sizes, limit, starts=()):
+ """consecutive indices grouped up to limit bytes, a new group at each of starts too"""
+ out, cur, n = [], [], 0
+ starts = set(starts)
+ for i, sz in enumerate(sizes):
+ if cur and (n + sz > limit or i in starts):
+ out.append(cur)
+ cur, n = [], 0
+ cur.append(i)
+ n += sz
+ if cur:
+ out.append(cur)
+ return out
def rgb565(r, g, b):
@@ -517,11 +571,16 @@ def pack_room(area, r, tiles, strings):
cols, rows = (gx1 - gx) // 20, (gy1 - gy) // 20
big = np.zeros((rows * 20, cols * 20), np.int32)
big[y - gy:y - gy + h, x - gx:x - gx + w] = px
- grid = bytearray()
- for ty in range(rows):
- for tx in range(cols):
- block = big[ty * 20:ty * 20 + 20, tx * 20:tx * 20 + 20]
- grid += struct.pack("<H", 0xFFFF if not block.any() else tiles.add(block.astype(np.uint16)))
+ # (new tiles numbered square by square: a screen's tiles are close together, in few blocks)
+ ids = {}
+ for cy in range(0, rows, 8):
+ for cx in range(0, cols, 8):
+ for ty in range(cy, min(rows, cy + 8)):
+ for tx in range(cx, min(cols, cx + 8)):
+ block = big[ty * 20:ty * 20 + 20, tx * 20:tx * 20 + 20]
+ if block.any():
+ ids[ty, tx] = tiles.add(block.astype(np.uint16))
+ grid = b"".join(struct.pack("<H", ids.get((ty, tx), 0xFFFF)) for ty in range(rows) for tx in range(cols))
coll, cw, ch = collision(area, r)
flags = r.get("flags", 0)
if r.get("ice"):
@@ -625,7 +684,8 @@ def main():
frames.append(encode_frame(im, ox, oy))
defines.append(("SPR_" + name.upper(), k))
sec_sprites = struct.pack("<I", len(spr_entries)) + b"".join(struct.pack("<HH", a, b) for a, b in spr_entries)
- sec_frames = table(frames)
+ blocks = Blocks()
+ sec_frames = blocks.place(frames, groups_by_size([len(f) for f in frames], BLOCK_FRAMES, [a for a, _ in spr_entries]))
# areas and rooms
tiles = TileSet()
@@ -662,7 +722,7 @@ def main():
doors += struct.pack("<hhhhHhhH", dx, dy, dw, dh, names[to], tx, ty, dirs[d])
ndoors += 1
sec_doors = bytes(door_index) + bytes(doors)
- sec_tiles = table(tiles.blobs)
+ sec_tiles = blocks.place(tiles.blobs, groups_by_size([len(t) for t in tiles.blobs], BLOCK_TILES))
# fonts
fonts = []
@@ -672,7 +732,7 @@ def main():
sec_fonts = table(fonts)
sections = [sec_sprites, sec_frames, sec_tiles, sec_rooms, bytes(grid_blob), bytes(coll_blob), sec_fonts,
- strings.blob(), sec_doors]
+ strings.blob(), sec_doors, blocks.section()]
head = b"UTD1" + struct.pack("<I", len(sections))
offs, body = [], bytearray()
base = len(head) + 4 * len(sections)
@@ -694,8 +754,9 @@ def main():
f.write("#define NUM_SPRITES %d\n#define NUM_ROOMS %d\n#define NUM_FONTS %d\n" % (len(spr_entries), len(packed), len(fonts)))
f.write("#endif\n")
sizes = dict(zip(SECTIONS, map(len, sections)))
- print("data.bin: %d bytes (%s); %d tiles, %d frames, %d rooms" % (
- len(blob), ", ".join("%s %d" % kv for kv in sizes.items()), len(tiles.blobs), len(frames), len(packed)))
+ print("data.bin: %d bytes (%s); %d tiles, %d frames, %d rooms, %d blocks (%d bytes unpacked)" % (
+ len(blob), ", ".join("%s %d" % kv for kv in sizes.items()), len(tiles.blobs), len(frames), len(packed),
+ len(blocks.raw), sum(map(len, blocks.raw))))
if __name__ == "__main__":