Skip to content

Free games for graphing calculators: NumWorks, TI-Nspire CX, Casio fx-CG50 and soon TI-84 Plus CE. Balatro, Celeste, Hollow Knight, Minecraft and more.

Cookie Clicker: its pictures packed, half a megabyte less, drawn the same

Mason363046050eb2d8f
diff --git a/games/cookieclicker/README.md b/games/cookieclicker/README.md
index ff7621ef..329c887a 100644
--- a/games/cookieclicker/README.md
+++ b/games/cookieclicker/README.md
@@ -89,4 +89,4 @@ On the heavens: the arrows go from upgrade to upgrade, EXE buys one (or chooses
**Cookie Clicker** is by **[Orteil](https://orteil.dashnet.org)** and Opti (DashNet). This version is written in C for the NumWorks calculator: its pictures, texts, numbers and rules are the game's own (version 2.058), read from the game's public mirror [ozh/cookieclicker](https://github.com/ozh/cookieclicker) by `tools/fetch.py`, `tools/extract.js` (which runs the game's script for its numbers and texts) and `tools/pack.py` (which packs them into `src/data.bin`). Its fonts are **Merriweather**, **Kavoon** and **Noto Sans** (SIL Open Font License: [Merriweather](../../LICENSES/OFL-1.1-Merriweather.txt), [Kavoon](../../LICENSES/OFL-1.1-Kavoon.txt), [Noto Sans](../../LICENSES/OFL-1.1-NotoSans.txt)). Not affiliated with Orteil or DashNet.
-Build: `make` (needs `arm-none-eabi-gcc` and Node.js for nwlink); `make host test` plays it on a computer, and `make data REF=<folder>` packs the data again from what `python3 -I tools/fetch.py <folder>` gathers.
+Build: `make` (needs `arm-none-eabi-gcc` and Node.js for nwlink); `make host test` plays it on a computer, and `make data REF=<folder>` packs the data again from what `python3 -I tools/fetch.py <folder>` gathers. Most pictures are packed and unpacked when drawn (`tools/pictures.py`, which can also pack those of `src/data.bin` again, every pixel as it was).
diff --git a/games/cookieclicker/src/data.bin b/games/cookieclicker/src/data.bin
index 1d6677df..ea103fac 100644
Binary files a/games/cookieclicker/src/data.bin and b/games/cookieclicker/src/data.bin differ
diff --git a/games/cookieclicker/src/data.h b/games/cookieclicker/src/data.h
index 8308c0d8..14c5866a 100644
--- a/games/cookieclicker/src/data.h
+++ b/games/cookieclicker/src/data.h
@@ -5,7 +5,7 @@
#define MILK_TILE 178
#define ICON_COLS 36
#define ICON_ROWS 37
-#define OFF_ICONS 1118216
+#define OFF_ICONS 639456
#define F_TITLE 0
#define F_NAME 1
#define F_NAME_SMALL 2
@@ -15,8 +15,8 @@
#define F_ITALIC 6
#define F_SMALL 7
#define F_KAVOON 8
-#define OFF_FONTS 1168800
-#define OFF_BUILDINGS 1168840
+#define OFF_FONTS 690040
+#define OFF_BUILDINGS 690080
#define N_BUILDINGS 20
#define B_CURSOR 0
#define B_GRANDMA 1
@@ -38,7 +38,7 @@
#define B_IDLEVERSE 17
#define B_CORTEX_BAKER 18
#define B_YOU 19
-#define OFF_TIERS 1169960
+#define OFF_TIERS 691200
#define N_TIERS 18
#define TIER__1 0
#define TIER__2 1
@@ -58,7 +58,7 @@
#define TIER_SYNERGY1 15
#define TIER_SYNERGY2 16
#define TIER_FORTUNE 17
-#define OFF_UPGRADES 1170392
+#define OFF_UPGRADES 691632
#define N_UPGRADES 876
#define U_REINFORCED_INDEX_FINGER 0
#define U_CARPAL_TUNNEL_PREVENTION_CREAM 1
@@ -936,7 +936,7 @@
#define U_UNDECILLION_FINGERS 873
#define U_OMNIPLAST_MOUSE 874
#define U_WRINKLER_AMBERGRIS 875
-#define OFF_ACHIEVS 1226456
+#define OFF_ACHIEVS 747696
#define N_ACHIEVS 643
#define A_WAKE_AND_BAKE 0
#define A_MAKING_SOME_DOUGH 1
@@ -1582,47 +1582,47 @@
#define A_ALL_ON_DECK 641
#define A_A_ROUND_OF_APPLAUSE 642
#define N_MILKS 25
-#define OFF_MILKS 1239316
-#define OFF_BUILDING_LINKS 1239520
-#define OFF_TIER_UNSHACKLE 1242080
+#define OFF_MILKS 760556
+#define OFF_BUILDING_LINKS 760760
+#define OFF_TIER_UNSHACKLE 763320
#define N_TIER_UNSHACKLE 18
-#define OFF_COOKIE_UPGRADES 1242116
+#define OFF_COOKIE_UPGRADES 763356
#define N_COOKIE_UPGRADES 248
-#define OFF_GRANDMA_SYNERGIES 1242612
+#define OFF_GRANDMA_SYNERGIES 763852
#define N_GRANDMA_SYNERGIES 18
-#define OFF_KITTENS 1242648
+#define OFF_KITTENS 763888
#define N_KITTENS 18
-#define OFF_GOLDEN_UPGRADES 1242684
+#define OFF_GOLDEN_UPGRADES 763924
#define N_GOLDEN_UPGRADES 10
-#define OFF_LASTING 1242704
+#define OFF_LASTING 763944
#define N_LASTING 8
-#define OFF_NO_PERM 1242720
+#define OFF_NO_PERM 763960
#define N_NO_PERM 6
-#define OFF_HEAVENLY 1242732
+#define OFF_HEAVENLY 763972
#define N_HEAVENLY 129
-#define OFF_EASTER_EGGS 1242992
+#define OFF_EASTER_EGGS 764232
#define N_EASTER_EGGS 20
-#define OFF_HALLOWEEN_DROPS 1243032
+#define OFF_HALLOWEEN_DROPS 764272
#define N_HALLOWEEN_DROPS 7
-#define OFF_HEART_DROPS 1243048
+#define OFF_HEART_DROPS 764288
#define N_HEART_DROPS 7
-#define OFF_SANTA_DROPS 1243064
+#define OFF_SANTA_DROPS 764304
#define N_SANTA_DROPS 14
-#define OFF_REINDEER_DROPS 1243092
+#define OFF_REINDEER_DROPS 764332
#define N_REINDEER_DROPS 7
-#define OFF_FORTUNES 1243108
+#define OFF_FORTUNES 764348
#define N_FORTUNES 25
-#define OFF_SHOW_IF 1243160
+#define OFF_SHOW_IF 764400
#define N_SHOW_IF 3
-#define OFF_CONFIRM 1243168
+#define OFF_CONFIRM 764408
#define N_CONFIRM 6
-#define OFF_BANK_ACHIEVS 1243184
+#define OFF_BANK_ACHIEVS 764424
#define N_BANK_ACHIEVS 48
-#define OFF_CPS_ACHIEVS 1243952
+#define OFF_CPS_ACHIEVS 765192
#define N_CPS_ACHIEVS 48
-#define OFF_UNLOCK_AT 1244720
+#define OFF_UNLOCK_AT 765960
#define N_UNLOCK_AT 213
-#define OFF_NEWS 1249832
+#define OFF_NEWS 771072
#define N_NEWS 128
#define NK_GRANDMA 0
#define NK_GRANDMA_THREAT 1
@@ -1643,8 +1643,9 @@
#define NK_FOOLS_RARE 16
#define NK_FOOLS_BUILDING 17
#define NK_FOOLS_LORE 18
-#define OFF_TEXT 1250856
-#define OFF_IMAGES 1478348
+#define OFF_TEXT 772096
+#define OFF_PRIORS 114300
+#define OFF_IMAGES 999588
#define N_IMAGES 1556
#define IM_COOKIE 0
#define IM_COOKIE_SHADOW 1
@@ -1932,6 +1933,6 @@
#define IM_FAMILIAR1_1 283
#define IM_FAMILIAR1_2 284
#define IM_FAMILIAR1_3 285
-#define DATA_SIZE 1503244
+#define DATA_SIZE 1024484
#endif
diff --git a/games/cookieclicker/src/gfx.c b/games/cookieclicker/src/gfx.c
index d5bfab06..4d8425e5 100644
--- a/games/cookieclicker/src/gfx.c
+++ b/games/cookieclicker/src/gfx.c
@@ -178,12 +178,13 @@ typedef struct {
int x, y, cw, ch, bits, stride; /* the part kept: outside it, clear */
} Img;
+static const uint8_t *unpacked(int im);
static Img img(int im) {
const ImageRec *r = image_rec(im);
Img m;
- m.w = r->w, m.h = r->h, m.x = r->x, m.y = r->y, m.cw = r->cw, m.ch = r->ch, m.bits = r->bits;
+ m.w = r->w, m.h = r->h, m.x = r->x, m.y = r->y, m.cw = r->cw, m.ch = r->ch, m.bits = r->bits & 15;
m.stride = (m.cw * m.bits + 7) >> 3;
- m.px = cc_data + r->px;
+ m.px = r->bits & IM_PACKED ? unpacked(im) : cc_data + r->px;
m.col = (const uint16_t *)(const void *)(cc_data + r->pal);
m.opa = cc_data + r->pal + 2 * (r->ncolors_1 + 1);
return m;
@@ -212,6 +213,144 @@ static inline uint16_t texel(const Img *m, int x, int y, int *a) {
return m->col[i];
}
+/* ------------------------------------------------------------------ packed pictures */
+/* Most pictures are packed (tools/pictures.py: its model() is unpack()'s): when one is drawn, it is unpacked into
+ * the cache, which keeps the ones drawn latest (40 KB holds what any screen of the game draws). */
+#define CACHE_BYTES 40960
+#define CACHE_SLOTS 254
+static uint8_t cache[CACHE_BYTES];
+static struct {
+ uint16_t im, len;
+ uint32_t off, used;
+} slot[CACHE_SLOTS]; /* in the order they lie in the cache */
+static int nslots;
+static uint32_t cache_end, cache_used, cache_clock;
+static uint8_t where[N_IMAGES]; /* a picture's slot + 1, 0 if it isn't in the cache */
+
+static void cache_drop(int s) {
+ where[slot[s].im] = 0;
+ cache_used -= slot[s].len;
+ for (int t = s; t + 1 < nslots; t++) slot[t] = slot[t + 1], where[slot[t].im] = (uint8_t)(t + 1);
+ nslots--;
+}
+/* room for len bytes at the cache's end, the pictures drawn longest ago dropped (but keep), the others moved
+ * together if need be */
+static uint8_t *cache_room(int im, uint32_t len, int keep) {
+ while (cache_used + len > CACHE_BYTES || nslots == CACHE_SLOTS) {
+ int old = -1;
+ for (int s = 0; s < nslots; s++)
+ if (slot[s].im != keep && (old < 0 || slot[s].used < slot[old].used)) old = s;
+ cache_drop(old);
+ }
+ if (cache_end + len > CACHE_BYTES) {
+ cache_end = 0;
+ for (int s = 0; s < nslots; s++) {
+ if (slot[s].off != cache_end) memmove(cache + cache_end, cache + slot[s].off, slot[s].len);
+ slot[s].off = cache_end, cache_end += slot[s].len;
+ }
+ }
+ slot[nslots].im = (uint16_t)im, slot[nslots].len = (uint16_t)len, slot[nslots].off = cache_end;
+ slot[nslots].used = ++cache_clock;
+ where[im] = (uint8_t)++nslots;
+ cache_used += len;
+ cache_end += len;
+ return cache + cache_end - len;
+}
+
+/* LZMA's range decoder, chances of a 0 out of 2048 */
+typedef struct {
+ const uint8_t *in;
+ uint32_t range, code;
+} Rc;
+static inline int rc_bit(Rc *rc, uint16_t *p) {
+ uint32_t bound = (rc->range >> 11) * *p;
+ int b;
+ if (rc->code < bound) rc->range = bound, *p += (2048 - *p) >> 4, b = 0;
+ else rc->code -= bound, rc->range -= bound, *p -= *p >> 4, b = 1;
+ if (rc->range < 1u << 24) rc->range <<= 8, rc->code = rc->code << 8 | *rc->in++;
+ return b;
+}
+
+/* the contexts (pictures.py's) */
+enum { GUESS = 0, NEW = 400, UNARY = 405, FIRST = 413, OTHER = 422, CONTEXTS = 423 };
+
+/* picture r's pixels, from its packed data in, into out (as a raw picture's); ref: its reference (map: its colours
+ * to this one's, the last for where it is clear outside its part), or NULL */
+static void unpack(const ImageRec *r, const uint8_t *in, const Img *ref, const uint8_t *map, int nref, uint8_t *out) {
+ int cw = r->cw, ch = r->ch, n = r->ncolors_1 + 1, bits = r->bits & 15, stride = (cw * bits + 7) >> 3;
+ uint16_t p[CONTEXTS];
+ uint8_t rows[2][256], mtf[256], cands[5];
+ memcpy(p, cc_data + OFF_PRIORS, sizeof p);
+ for (int i = 0; i < n; i++) mtf[i] = (uint8_t)i;
+ int seen = 0;
+ Rc rc = {in + 4, 0xFFFFFFFFu, (uint32_t)in[0] << 24 | (uint32_t)in[1] << 16 | (uint32_t)in[2] << 8 | in[3]};
+ for (int y = 0; y < ch; y++) {
+ uint8_t *row = rows[y & 1], *up = rows[(y & 1) ^ 1], *o = out + y * stride;
+ for (int x = 0; x < cw; x++) {
+ int L = x ? row[x - 1] : -1, U = y ? up[x] : -1;
+ int UL = x && y ? up[x - 1] : -1, UR = y && x + 1 < cw ? up[x + 1] : -1;
+ int shape = (L >= 0 && L == U) | (U >= 0 && U == UL) << 1 | (L >= 0 && L == UL) << 2 | (U >= 0 && U == UR) << 3;
+ int g[5] = {-1, L, U, UL, UR}, j = 0, got = -1;
+ if (ref) {
+ int v = index_at(ref, r->x + x, r->y + y), m = map[v < 0 ? nref : v];
+ g[0] = m == 255 ? -1 : m;
+ }
+ for (int s = 0; s < 5 && got < 0; s++) { /* (the neighbours, each once: is it this one?) */
+ int c = g[s], k = 0;
+ if (c < 0) continue;
+ while (k < j && cands[k] != c) k++;
+ if (k < j) continue;
+ if (!rc_bit(&rc, &p[GUESS + (s * 5 + j) * 16 + shape])) got = c;
+ else cands[j++] = (uint8_t)c;
+ }
+ if (got < 0) {
+ if (seen < n && !rc_bit(&rc, &p[NEW + (j < 4 ? j : 4)])) got = seen++;
+ else { /* (a colour seen before: its rank among the others, the latest first) */
+ int k = 0, q = 1;
+ while (k < 8 && rc_bit(&rc, &p[UNARY + k])) k++;
+ for (int t = k - 1; t >= 0; t--) q = q << 1 | rc_bit(&rc, &p[t == k - 1 ? FIRST + k : OTHER]);
+ for (int i = 0; got < 0; i++) {
+ int c = mtf[i], kk = 0;
+ while (kk < j && cands[kk] != c) kk++;
+ if (kk == j && --q == 0) got = c;
+ }
+ }
+ }
+ row[x] = (uint8_t)got;
+ if (mtf[0] != got) {
+ int i = 1;
+ while (mtf[i] != got) i++;
+ memmove(mtf + 1, mtf, (size_t)i);
+ mtf[0] = (uint8_t)got;
+ }
+ if (bits == 8) o[x] = (uint8_t)got;
+ else if (x & 1) o[x >> 1] |= (uint8_t)(got << 4);
+ else o[x >> 1] = (uint8_t)got;
+ }
+ }
+}
+
+/* a packed picture's pixels, unpacked (its reference first) */
+static const uint8_t *unpacked(int im) {
+ if (where[im]) {
+ slot[where[im] - 1].used = ++cache_clock;
+ return cache + slot[where[im] - 1].off;
+ }
+ const ImageRec *r = image_rec(im);
+ const uint8_t *in = cc_data + r->px;
+ int ref = -1;
+ if (r->bits & IM_REF) ref = in[0] | in[1] << 8, unpacked(ref);
+ uint8_t *out = cache_room(im, (uint32_t)(((r->cw * (r->bits & 15) + 7) >> 3) * r->ch), ref);
+ if (ref < 0) {
+ unpack(r, in, NULL, NULL, 0, out);
+ } else {
+ Img m = img(ref); /* (where it is now) */
+ int nref = image_rec(ref)->ncolors_1 + 1;
+ unpack(r, in + 3 + nref, &m, in + 2, nref, out);
+ }
+ return out;
+}
+
/* a row of n pixels of the picture from (sx, sy), on p: each format its own loop (most of what is drawn) */
static void row(const Img *m, int sx, int sy, int n, uint16_t *p, int alpha) {
sx -= m->x, sy -= m->y;
diff --git a/games/cookieclicker/src/res.h b/games/cookieclicker/src/res.h
index 06ee632a..ee20e2a9 100644
--- a/games/cookieclicker/src/res.h
+++ b/games/cookieclicker/src/res.h
@@ -1,8 +1,8 @@
/* Cookie Clicker: data.bin (tools/pack.py), read where it lies in flash.
*
* Pictures (IM_*) are the part of them that isn't clear, a byte, half a byte or nothing a
- * pixel (tools/pictures.py), with a palette of colours and opacities; the icons are the
- * game's icon sheet at 24 pixels; fonts (F_*)
+ * pixel, with a palette of colours and opacities; most are packed, and unpacked by gfx.c
+ * when drawn (tools/pictures.py); the icons are the game's icon sheet at 24 pixels; fonts (F_*)
* are 4-bit glyphs; texts are the game's own, in a small markup (see text.c); and
* the game's tables: buildings, tiers, upgrades, achievements, milks. */
#ifndef CC_RES_H
@@ -16,11 +16,11 @@ extern const uint8_t cc_data[];
typedef struct {
uint8_t w, h;
uint8_t x, y, cw, ch; /* the part kept (outside it, clear) */
- uint8_t bits, ncolors_1; /* bits a pixel (FMT_*); colours - 1 */
+ uint8_t bits, ncolors_1; /* bits a pixel (FMT_*), | IM_PACKED, | IM_REF; colours - 1 */
uint32_t px, pal; /* offsets: the part's pixels (each row a whole number of bytes, the first pixel
- lowest), the palette (colours RGB565, then their opacities) */
+ lowest) or its packed data, the palette (colours RGB565, then their opacities) */
} ImageRec;
-enum { FMT_P0 = 0, FMT_P4 = 4, FMT_P8 = 8 }; /* (P0: all of one colour) */
+enum { FMT_P0 = 0, FMT_P4 = 4, FMT_P8 = 8, IM_PACKED = 0x80, IM_REF = 0x40 }; /* (P0: all of one colour) */
typedef struct {
double price, cps;
diff --git a/games/cookieclicker/tools/pack.py b/games/cookieclicker/tools/pack.py
index b36edf1f..e99b9f6a 100644
--- a/games/cookieclicker/tools/pack.py
+++ b/games/cookieclicker/tools/pack.py
@@ -89,9 +89,9 @@ class Pack:
def __init__(self):
self.blob = bytearray(pictures.MAGIC)
self.images = [] # the pictures, as drawn (pictures.plane)
- self.groups = [] # (the pictures whose pixels are shared: pictures.lay_out)
+ self.labels = [] # their names, the icons' "icon <x> <y>" (pictures.lay_out)
self.names = {}
- self.recs = None
+ self.recs = self.priors = None
self.defines = []
def align(self, n=4):
@@ -104,7 +104,7 @@ class Pack:
self.blob += data
return off
- def image(self, name, img, fmt="p8", ncolors=256, group=None):
+ def image(self, name, img, fmt="p8", ncolors=256, place=None):
"""adds a picture (RGB565, or quantized to a palette of ncolors, 16 for p4); returns its index"""
w, h = img.size
if fmt == "rgb":
@@ -120,12 +120,12 @@ class Pack:
if name in self.names:
raise SystemExit(f"pack: two pictures named {name}")
self.names[name] = len(self.images) - 1
- self.groups.append(group or pictures.group(name))
+ self.labels.append(name or "icon %d %d" % place)
return len(self.images) - 1
def lay_out(self):
"""the pictures into data.bin"""
- self.recs = pictures.lay_out(self.images, self.groups, self.put)
+ self.recs, self.priors = pictures.lay_out(self.images, self.labels, self.put)
def load(ref, rel):
@@ -758,10 +758,11 @@ def main(ref, game_json):
if np.asarray(cell)[..., 3].max() < 8:
icon_index.append(0xFFFF)
continue
- icon_index.append(P.image(None, premul_resize(cell, (24, 24)), "p4", group=f"icons{y}"))
+ icon_index.append(P.image(None, premul_resize(cell, (24, 24)), "p4", place=(x, y)))
P.lay_out()
icons_off = P.put(struct.pack(f"<{len(icon_index)}H", *icon_index))
- defs += [f"#define ICON_COLS {cols}", f"#define ICON_ROWS {rows}", f"#define OFF_ICONS {icons_off}"]
+ defs += [f"#define ICON_COLS {cols}", f"#define ICON_ROWS {rows}", f"#define OFF_ICONS {icons_off}",
+ f"#define OFF_PRIORS {P.priors}"]
# ---- fonts
F = os.path.join(ref, "fonts")
@@ -967,13 +968,8 @@ def main(ref, game_json):
f.write("\n".join(lines))
if UNKNOWN:
print("pack: characters left out:", "".join(sorted(UNKNOWN)))
- pictures.check(P.images, bytes(P.blob), dict(N_IMAGES=len(P.images), OFF_IMAGES=img_off))
- sizes = {}
- for g, (w, h, x, y, cw, ch, bits, n, px, pal) in zip(P.groups, P.recs):
- g = re.sub(r"\d+$", "", g)
- sizes[g] = sizes.get(g, 0) + (cw * bits + 7) // 8 * ch + 3 * n
- print(f"data.bin: {len(P.blob)} bytes, {len(P.images)} pictures, texts {len(T.blob)}")
- print(" pictures (before sharing): " + ", ".join(f"{k} {v // 1024} KB" for k, v in sorted(sizes.items(), key=lambda kv: -kv[1])))
+ pictures.check(P.images, bytes(P.blob), dict(N_IMAGES=len(P.images), OFF_IMAGES=img_off, OFF_PRIORS=P.priors))
+ print(f"data.bin: {len(P.blob)} bytes, {len(P.images)} pictures ({icons_off} bytes), texts {len(T.blob)}")
if __name__ == "__main__":
diff --git a/games/cookieclicker/tools/pictures.py b/games/cookieclicker/tools/pictures.py
index d39eb35d..c43a28a1 100644
--- a/games/cookieclicker/tools/pictures.py
+++ b/games/cookieclicker/tools/pictures.py
@@ -8,13 +8,26 @@ A picture is its colours (RGB565) and opacities, pixel by pixel; a clear pixel's
- only the part of it that isn't clear (outside it, clear);
- a palette of its colours and opacities, each once (a palette seen before is not stored again);
- its pixels as indices to it: none (one colour), half a byte a pixel (16 colours or fewer, the first pixel
- lowest) or a byte, each row a whole number of bytes. The pixels of each group of pictures (by name, ROW_*,
- MILK_*..., the icons by rows of the sheet) are laid out with tools/share.py: a string seen before, or inside
- another, is not stored again; tails overlap heads.
+ lowest) or a byte, each row a whole number of bytes. The raw pixels of each group of pictures (by name,
+ BORDER_*, CURSOR_R*...) are laid out with tools/share.py: a string seen before, or inside another, is not
+ stored again; tails overlap heads.
+
+The pictures drawn on every frame of the game's screen (RAW, and the heavens' stars) are kept so, read where they
+lie; the others are packed (gfx.c unpacks one into a cache in RAM when it is drawn, its pixels then as a raw
+picture's): each pixel, in rows, is told from the ones already there by a range coder (LZMA's) with adaptive
+chances, as gfx.c's unpack(): is it the pixel on its left, above, above on the left, above on the right (each
+once)? If not, a colour not seen yet (the palette is in the order the colours come), or one seen before, by how
+recently (its rank, past those neighbours). The chances start at PRIORS, the same for every picture, learnt from
+them all. An icon can be told from another (its reference: one above it or on its left in the sheet, which may
+have one too, up to DEPTH): the reference's colours mapped to its own are the first guess for each pixel. That is
+how the icons that are the same picture in other colours (the tiers of the buildings' upgrades) are kept nearly
+once.
Its record (IMAGES, 16 bytes; res.h's ImageRec): w, h, then the part kept: x, y, w, h (u8 each), the bits a pixel
-(0, 4, 8), colours - 1 (u8 each), the pixels' offset, the palette's (u32): its colours (RGB565), then their
-opacities (u8 each)."""
+(0, 4, 8; | PACKED, | REF), colours - 1 (u8 each), the pixels' offset (or the packed data's), the palette's (u32):
+its colours (RGB565), then their opacities (u8 each). Packed data: with REF, the reference's index (u16) and a map
+from its colours to the picture's, one byte each (0xFF none), the last one for where the reference is clear
+outside its part; then the range coder's bytes."""
import os
import re
import struct
@@ -27,6 +40,183 @@ sys.path.insert(0, os.path.join(HERE, "..", "..", "..", "tools"))
from share import share # noqa: E402
MAGIC = b"CCK2"
+PACKED, REF = 0x80, 0x40
+DEPTH = 4
+CACHE = 40960 # (gfx.c's CACHE_BYTES: a packed picture and its reference fit in half of it)
+# drawn on every frame of the game's screen, or over all of it (the heavens' stars): kept raw
+RAW = re.compile(r"(SHOWER|BG_BLUE|BG_GRANDMAS|COOKIE|FLARE|GLINT|CURSOR_R|SMALL_COOKIE|BORDER|STORE_TILE|PANEL|"
+ r"UPGRADE_FRAME|MONEY|HEAVENLY_MONEY|LUMP_MONEY|GOLD_COOKIE|WRATH_COOKIE|WRINKLER|SHINY_WRINKLER|"
+ r"DARK_NOISE|STARS)")
+
+# ---------------------------------------------------------------------------------------------- packed pictures
+# the coder's contexts (gfx.c's): a neighbour's guess (by which neighbour, its place among the guesses and how the
+# neighbours match), a colour not seen yet (by how many guesses), a rank (Elias gamma: its length in unary, the
+# first bit after its top one, the others)
+GUESS, NEW, UNARY, FIRST, OTHER = 0, 400, 405, 413, 422
+CONTEXTS = 423
+
+
+def model(cw, ch, n, guess, bit, idx=None):
+ """the pixels (cw x ch, n colours) coded (idx) or decoded (idx None) with bit(context, b or None), which codes
+ b (or decodes it) and gives it; guess: the reference's guesses (-1 none) or None. Gives the pixels."""
+ out = [[0] * cw for _ in range(ch)]
+ mtf, seen = list(range(n)), 0 # (the colours, the latest first)
+ for y in range(ch):
+ row, up = out[y], out[y - 1] if y else None
+ for x in range(cw):
+ v = idx[y][x] if idx is not None else None
+ L = row[x - 1] if x else -1
+ U = up[x] if y else -1
+ UL = up[x - 1] if x and y else -1
+ UR = up[x + 1] if y and x + 1 < cw else -1
+ shape = ((L >= 0 and L == U) | (U >= 0 and U == UL) << 1 | (L >= 0 and L == UL) << 2
+ | (U >= 0 and U == UR) << 3)
+ cands, j, got = [], 0, None
+ for s, c in enumerate((guess[y][x] if guess else -1, L, U, UL, UR)):
+ if c < 0 or c in cands:
+ continue
+ if not bit(GUESS + (s * 5 + j) * 16 + shape, None if v is None else int(v != c)):
+ got = c
+ break
+ cands.append(c)
+ j += 1
+ if got is None:
+ if seen < n and not bit(NEW + min(j, 4), None if v is None else int(v != seen)):
+ got = seen
+ seen += 1
+ else:
+ others = [c for c in mtf if c not in cands]
+ q = others.index(v) + 1 if v is not None else 0
+ k = 0
+ while k < 8 and bit(UNARY + k, None if v is None else int(q >> (k + 1) > 0)):
+ k += 1
+ r = 1
+ for t in range(k - 1, -1, -1):
+ r = r << 1 | bit(FIRST + k if t == k - 1 else OTHER, None if v is None else q >> t & 1)
+ got = others[r - 1]
+ row[x] = got
+ if mtf[0] != got:
+ mtf.remove(got)
+ mtf.insert(0, got)
+ return out
+
+
+class Encoder:
+ """LZMA's range coder; the first byte (always 0) left out, and only the bytes the decoder needs at the end (it
+ reads two more: what follows)"""
+
+ def __init__(self, probs):
+ self.low, self.range, self.cache, self.pending, self.out, self.p = 0, 0xFFFFFFFF, 0, 1, bytearray(), probs
+
+ def shift(self):
+ if self.low < 0xFF000000 or self.low >= 1 << 32:
+ carry, b = self.low >> 32, self.cache
+ while self.pending:
+ self.out.append((b + carry) & 255)
+ b, self.pending = 255, self.pending - 1
+ self.cache = self.low >> 24 & 255
+ self.pending += 1
+ self.low = self.low << 8 & 0xFFFFFFFF
+
+ def bit(self, ctx, b):
+ p = self.p[ctx]
+ bound = (self.range >> 11) * p
+ if b:
+ self.low += bound
+ self.range -= bound
+ self.p[ctx] = p - (p >> 4)
+ else:
+ self.range = bound
+ self.p[ctx] = p + ((2048 - p) >> 4)
+ if self.range < 1 << 24:
+ self.range <<= 8
+ self.shift()
+ return b
+
+ def finish(self):
+ self.low = (self.low + 0xFFFF) & ~0xFFFF # (any two bytes after these read as this)
+ for _ in range(5):
+ self.shift()
+ assert self.out[0] == 0 and self.out[-2:] == b"\0\0"
+ return bytes(self.out[1:-2])
+
+
+class Decoder:
+ def __init__(self, data, at, probs):
+ self.d, self.at, self.range, self.p = data, at + 4, 0xFFFFFFFF, probs
+ self.code = int.from_bytes(data[at:at + 4], "big")
+
+ def bit(self, ctx, _):
+ p = self.p[ctx]
+ bound = (self.range >> 11) * p
+ if self.code < bound:
+ self.range = bound
+ self.p[ctx] = p + ((2048 - p) >> 4)
+ b = 0
+ else:
+ self.code -= bound
+ self.range -= bound
+ self.p[ctx] = p - (p >> 4)
+ b = 1
+ if self.range < 1 << 24:
+ self.range <<= 8
+ self.code = (self.code << 8 | (self.d[self.at] if self.at < len(self.d) else 0)) & 0xFFFFFFFF
+ self.at += 1
+ return b
+
+
+def guesses(part, ref, mapping):
+ """the reference's guesses for a part (x, y, cw, ch) of a picture: its colours mapped (the last entry: where
+ it is clear outside its part)"""
+ x, y, cw, ch = part
+ rx, ry, rcw, rch, ridx = ref
+ out = []
+ for yy in range(y, y + ch):
+ row = []
+ for xx in range(x, x + cw):
+ v = ridx[yy - ry][xx - rx] if 0 <= xx - rx < rcw and 0 <= yy - ry < rch else len(mapping) - 1
+ m = mapping[v]
+ row.append(-1 if m == 255 else m)
+ out.append(row)
+ return out
+
+
+def best_mapping(part, idx, ref, nref):
+ """for each of the reference's colours (and its clear outside), the picture's colour found most with it"""
+ x, y, cw, ch = part
+ rx, ry, rcw, rch, ridx = ref
+ count = {}
+ for yy in range(ch):
+ for xx in range(cw):
+ X, Y = x + xx, y + yy
+ v = ridx[Y - ry][X - rx] if 0 <= X - rx < rcw and 0 <= Y - ry < rch else nref
+ k = (v, idx[yy][xx])
+ count[k] = count.get(k, 0) + 1
+ mapping = [255] * (nref + 1)
+ for (a, b), _ in sorted(count.items(), key=lambda e: (-e[1], e[0])):
+ if mapping[a] == 255:
+ mapping[a] = b
+ return mapping
+
+
+def pack(idx, n, guess, priors):
+ e = Encoder(list(priors))
+ model(len(idx[0]), len(idx), n, guess, e.bit, idx)
+ return e.finish()
+
+
+def learn(pics):
+ """chances to start from, from how often each context's bits are 0: pics, (idx, n, guess)"""
+ zeros, total = [0] * CONTEXTS, [0] * CONTEXTS
+
+ def bit(ctx, b):
+ total[ctx] += 1
+ zeros[ctx] += 1 - b
+ return b
+ for idx, n, guess in pics:
+ model(len(idx[0]), len(idx), n, guess, bit, idx)
+ return [min(max((4096 * zeros[c] + total[c]) // (2 * total[c]) if total[c] else 1024, 31), 2017)
+ for c in range(CONTEXTS)]
def plane(colours, opacities):
@@ -36,10 +226,10 @@ def plane(colours, opacities):
return np.where(a > 0, c << 8 | a, 0).astype(np.uint32)
-def lay_out(pictures, groups, put):
- """pictures: (h x w) planes (plane()); groups: a name for each, the pictures that share their pixels
- (tools/share.py, slow on many); put(bytes, align) adds bytes to data.bin, giving their offset.
- Returns the records (IMAGES), in order."""
+def lay_out(pictures, names, put):
+ """pictures: (h x w) planes (plane()); names: their IM_ names, the icons' "icon <x> <y>" (their place in the
+ sheet); put(bytes, align) adds bytes to data.bin, giving their offset. Returns the records (IMAGES), in order,
+ and the offset of PRIORS."""
kept, pals, recs = [], {}, []
for p in pictures:
h, w = p.shape
@@ -62,46 +252,96 @@ def lay_out(pictures, groups, put):
if n > 256:
raise SystemExit(f"pictures: a picture of {n} colours")
bits = 0 if n == 1 else 4 if n <= 16 else 8
- if bits == 4:
- if idx.shape[1] % 2:
- idx = np.concatenate([idx, np.zeros((idx.shape[0], 1), np.uint8)], 1)
- px = (idx[:, 0::2] | idx[:, 1::2] << 4).astype(np.uint8).tobytes()
- else:
- px = idx.tobytes() if bits else b""
pal = b"".join(struct.pack("<H", int(c) >> 8) for c in colours) + bytes(int(c) & 255 for c in colours)
if pal not in pals:
pals[pal] = put(pal, 2)
- kept.append((w, h, x0, y0, x1 - x0, y1 - y0, bits, n, px, pals[pal]))
- at = {} # picture -> its pixels' offset
+ kept.append(dict(w=w, h=h, part=(x0, y0, x1 - x0, y1 - y0), bits=bits, n=n, idx=idx.tolist(), pal=pals[pal]))
+ packed = [i for i, k in enumerate(kept) if k and k["bits"] and not RAW.match(names[i])]
+
+ # the icons' references: the one above or on the left (up to 3 away) that makes it shortest, if any
+ priors = learn([(kept[i]["idx"], kept[i]["n"], None) for i in packed])
+ at = {tuple(map(int, nm.split()[1:])): i for i, nm in enumerate(names) if nm.startswith("icon ")}
+ ref, depth = {}, {}
+ icons = [i for i in packed if names[i].startswith("icon ")]
+ for i in sorted(icons, key=lambda i: tuple(map(int, names[i].split()[:0:-1]))): # (row by row)
+ k = kept[i]
+ cx, cy = map(int, names[i].split()[1:])
+ best = (len(pack(k["idx"], k["n"], None, priors)), None, None)
+ for j in [at.get((cx, cy - d)) for d in (1, 2, 3)] + [at.get((cx - d, cy)) for d in (1, 2, 3)]:
+ if j not in icons or depth.get(j, 0) >= DEPTH or (kept[j]["w"], kept[j]["h"]) != (k["w"], k["h"]):
+ continue
+ r = kept[j]["part"] + (kept[j]["idx"],)
+ mapping = best_mapping(k["part"], k["idx"], r, kept[j]["n"])
+ size = len(pack(k["idx"], k["n"], guesses(k["part"], r, mapping), priors)) + 2 + len(mapping)
+ if size < best[0]:
+ best = (size, j, mapping)
+ if best[1] is not None:
+ ref[i] = best[1:]
+ depth[i] = depth.get(best[1], 0) + 1
+
+ def guess_of(i):
+ if i not in ref:
+ return None
+ j, mapping = ref[i]
+ return guesses(kept[i]["part"], kept[j]["part"] + (kept[j]["idx"],), mapping)
+ priors = learn([(kept[i]["idx"], kept[i]["n"], guess_of(i)) for i in packed])
+ priors_at = put(struct.pack(f"<{CONTEXTS}H", *priors), 2)
+
+ size = lambda i: (kept[i]["part"][2] * kept[i]["bits"] + 7) // 8 * kept[i]["part"][3]
+ for i in packed:
+ if size(i) + (size(ref[i][0]) if i in ref else 0) > CACHE // 2:
+ raise SystemExit(f"pictures: {names[i]} too big to pack")
+ data = {}
+ for i in packed:
+ head = struct.pack("<H", ref[i][0]) + bytes(ref[i][1]) if i in ref else b""
+ data[i] = head + pack(kept[i]["idx"], kept[i]["n"], guess_of(i), priors)
+ for i, k in enumerate(kept):
+ if k and k["bits"] and i not in data:
+ idx = np.array(k["idx"], np.uint8)
+ if k["bits"] == 4:
+ if idx.shape[1] % 2:
+ idx = np.concatenate([idx, np.zeros((idx.shape[0], 1), np.uint8)], 1)
+ data[i] = (idx[:, 0::2] | idx[:, 1::2] << 4).astype(np.uint8).tobytes()
+ else:
+ data[i] = idx.tobytes()
+ # the raw pixels of each group shared (tools/share.py); the packed data one after the other
+ offs = {}
+ groups = [group(nm) for nm in names]
for g in dict.fromkeys(groups):
- mine = [i for i, k in enumerate(kept) if k and k[8] and groups[i] == g]
+ mine = [i for i in data if i not in packed and groups[i] == g]
if mine:
- blob, offs = share([kept[i][8] for i in mine])
+ blob, o = share([data[i] for i in mine])
base = put(blob, 1)
- at.update((i, base + o) for i, o in zip(mine, offs))
+ offs.update((i, base + x) for i, x in zip(mine, o))
+ for i in packed:
+ offs[i] = put(data[i], 1)
+ put(b"\0\0", 1) # (the range decoder reads two bytes past the last)
for i, (p, k) in enumerate(zip(pictures, kept)):
if k is None:
recs.append((p.shape[1], p.shape[0], 0, 0, 0, 0, 0, 1, 0, 0))
else:
- recs.append(k[:8] + (at.get(i, 0), k[9]))
- return recs
+ flags = (PACKED if i in packed else 0) | (REF if i in ref else 0)
+ recs.append((k["w"], k["h"]) + k["part"] + (k["bits"] | flags, k["n"], offs.get(i, 0), k["pal"]))
+ return recs, priors_at
def group(name):
- """the group a picture's pixels are shared in: by its name (ROW_FARM: ROW), or icons by their row"""
+ """the group a picture's raw pixels are shared in: by its name (ROW_FARM: ROW), the icons by rows of the sheet"""
+ if name.startswith("icon "):
+ return "icons" + name.split()[2]
return re.sub(r"(_.*|\d+)$", "", name)
-def groups_of(data, h):
- g = ["?"] * h["N_IMAGES"]
+def names_of(data, h):
+ names = ["?"] * h["N_IMAGES"]
cols = h["ICON_COLS"]
for k, i in enumerate(struct.unpack_from(f"<{cols * h['ICON_ROWS']}H", data, h["OFF_ICONS"])):
if i != 0xFFFF:
- g[i] = f"icons{k // cols}"
+ names[i] = f"icon {k % cols} {k // cols}"
for k, v in h.items():
if k.startswith("IM_"):
- g[v] = group(k[3:])
- return g
+ names[v] = k[3:]
+ return names
def records(recs):
@@ -111,16 +351,20 @@ def records(recs):
def pictures_of(data, h):
"""the pictures of a data.bin (and its data.h's numbers), as planes"""
- out = []
+ out = {}
old = data[:4] == b"CCK1" # (before: no part, pictures of a byte, half a byte or RGB565 a pixel)
- for i in range(h["N_IMAGES"]):
+ parts = {} # picture -> (x, y, cw, ch, indices)
+
+ def read(i):
+ if i in out:
+ return
at = h["OFF_IMAGES"] + 16 * i
if old:
w, hh, fmt, n1, _, px, pal = struct.unpack_from("<HHBBHII", data, at)
if fmt == 0:
c = np.frombuffer(data, "<u2", w * hh, px).reshape(hh, w)
- out.append(plane(c, np.full((hh, w), 255)))
- continue
+ out[i] = plane(c, np.full((hh, w), 255))
+ return
x, y, cw, ch, bits = 0, 0, w, hh, 8 if fmt == 1 else 4
else:
w, hh, x, y, cw, ch, bits, n1, px, pal = struct.unpack_from("<BBBBBBBBII", data, at)
@@ -129,16 +373,30 @@ def pictures_of(data, h):
n = n1 + 1
cols = np.frombuffer(data, "<u2", n, pal).astype(np.uint32)
opa = np.frombuffer(data, np.uint8, n, pal + 2 * n).astype(np.uint32)
- if bits == 8:
+ if bits & PACKED:
+ guess = None
+ if bits & REF:
+ j = struct.unpack_from("<H", data, px)[0]
+ read(j)
+ nref = struct.unpack_from("<B", data, h["OFF_IMAGES"] + 16 * j + 7)[0] + 1
+ mapping = list(data[px + 2:px + 3 + nref])
+ guess = guesses((x, y, cw, ch), parts[j], mapping)
+ px += 3 + nref
+ priors = list(struct.unpack_from(f"<{CONTEXTS}H", data, h["OFF_PRIORS"]))
+ idx = np.array(model(cw, ch, n, guess, Decoder(data, px, priors).bit), np.uint8)
+ elif bits == 8:
idx = np.frombuffer(data, np.uint8, cw * ch, px).reshape(ch, cw)
elif bits == 4:
b = np.frombuffer(data, np.uint8, (cw + 1) // 2 * ch, px).reshape(ch, (cw + 1) // 2)
idx = np.stack([b & 15, b >> 4], 2).reshape(ch, -1)[:, :cw]
else:
idx = np.zeros((ch, cw), np.uint8)
+ parts[i] = (x, y, cw, ch, idx.tolist())
p[y:y + ch, x:x + cw] = plane(cols[idx], opa[idx])
- out.append(p)
- return out
+ out[i] = p
+ for i in range(h["N_IMAGES"]):
+ read(i)
+ return [out[i] for i in range(h["N_IMAGES"])]
def check(pictures, data, h):
@@ -169,7 +427,7 @@ def main():
out.append(0)
out.extend(b)
return len(out) - len(b)
- recs = lay_out(pics, groups_of(data, h), put)
+ recs, priors = lay_out(pics, names_of(data, h), put)
start, end = h["OFF_ICONS"], h["OFF_IMAGES"] # (the rest: what isn't pictures)
while (len(out) - start) % 8: # (its tables of doubles stay 8-aligned)
out.append(0)
@@ -184,13 +442,17 @@ def main():
out.append(0)
images = len(out)
out += records(recs)
- check(pics, bytes(out), dict(h, OFF_IMAGES=images))
+ check(pics, bytes(out), dict(h, OFF_IMAGES=images, OFF_PRIORS=priors))
lines = []
for line in open(h_path).read().split("\n"):
m = re.match(r"#define (OFF_\w+|DATA_SIZE) (\d+)$", line)
+ if m and m.group(1) == "OFF_PRIORS":
+ continue
if m:
v = int(m.group(2))
v = images if m.group(1) == "OFF_IMAGES" else len(out) if m.group(1) == "DATA_SIZE" else v + moved
+ if m.group(1) == "OFF_IMAGES":
+ lines.append(f"#define OFF_PRIORS {priors}")
line = f"#define {m.group(1)} {v}"
lines.append(line)
open(bin_path, "wb").write(out)