Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

Release 2026-09-27: janas-get, a model in one command

camauri4534903c229e
diff --git a/ChangeLog.md b/ChangeLog.md
index 7eeee51..8bcdfc9 100644
--- a/ChangeLog.md
+++ b/ChangeLog.md
@@ -2,6 +2,25 @@
> Curated, user-facing summary of completed work, newest first.
+## [2026-09-27] - janas-get: a model in one command
+
+- **`janas-get`** downloads a model of its catalog (the thirteen of
+ MODELS.md: Qwen3, Qwen3.5, Qwen3.6, Qwen3-Next, Coder-Next, the
+ embedding model, Gemma 4) from Hugging Face, checks the SHA-256 of
+ every file against MODELS.md, runs the converters of the same build
+ (gguf2jns, jns_planes) and makes the prediction file beside the model -
+ hf2jns_mtp over range requests, or Gemma 4's assistant - with its
+ fingerprint too. Downloads resume; files already there are checked, not
+ fetched again; the GGUF goes as soon as it is converted, so the disk
+ holds two copies at the most. `janas-get hf:<owner>/<repo>/<file>` takes
+ any GGUF, checked against the SHA-256 Hugging Face lists.
+- Its own HTTPS download in pieces (the client of hf2jns_mtp) and its own
+ SHA-256, on the x86 SHA extensions where the CPU has them (1.4 GB/s on
+ the development laptop): no curl, no sha256sum.
+- **`janas-try.sh` gets its models through it**: the catalog and the
+ fingerprints are in one place. A test checks the catalog against
+ MODELS.md, digit by digit.
+
## [2026-09-27] - janas-try tries Gemma
- **`janas-try.sh --level gemma`**: Gemma-4-E4B and its assistant,
diff --git a/README.md b/README.md
index 4cf0ce1..1dfa4a1 100644
--- a/README.md
+++ b/README.md
@@ -213,6 +213,25 @@ sudo pacman -S vulkan-headers glslang vulkan-tools # the same, Arch
## Getting a model
+**The short way: `janas-get`.** One command downloads a model from
+Hugging Face, checks the SHA-256 of every file against [MODELS.md](MODELS.md),
+converts it and makes its prediction file beside it (an MTP block, or Gemma
+4's assistant), which the chat then finds by itself:
+
+```sh
+bin/x86_64-linux/janas-get list # the models it knows
+bin/x86_64-linux/janas-get gemma-4-e4b # into models/
+bin/x86_64-linux/janas-chat models/gemma-4-e4b-it-q4km.jns
+```
+
+A download stopped halfway resumes where it stopped, a file already there
+is checked rather than fetched again, and the GGUF goes once converted
+(`--keep` keeps it). Any other GGUF: `janas-get
+hf:<owner>/<repo>/<file.gguf>`, checked against the SHA-256 Hugging Face
+lists for it. The memory column of `janas-get list` is the smallest
+machine each model is meant for. The rest of this section is the same by
+hand.
+
Janas reads its own format, `.jns`, converted from a **Q4_K_M** GGUF (Q3_K,
IQ3_S, IQ4_NL, IQ4_XS, Q4_0, Q4_1 and Q5_1 convert too, and so do unsloth's
"UD" files made of them; IQ2 and IQ1 types, not yet). Keep both
diff --git a/src/common/hf_download.c b/src/common/hf_download.c
new file mode 100644
index 0000000..0ee15a1
--- /dev/null
+++ b/src/common/hf_download.c
@@ -0,0 +1,186 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later
+ Copyright (C) 2026 Maurizio Cammalleri */
+/*
+ * hf_download.c - whole files of a Hugging Face repository, for janas-get
+ * (see hf_fetch.h): the size and SHA-256 Hugging Face lists for a file,
+ * and a download in pieces that resumes and hashes as it goes.
+ *
+ * Pieces rather than one long transfer: each is a request of its own on
+ * the address the redirect gave, so a broken connection costs one piece,
+ * and the signed address of the CDN, if it expires in a long download, is
+ * asked for again.
+ */
+#define _GNU_SOURCE
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "common/hf_fetch.h"
+#include "common/sha256.h"
+
+#define PIECE ((int64_t)128 << 20)
+#define TRIES 6
+
+int janas_hf_info(const char *repo, const char *rev, const char *path,
+ uint64_t *size, char sha[65], char *err, size_t err_len)
+{
+ /* the listing of the file's directory */
+ char url[1536], dir[1024] = "";
+ const char *slash = strrchr(path, '/');
+ if (slash)
+ snprintf(dir, sizeof(dir), "/%.*s", (int)(slash - path), path);
+ int k = snprintf(url, sizeof(url),
+ "https://huggingface.co/api/models/%s/tree/%s%s", repo,
+ rev ? rev : "main", dir);
+ if (k < 0 || (size_t)k >= sizeof(url)) {
+ snprintf(err, err_len, "%s: a name too long", path);
+ return -1;
+ }
+ struct janas_buf b = {0};
+ if (janas_hf_get_url(url, NULL, NULL, &b, NULL, 0, err, err_len) != 0) {
+ janas_buf_free(&b);
+ return -1;
+ }
+ struct janas_json_doc *d = janas_json_parse(b.p, b.n, err, err_len);
+ janas_buf_free(&b);
+ if (!d)
+ return -1;
+ int found = -1;
+ const struct janas_json *root = janas_json_root(d);
+ for (const struct janas_json *e =
+ root && root->type == JANAS_JSON_ARRAY ? root->child : NULL;
+ e; e = e->next) {
+ if (!janas_json_is(janas_json_get(e, "path"), path))
+ continue;
+ *size = (uint64_t)janas_json_num(janas_json_get(e, "size"), 0);
+ const char *oid =
+ janas_json_str(janas_json_get(janas_json_get(e, "lfs"), "oid"));
+ snprintf(sha, 65, "%s", oid && strlen(oid) == 64 ? oid : "");
+ found = 0;
+ break;
+ }
+ janas_json_free(d);
+ if (found != 0)
+ snprintf(err, err_len, "%s: not in %s", path, repo);
+ return found;
+}
+
+/* The address the file answers at once redirected */
+static int resolve(const char *repo, const char *rev, const char *path,
+ char *final, size_t len, char *err, size_t err_len)
+{
+ char url[1536];
+ int k =
+ snprintf(url, sizeof(url), "https://huggingface.co/%s/resolve/%s/%s",
+ repo, rev ? rev : "main", path);
+ if (k < 0 || (size_t)k >= sizeof(url)) {
+ snprintf(err, err_len, "%s: a name too long", path);
+ return -1;
+ }
+ struct janas_buf b = {0};
+ int64_t a = 0, z = 1;
+ int r = janas_hf_get_url(url, &a, &z, &b, final, len, err, err_len);
+ janas_buf_free(&b);
+ return r;
+}
+
+int janas_hf_download(const char *repo, const char *rev, const char *path,
+ const char *out_path, uint64_t size, char sha[65],
+ void (*progress)(uint64_t, uint64_t, void *), void *arg,
+ char *err, size_t err_len)
+{
+ int fd = open(out_path, O_RDWR | O_CREAT | O_CLOEXEC, 0644);
+ if (fd < 0) {
+ snprintf(err, err_len, "cannot write %s", out_path);
+ return -1;
+ }
+ struct stat st;
+ uint64_t done = fstat(fd, &st) == 0 ? (uint64_t)st.st_size : 0;
+ if (done > size) { /* not this file: start again */
+ if (ftruncate(fd, 0) != 0) {
+ close(fd);
+ snprintf(err, err_len, "cannot truncate %s", out_path);
+ return -1;
+ }
+ done = 0;
+ }
+ struct janas_sha256 h;
+ janas_sha256_init(&h);
+ struct janas_buf b = {0};
+ /* what is already there: hashed as it is, then continued */
+ if (done) {
+ size_t cap = (size_t)8 << 20;
+ char *buf = malloc(cap);
+ uint64_t at = 0;
+ while (buf && at < done) {
+ size_t want = done - at < cap ? (size_t)(done - at) : cap;
+ ssize_t r = pread(fd, buf, want, (off_t)at);
+ if (r <= 0)
+ break;
+ janas_sha256_update(&h, buf, (size_t)r);
+ at += (uint64_t)r;
+ }
+ free(buf);
+ if (at != done) {
+ close(fd);
+ snprintf(err, err_len, "cannot read %s", out_path);
+ return -1;
+ }
+ if (progress)
+ progress(done, size, arg);
+ }
+ char *final = malloc(8192);
+ int ok = final != NULL, tries = 0;
+ if (final)
+ final[0] = 0;
+ while (ok && done < size) {
+ if (!final[0] &&
+ resolve(repo, rev, path, final, 8192, err, err_len) != 0) {
+ if (++tries >= TRIES)
+ ok = 0;
+ continue;
+ }
+ int64_t a = (int64_t)done, z = (int64_t)(size - done) < PIECE
+ ? (int64_t)size
+ : (int64_t)done + PIECE;
+ b.n = 0;
+ if (janas_hf_get_url(final, &a, &z, &b, NULL, 0, err, err_len) != 0 ||
+ b.oom || (int64_t)b.n != z - a) {
+ final[0] = 0; /* ask for the address again */
+ if (++tries >= TRIES)
+ ok = 0;
+ continue;
+ }
+ tries = 0;
+ size_t w = 0;
+ while (w < b.n) {
+ ssize_t r = pwrite(fd, b.p + w, b.n - w, (off_t)(done + w));
+ if (r <= 0) {
+ snprintf(err, err_len, "cannot write %s (disk full?)",
+ out_path);
+ ok = 0;
+ break;
+ }
+ w += (size_t)r;
+ }
+ if (!ok)
+ break;
+ janas_sha256_update(&h, b.p, b.n);
+ done += b.n;
+ if (progress)
+ progress(done, size, arg);
+ }
+ free(final);
+ janas_buf_free(&b);
+ if (close(fd) != 0 && ok) {
+ snprintf(err, err_len, "cannot write %s", out_path);
+ ok = 0;
+ }
+ if (!ok)
+ return -1;
+ janas_sha256_hex(&h, sha);
+ return 0;
+}
diff --git a/src/common/hf_fetch.c b/src/common/hf_fetch.c
index bf9e78c..b832105 100644
--- a/src/common/hf_fetch.c
+++ b/src/common/hf_fetch.c
@@ -116,6 +116,13 @@ static int get_url(const char *url, const int64_t *from, const int64_t *to,
return -1;
}
+int janas_hf_get_url(const char *url, const int64_t *from, const int64_t *to,
+ struct janas_buf *out, char *final, size_t final_len,
+ char *err, size_t err_len)
+{
+ return get_url(url, from, to, out, final, final_len, err, err_len);
+}
+
int janas_hf_get(const char *repo, const char *rev, const char *path,
const int64_t *from, const int64_t *to, struct janas_buf *out,
char *err, size_t err_len)
diff --git a/src/common/hf_fetch.h b/src/common/hf_fetch.h
index 4e66f38..bfd129c 100644
--- a/src/common/hf_fetch.h
+++ b/src/common/hf_fetch.h
@@ -24,6 +24,28 @@ int janas_hf_get(const char *repo, const char *rev, const char *path,
const int64_t *from, const int64_t *to, struct janas_buf *out,
char *err, size_t err_len);
+/* The same by address (a file of the CDN, as a redirect gave it): with
+ final, the address the request ended at after the redirects. */
+int janas_hf_get_url(const char *url, const int64_t *from, const int64_t *to,
+ struct janas_buf *out, char *final, size_t final_len,
+ char *err, size_t err_len);
+
+/* A file of repo at rev (NULL: main): its size and, for a file kept by
+ LFS (weights), the SHA-256 Hugging Face gives for it (sha empty
+ otherwise). 0, or -1 with the reason in err. */
+int janas_hf_info(const char *repo, const char *rev, const char *path,
+ uint64_t *size, char sha[65], char *err, size_t err_len);
+
+/* Downloads file path of repo at rev to out_path, resuming a partial file
+ (its bytes hashed first), in pieces, each tried again if the connection
+ breaks; the SHA-256 of the whole written to sha. size: the file's, from
+ janas_hf_info. progress(done, size, arg) after each piece (may be NULL).
+ 0, or -1 with the reason in err. */
+int janas_hf_download(const char *repo, const char *rev, const char *path,
+ const char *out_path, uint64_t size, char sha[65],
+ void (*progress)(uint64_t done, uint64_t size, void *arg),
+ void *arg, char *err, size_t err_len);
+
/* The bf16 tensors of repo's checkpoint (one model.safetensors, or shards
listed by model.safetensors.index.json) whose names start with prefix,
written to out_path as one safetensors file. The number of tensors, or
diff --git a/src/common/sha256.c b/src/common/sha256.c
new file mode 100644
index 0000000..b4a5e59
--- /dev/null
+++ b/src/common/sha256.c
@@ -0,0 +1,213 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later
+ Copyright (C) 2026 Maurizio Cammalleri */
+/*
+ * sha256.c - SHA-256 (see sha256.h): the portable compression function of
+ * FIPS 180-4, and one on the x86 SHA extensions (sha256rnds2, sha256msg1,
+ * sha256msg2) chosen at run time, several times faster.
+ */
+#include "sha256.h"
+
+#include <fcntl.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+static const uint32_t K[64] = {
+ 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1,
+ 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
+ 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786,
+ 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
+ 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147,
+ 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
+ 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b,
+ 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
+ 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a,
+ 0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
+ 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2};
+
+static inline uint32_t ror(uint32_t x, int n)
+{
+ return (x >> n) | (x << (32 - n));
+}
+
+static void blocks_c(uint32_t *h, const uint8_t *p, size_t nb)
+{
+ for (; nb; nb--, p += 64) {
+ uint32_t w[64];
+ for (int i = 0; i < 16; i++)
+ w[i] = (uint32_t)p[4 * i] << 24 | (uint32_t)p[4 * i + 1] << 16 |
+ (uint32_t)p[4 * i + 2] << 8 | p[4 * i + 3];
+ for (int i = 16; i < 64; i++) {
+ uint32_t s0 =
+ ror(w[i - 15], 7) ^ ror(w[i - 15], 18) ^ (w[i - 15] >> 3);
+ uint32_t s1 =
+ ror(w[i - 2], 17) ^ ror(w[i - 2], 19) ^ (w[i - 2] >> 10);
+ w[i] = w[i - 16] + s0 + w[i - 7] + s1;
+ }
+ uint32_t a = h[0], b = h[1], c = h[2], d = h[3], e = h[4], f = h[5],
+ g = h[6], k = h[7];
+ for (int i = 0; i < 64; i++) {
+ uint32_t t1 = k + (ror(e, 6) ^ ror(e, 11) ^ ror(e, 25)) +
+ ((e & f) ^ (~e & g)) + K[i] + w[i];
+ uint32_t t2 = (ror(a, 2) ^ ror(a, 13) ^ ror(a, 22)) +
+ ((a & b) ^ (a & c) ^ (b & c));
+ k = g;
+ g = f;
+ f = e;
+ e = d + t1;
+ d = c;
+ c = b;
+ b = a;
+ a = t1 + t2;
+ }
+ h[0] += a;
+ h[1] += b;
+ h[2] += c;
+ h[3] += d;
+ h[4] += e;
+ h[5] += f;
+ h[6] += g;
+ h[7] += k;
+ }
+}
+
+#if defined(__x86_64__)
+#include <immintrin.h>
+
+/* The x86 SHA extensions: the state as (ABEF, CDGH), four rounds per
+ sha256rnds2 pair, the schedule by sha256msg1 and sha256msg2. */
+__attribute__((target("sha,sse4.1"))) static void
+blocks_ni(uint32_t *h, const uint8_t *p, size_t nb)
+{
+ const __m128i bswap =
+ _mm_set_epi64x(0x0c0d0e0f08090a0bll, 0x0405060700010203ll);
+ __m128i tmp = _mm_loadu_si128((const __m128i *)h);
+ __m128i st1 = _mm_loadu_si128((const __m128i *)(h + 4));
+ tmp = _mm_shuffle_epi32(tmp, 0xb1); /* CDAB */
+ st1 = _mm_shuffle_epi32(st1, 0x1b); /* EFGH */
+ __m128i st0 = _mm_alignr_epi8(tmp, st1, 8); /* ABEF */
+ st1 = _mm_blend_epi16(st1, tmp, 0xf0); /* CDGH */
+ for (; nb; nb--, p += 64) {
+ __m128i a0 = st0, c0 = st1, msg, m[4];
+ for (int i = 0; i < 4; i++)
+ m[i] = _mm_shuffle_epi8(
+ _mm_loadu_si128((const __m128i *)(p + 16 * i)), bswap);
+ for (int r = 0; r < 16; r++) {
+ __m128i w = m[r & 3];
+ if (r >= 4) {
+ /* the next four words of the schedule, into m[r & 3] */
+ __m128i t = _mm_alignr_epi8(m[(r - 1) & 3], m[(r - 2) & 3], 4);
+ w = _mm_add_epi32(
+ _mm_sha256msg1_epu32(m[r & 3], m[(r + 1) & 3]), t);
+ w = _mm_sha256msg2_epu32(w, m[(r - 1) & 3]);
+ m[r & 3] = w;
+ }
+ msg =
+ _mm_add_epi32(w, _mm_loadu_si128((const __m128i *)(K + 4 * r)));
+ st1 = _mm_sha256rnds2_epu32(st1, st0, msg);
+ msg = _mm_shuffle_epi32(msg, 0x0e);
+ st0 = _mm_sha256rnds2_epu32(st0, st1, msg);
+ }
+ st0 = _mm_add_epi32(st0, a0);
+ st1 = _mm_add_epi32(st1, c0);
+ }
+ tmp = _mm_shuffle_epi32(st0, 0x1b); /* FEBA */
+ st1 = _mm_shuffle_epi32(st1, 0xb1); /* DCHG */
+ st0 = _mm_blend_epi16(tmp, st1, 0xf0); /* DCBA */
+ st1 = _mm_alignr_epi8(st1, tmp, 8); /* ABEF -> HGFE */
+ _mm_storeu_si128((__m128i *)h, st0);
+ _mm_storeu_si128((__m128i *)(h + 4), st1);
+}
+#endif
+
+static void blocks(uint32_t *h, const uint8_t *p, size_t nb)
+{
+#if defined(__x86_64__)
+ static int ni = -1;
+ if (ni < 0)
+ ni = __builtin_cpu_supports("sha") && __builtin_cpu_supports("sse4.1");
+ if (ni && !getenv("JANAS_SHA256_C")) {
+ blocks_ni(h, p, nb);
+ return;
+ }
+#endif
+ blocks_c(h, p, nb);
+}
+
+void janas_sha256_init(struct janas_sha256 *s)
+{
+ static const uint32_t h0[8] = {0x6a09e667, 0xbb67ae85, 0x3c6ef372,
+ 0xa54ff53a, 0x510e527f, 0x9b05688c,
+ 0x1f83d9ab, 0x5be0cd19};
+ memcpy(s->h, h0, sizeof(h0));
+ s->len = 0;
+ s->n = 0;
+}
+
+void janas_sha256_update(struct janas_sha256 *s, const void *data, size_t n)
+{
+ const uint8_t *p = data;
+ s->len += n;
+ if (s->n) {
+ size_t k = 64 - s->n < n ? 64 - s->n : n;
+ memcpy(s->buf + s->n, p, k);
+ s->n += k;
+ p += k;
+ n -= k;
+ if (s->n < 64)
+ return;
+ blocks(s->h, s->buf, 1);
+ s->n = 0;
+ }
+ if (n >= 64) {
+ blocks(s->h, p, n / 64);
+ p += n / 64 * 64;
+ n %= 64;
+ }
+ memcpy(s->buf, p, n);
+ s->n = n;
+}
+
+void janas_sha256_hex(struct janas_sha256 *s, char out[65])
+{
+ uint64_t bits = s->len * 8;
+ uint8_t pad[72] = {0x80};
+ size_t k = (s->n < 56 ? 56 : 120) - s->n;
+ for (int i = 0; i < 8; i++)
+ pad[k + i] = (uint8_t)(bits >> (56 - 8 * i));
+ uint64_t len = s->len;
+ janas_sha256_update(s, pad, k + 8);
+ s->len = len;
+ static const char hex[] = "0123456789abcdef";
+ for (int i = 0; i < 8; i++)
+ for (int j = 0; j < 4; j++) {
+ uint8_t b = (uint8_t)(s->h[i] >> (24 - 8 * j));
+ out[8 * i + 2 * j] = hex[b >> 4];
+ out[8 * i + 2 * j + 1] = hex[b & 15];
+ }
+ out[64] = 0;
+}
+
+int janas_sha256_file(const char *path, char out[65])
+{
+ int fd = open(path, O_RDONLY | O_CLOEXEC);
+ if (fd < 0)
+ return -1;
+ size_t cap = (size_t)8 << 20;
+ uint8_t *buf = malloc(cap);
+ if (!buf) {
+ close(fd);
+ return -1;
+ }
+ struct janas_sha256 s;
+ janas_sha256_init(&s);
+ ssize_t r;
+ while ((r = read(fd, buf, cap)) > 0)
+ janas_sha256_update(&s, buf, (size_t)r);
+ free(buf);
+ close(fd);
+ if (r < 0)
+ return -1;
+ janas_sha256_hex(&s, out);
+ return 0;
+}
diff --git a/src/common/sha256.h b/src/common/sha256.h
new file mode 100644
index 0000000..b51459f
--- /dev/null
+++ b/src/common/sha256.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later
+ Copyright (C) 2026 Maurizio Cammalleri */
+/*
+ * sha256.h - SHA-256 (FIPS 180-4), to check the files the tools download
+ * and write against the fingerprints of MODELS.md. With the x86 SHA
+ * extensions where the CPU has them: a converted model is up to 48 GB.
+ */
+#ifndef JANAS_COMMON_SHA256_H
+#define JANAS_COMMON_SHA256_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+struct janas_sha256 {
+ uint32_t h[8];
+ uint64_t len; /* bytes hashed */
+ uint8_t buf[64];
+ size_t n; /* bytes waiting in buf */
+};
+
+void janas_sha256_init(struct janas_sha256 *s);
+void janas_sha256_update(struct janas_sha256 *s, const void *data, size_t n);
+/* The digest as 64 lowercase hex digits and a NUL. */
+void janas_sha256_hex(struct janas_sha256 *s, char out[65]);
+/* The SHA-256 of a whole file, as hex; 0, or -1 if it cannot be read. */
+int janas_sha256_file(const char *path, char out[65]);
+
+#endif
diff --git a/src/get/catalog.c b/src/get/catalog.c
new file mode 100644
index 0000000..66a103e
--- /dev/null
+++ b/src/get/catalog.c
@@ -0,0 +1,226 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later
+ Copyright (C) 2026 Maurizio Cammalleri */
+/*
+ * catalog.c - the models janas-get knows (see catalog.h). Every SHA-256
+ * here is one of MODELS.md; a model added here is added there too, and to
+ * janas-try.sh's levels through this tool.
+ */
+#include <string.h>
+
+#include "catalog.h"
+
+const struct get_model get_catalog[] = {
+ {.name = "qwen3-4b",
+ .what = "Qwen3-4B, dense - the quick start",
+ .repo = "Qwen/Qwen3-4B-GGUF",
+ .parts = {"Qwen3-4B-Q4_K_M.gguf"},
+ .part_sha =
+ {"7485fe6f11af29433bc51cab58009521f205840f5b4ae3a32fa7f92e8534fdf5"},
+ .gguf_gb = 2.5,
+ .out = "qwen3-4b-q4km.jns",
+ .flat_sha =
+ "e8ed44bdd3c612ce3ec1204a4b9db671115efab4a2ada384c70cf6492d430291",
+ .jns_sha =
+ "1cbd8ebfdf14aee05777f5278e655382f040cdedc21d282e93806fc8cf279fc5",
+ .ram_gb = 8},
+ {.name = "qwen3.5-0.8b",
+ .what = "Qwen3.5-0.8B, dense",
+ .repo = "unsloth/Qwen3.5-0.8B-GGUF",
+ .parts = {"Qwen3.5-0.8B-Q4_K_M.gguf"},
+ .part_sha =
+ {"bd258782e35f7f458f8aced1adc053e6e92e89bc735ba3be89d38a06121dc517"},
+ .gguf_gb = 0.53,
+ .out = "qwen3.5-0.8b-q4km.jns",
+ .flat_sha =
+ "75a3b62c4a10cad87a7ce64bf89485cfcc564fa949f2c5af8c7051615dc26124",
+ .ram_gb = 2},
+ {.name = "qwen3.5-2b",
+ .what = "Qwen3.5-2B, dense, with its MTP block",
+ .repo = "unsloth/Qwen3.5-2B-GGUF",
+ .parts = {"Qwen3.5-2B-Q4_K_M.gguf"},
+ .part_sha =
+ {"aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223"},
+ .gguf_gb = 1.28,
+ .out = "qwen3.5-2b-q4km.jns",
+ .flat_sha =
+ "286118ea3ad1903890a593582e22bed82859bf7c38b0ff9ed043050e5bd572a6",
+ .extra = EXTRA_HF2JNS,
+ .extra_src = "hf:Qwen/Qwen3.5-2B@15852e8c16360a2fea060d615a32b45270f8a8fc",
+ .extra_out = "qwen3.5-2b-mtp-q4k.jns",
+ .extra_sha =
+ "19c2d7eba1800acb10a06d7b38709ec92776e852cb2f25032ddd8977d17f2b05",
+ .ram_gb = 4},
+ {.name = "qwen3.5-9b",
+ .what = "Qwen3.5-9B, dense, with its MTP block",
+ .repo = "unsloth/Qwen3.5-9B-GGUF",
+ .parts = {"Qwen3.5-9B-Q4_K_M.gguf"},
+ .part_sha =
+ {"03b74727a860a56338e042c4420bb3f04b2fec5734175f4cb9fa853daf52b7e8"},
+ .gguf_gb = 5.68,
+ .out = "qwen3.5-9b-q4km.jns",
+ .flat_sha =
+ "e40ec6c58ef2390c7a4ce8c9b7b6b384ed4e55c9980f646d1c98e83e2c018115",
+ .extra = EXTRA_HF2JNS,
+ .extra_src = "hf:Qwen/Qwen3.5-9B@c202236235762e1c871ad0ccb60c8ee5ba337b9a",
+ .extra_out = "qwen3.5-9b-mtp-q4k.jns",
+ .extra_sha =
+ "67f31f57dbc78ca8c02ef5e3aa137c5df3836a3decda20b6ea057d004089c361",
+ .ram_gb = 8},
+ {.name = "qwen3-30b-a3b",
+ .what = "Qwen3-30B-A3B, mixture of experts",
+ .repo = "Qwen/Qwen3-30B-A3B-GGUF",
+ .parts = {"Qwen3-30B-A3B-Q4_K_M.gguf"},
+ .part_sha =
+ {"0d003f6662faee786ed5da3e31b29c978de5ae5d275c8794c606a7f3c01aa8f5"},
+ .gguf_gb = 18.6,
+ .out = "qwen3-30b-a3b-q4km.jns",
+ .flat_sha =
+ "6906de51f2923b3be3a83eb040eaaeb090cba3389e9217648abe08bd678376e9",
+ .jns_sha =
+ "3fa0185b7eecd43e222d420f68aae28a86b3b90392fd7e4052b5cd337318e895",
+ .ram_gb = 16},
+ {.name = "qwen3.6-35b-a3b",
+ .what = "Qwen3.6-35B-A3B, mixture of experts, MTP inside",
+ .repo = "bartowski/Qwen_Qwen3.6-35B-A3B-GGUF",
+ .parts = {"Qwen_Qwen3.6-35B-A3B-Q4_K_M.gguf"},
+ .part_sha =
+ {"b46fedd33e0bfb0cae308aa3c158d0a4b2c4a1d2185a1ed6f093cdaf39064772"},
+ .gguf_gb = 22.3,
+ .out = "qwen3.6-35b-a3b-q4km.jns",
+ .flat_sha =
+ "31772037300f332d712f4ef8dd5e7251c7cd76cf6b9dab33b089394b7c079fb2",
+ .jns_sha =
+ "8feaeb66d93593b9876b564ca3878f2a9252e46b8d4e2d06caa4e19d43dc6464",
+ .ram_gb = 16},
+ {.name = "qwen3-next-80b-a3b",
+ .what = "Qwen3-Next-80B-A3B-Instruct, with its MTP block",
+ .repo = "Qwen/Qwen3-Next-80B-A3B-Instruct-GGUF",
+ .parts = {"Qwen3-Next-80B-A3B-Instruct-Q4_K_M.gguf"},
+ .part_sha =
+ {"d103b2733ec1012a52d01edda66b7e5c24ae50508c9f99f5297ea459ef3c061a"},
+ .gguf_gb = 48.4,
+ .out = "qwen3-next-80b-a3b-q4km.jns",
+ .flat_sha =
+ "fa9a50dd910de4e78064ecff17aef43286694277eea3a89a57e9921ca2bfd2da",
+ .jns_sha =
+ "fa3255c73106391e9cb5a97efd30f77a928fb4518d40060f795a8cff6656daed",
+ .extra = EXTRA_HF2JNS,
+ .extra_src = "hf:Qwen/Qwen3-Next-80B-A3B-Instruct",
+ .extra_out = "qwen3-next-80b-a3b-mtp-q4k.jns",
+ .extra_sha =
+ "5c9bc4cb6f739193fac5e4daa30c922f73255acf41443ecfc0123ef084584559",
+ .ram_gb = 32},
+ {.name = "qwen3-coder-next",
+ .what = "Qwen3-Coder-Next, mixture of experts, for code",
+ .repo = "Qwen/Qwen3-Coder-Next-GGUF",
+ .parts =
+ {"Qwen3-Coder-Next-Q4_K_M/Qwen3-Coder-Next-Q4_K_M-00001-of-00004.gguf",
+ "Qwen3-Coder-Next-Q4_K_M/Qwen3-Coder-Next-Q4_K_M-00002-of-00004.gguf",
+ "Qwen3-Coder-Next-Q4_K_M/Qwen3-Coder-Next-Q4_K_M-00003-of-00004.gguf",
+ "Qwen3-Coder-Next-Q4_K_M/Qwen3-Coder-Next-Q4_K_M-00004-of-00004.gguf"},
+ .part_sha =
+ {"6bcfc9f9c37901eeb92172e2ab871224dab36a453d263bcb2547f737409534da",
+ "817def0691ee9d08bf3dc4444be7aed29c9e52091e8fa9d97901ce7e7f6f01d3",
+ "23aa634d47dca9b4ca3ea249384e6f01951b24c83cdc076f37f6f43d6c99883f",
+ "249c768cc5f130dc731567d6edcbdacc48e14dec9e02c5dbe2b2185d2c5bdb2b"},
+ .gguf_gb = 48.4,
+ .out = "qwen3-coder-next-q4km.jns",
+ .flat_sha =
+ "5912d964247002473eff1811058b4ffadf7b9bb47ae336c486cdf2964de68382",
+ .jns_sha =
+ "0276b0697a05075fec8ab076a757205537804ff41dd7e9dadace7c6993763563",
+ .ram_gb = 32},
+ {.name = "qwen3-embedding-0.6b",
+ .what = "Qwen3-Embedding-0.6B, for janas-server --embedding-model",
+ .repo = "Qwen/Qwen3-Embedding-0.6B-GGUF",
+ .parts = {"Qwen3-Embedding-0.6B-Q8_0.gguf"},
+ .part_sha =
+ {"06507c7b42688469c4e7298b0a1e16deff06caf291cf0a5b278c308249c3e439"},
+ .gguf_gb = 0.64,
+ .out = "qwen3-embedding-0.6b-q8.jns",
+ .flat_sha =
+ "1e8eb8cbeba7a1a657354d259c5730bb79f6087160a9ca8da901bd08bdbe943f",
+ .ram_gb = 2},
+ {.name = "gemma-4-e2b",
+ .what = "Gemma-4-E2B-it, with its assistant",
+ .repo = "unsloth/gemma-4-E2B-it-GGUF",
+ .parts = {"gemma-4-E2B-it-Q4_K_M.gguf"},
+ .part_sha =
+ {"740185b21d22ceb83a11c3aa62ad5842ef32c70f6096d756bbee85a1e4ec34b8"},
+ .gguf_gb = 3.1,
+ .out = "gemma-4-e2b-it-q4km.jns",
+ .flat_sha =
+ "1f18e08878df41f377d563b38048d41841c7973ddea845cd9672fcf23cbd864b",
+ .extra = EXTRA_GGUF,
+ .extra_src = "mtp-gemma-4-E2B-it.gguf",
+ .extra_src_sha =
+ "9eba819938efccfd6044f8af84e3bbfddc639a2bcf32ebc36420e6a649191919",
+ .extra_out = "gemma-4-e2b-it-mtp.jns",
+ .extra_sha =
+ "79df1f3c817542b3867a0940b1a5b7330d478bddb4102ec90ff817334877cd62",
+ .ram_gb = 4},
+ {.name = "gemma-4-e4b",
+ .what = "Gemma-4-E4B-it, with its assistant",
+ .repo = "unsloth/gemma-4-E4B-it-GGUF",
+ .parts = {"gemma-4-E4B-it-Q4_K_M.gguf"},
+ .part_sha =
+ {"85a896a047553e842f25297ee5b031d64ff30147d9c4af17b1e4b394cd1fab87"},
+ .gguf_gb = 5.0,
+ .out = "gemma-4-e4b-it-q4km.jns",
+ .flat_sha =
+ "7ec828e9bdc1ada2eda907e93b87072f3180f2e8cba5ccedcd360d1a53858675",
+ .extra = EXTRA_GGUF,
+ .extra_src = "mtp-gemma-4-E4B-it.gguf",
+ .extra_src_sha =
+ "b6a723115efa510d3b3215db1e26790dae84cd08c2134a764f3d194f1f0c3376",
+ .extra_out = "gemma-4-e4b-it-mtp.jns",
+ .extra_sha =
+ "7c6fdb74ba1eb11f9d47b15097a09cce0142aea380a41f1986ba7b8ec4785b53",
+ .ram_gb = 8},
+ {.name = "gemma-4-12b",
+ .what = "Gemma-4-12B-it, dense, with its assistant",
+ .repo = "unsloth/gemma-4-12b-it-GGUF",
+ .parts = {"gemma-4-12b-it-Q4_K_M.gguf"},
+ .part_sha =
+ {"0a270ec9fe6b34f4a0d33992b6135117b484ebc4766ab76b51d4ae8c457e4c42"},
+ .gguf_gb = 7.1,
+ .out = "gemma-4-12b-it-q4km.jns",
+ .flat_sha =
+ "cae09e04598891dea6e9cd93fb496fbde08d822ac35de4ff45dbfab1ea25c4dd",
+ .extra = EXTRA_GGUF,
+ .extra_src = "mtp-gemma-4-12b-it.gguf",
+ .extra_src_sha =
+ "145db9094bc0f85f1701e255a2ed216dcc9800fc8bc8631ad00905b456bd451b",
+ .extra_out = "gemma-4-12b-it-mtp.jns",
+ .extra_sha =
+ "35e640f20dd74e862df9c1d949ad4edca9f58beef0cb29e99cccc6c99297826f",
+ .ram_gb = 16},
+ {.name = "gemma-4-26b-a4b",
+ .what = "Gemma-4-26B-A4B-it, mixture of experts, with its assistant",
+ .repo = "unsloth/gemma-4-26B-A4B-it-GGUF",
+ .parts = {"gemma-4-26B-A4B-it-UD-Q4_K_M.gguf"},
+ .part_sha =
+ {"f2c28b3dc4776931ac6f879e11f203dec637ea0f14267a86ec8f6165f63f293f"},
+ .gguf_gb = 16.9,
+ .out = "gemma-4-26b-a4b-it-udq4km.jns",
+ .flat_sha =
+ "35c3845bbc21de52bf630d3c7455b4bb5379c34487b5925e020ec2e573931666",
+ .extra = EXTRA_GGUF,
+ .extra_src = "mtp-gemma-4-26B-A4B-it.gguf",
+ .extra_src_sha =
+ "6326fb9f5e487aa8dcdd313a091e3c67724cb2a666ec3b7d2895b5b26d93ed1b",
+ .extra_out = "gemma-4-26b-it-mtp.jns",
+ .extra_sha =
+ "6ba24ee91fe6fe904b29919acbb654a66d9caa0aba67db1df9f921a76e5bcedf",
+ .ram_gb = 16},
+};
+
+const int get_catalog_n = (int)(sizeof(get_catalog) / sizeof(get_catalog[0]));
+
+const struct get_model *get_find(const char *name)
+{
+ for (int i = 0; i < get_catalog_n; i++)
+ if (strcmp(get_catalog[i].name, name) == 0)
+ return &get_catalog[i];
+ return NULL;
+}
diff --git a/src/get/catalog.h b/src/get/catalog.h
new file mode 100644
index 0000000..8de7fc6
--- /dev/null
+++ b/src/get/catalog.h
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later
+ Copyright (C) 2026 Maurizio Cammalleri */
+/*
+ * catalog.h - the models janas-get knows: where each comes from, and the
+ * SHA-256 of every file on the way, as MODELS.md lists them.
+ */
+#ifndef JANAS_GET_CATALOG_H
+#define JANAS_GET_CATALOG_H
+
+#define GET_MAX_PARTS 4
+
+/* how the model's multi-token prediction file is made, if it has one */
+enum get_extra {
+ EXTRA_NONE,
+ EXTRA_GGUF, /* a GGUF of its own (Gemma 4's assistant), gguf2jns */
+ EXTRA_HF2JNS /* the block of the original checkpoint, hf2jns_mtp hf: */
+};
+
+struct get_model {
+ const char *name; /* what janas-get takes */
+ const char *what; /* a line for the list */
+ const char *repo; /* the GGUF's repository */
+ const char *parts[GET_MAX_PARTS]; /* its file(s), the first given to
+ gguf2jns */
+ const char *part_sha[GET_MAX_PARTS];
+ double gguf_gb; /* download */
+ const char *out; /* the .jns, beside the downloads */
+ const char *flat_sha; /* what gguf2jns writes (the .jns itself when
+ there are no planes) */
+ const char *jns_sha; /* what jns_planes writes, NULL: one step */
+ enum get_extra extra;
+ const char *extra_src; /* GGUF file of repo, or hf:repo[@rev] */
+ const char *extra_src_sha; /* of that GGUF (EXTRA_GGUF) */
+ const char *extra_out;
+ const char *extra_sha;
+ int ram_gb; /* the smallest machine it is meant for */
+};
+
+extern const struct get_model get_catalog[];
+extern const int get_catalog_n;
+
+const struct get_model *get_find(const char *name);
+
+#endif
diff --git a/src/get/main.c b/src/get/main.c
new file mode 100644
index 0000000..e1c98de
--- /dev/null
+++ b/src/get/main.c
@@ -0,0 +1,240 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later
+ Copyright (C) 2026 Maurizio Cammalleri */
+/*
+ * janas-get - a model ready for Janas in one command: downloaded from
+ * Hugging Face, checked, converted, its prediction file made beside it,
+ * every file compared with the fingerprints of MODELS.md.
+ *
+ * janas-get list
+ * janas-get <model> [--dir DIR] [--keep] [--no-mtp]
+ * janas-get hf:<owner>/<repo>/<file.gguf> [--dir DIR] [--keep]
+ *
+ * A download stopped halfway resumes where it stopped; a file already
+ * there is checked, not fetched again. The GGUF goes once converted,
+ * unless --keep.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/statvfs.h>
+
+#include "catalog.h"
+#include "common/sysinfo.h"
+#include "steps.h"
+
+static void usage(void)
+{
+ fprintf(stderr,
+ "usage: janas-get list\n"
+ " janas-get <model> [--dir DIR] [--keep] [--no-mtp]\n"
+ " janas-get hf:<owner>/<repo>/<file.gguf> [--dir DIR] "
+ "[--keep]\n"
+ " --dir DIR where the models go (default: models)\n"
+ " --keep keep the downloaded GGUF once converted\n"
+ " --no-mtp without the model's prediction file\n");
+}
+
+static void list(void)
+{
+ printf("%-22s %8s %7s %s\n", "model", "download", "memory", "");
+ for (int i = 0; i < get_catalog_n; i++) {
+ const struct get_model *m = &get_catalog[i];
+ printf("%-22s %6.1f GB %4d GB %s\n", m->name, m->gguf_gb, m->ram_gb,
+ m->what);
+ }
+ printf("\nFingerprints and licences: MODELS.md. Any other GGUF: "
+ "janas-get hf:<owner>/<repo>/<file.gguf>\n");
+}
+
+static int exists(const char *path)
+{
+ struct stat st;
+ return stat(path, &st) == 0;
+}
+
+/* a converter's output compared with MODELS.md; 0 when it matches */
+static int check(const char *file, const char *want, const char *who)
+{
+ int r = get_same(file, want);
+ if (r == 1) {
+ printf(" %s: fingerprint as in MODELS.md\n", who);
+ return 0;
+ }
+ fprintf(stderr,
+ "janas-get: %s wrote %s, and its SHA-256 is not the one of "
+ "MODELS.md: please report it (an issue on GitHub)\n",
+ who, file);
+ return -1;
+}
+
+/* gguf2jns from the first part into out */
+static int convert(const char *first, const char *out)
+{
+ char *argv[] = {"gguf2jns", (char *)first, (char *)out, NULL};
+ printf("== converting (gguf2jns)\n");
+ fflush(stdout);
+ return get_run("gguf2jns", argv) == 0 ? 0 : -1;
+}
+
+static int from_catalog(const struct get_model *m, const char *dir, int keep,
+ int no_mtp)
+{
+ char out[4096], first[4096], flat[4200];
+ snprintf(out, sizeof(out), "%s/%s", dir, m->out);
+ get_local(first, sizeof(first), dir, m->parts[0]);
+ if (m->ram_gb &&
+ janas_mem_total() < (uint64_t)m->ram_gb * (1ull << 30) * 85 / 100)
+ printf("note: %s is meant for machines with %d GB of memory or "
+ "more; this one has %.1f GB\n",
+ m->name, m->ram_gb, (double)janas_mem_total() / (1ull << 30));
+ int have = exists(out) &&
+ get_same(out, m->jns_sha ? m->jns_sha : m->flat_sha) == 1;
+ if (have) {
+ printf("%s: already here, fingerprint checked\n", out);
+ } else {
+ /* room: two copies at the most - the GGUF and what gguf2jns
+ writes, then (the GGUF gone) that and what jns_planes writes */
+ struct statvfs vf;
+ double need = m->gguf_gb * 2.0 * 1e9;
+ if (statvfs(dir, &vf) == 0 &&
+ (double)vf.f_bavail * (double)vf.f_frsize < need) {
+ fprintf(stderr,
+ "janas-get: %s needs about %.0f GB free in %s while it "
+ "works; there are %.0f\n",
+ m->name, need / 1e9, dir,
+ (double)vf.f_bavail * (double)vf.f_frsize / 1e9);
+ return 1;
+ }
+ printf("== %s: downloading\n", m->name);
+ for (int p = 0; p < GET_MAX_PARTS && m->parts[p]; p++)
+ if (get_fetch(m->repo, m->parts[p], dir, m->part_sha[p]) != 0)
+ return 1;
+ snprintf(flat, sizeof(flat), "%s.flat", out);
+ const char *conv = m->jns_sha ? flat : out;
+ if (convert(first, conv) != 0 ||
+ check(conv, m->flat_sha, "gguf2jns") != 0)
+ return 1;
+ if (!keep)
+ for (int p = 0; p < GET_MAX_PARTS && m->parts[p]; p++) {
+ char part[4096];
+ get_local(part, sizeof(part), dir, m->parts[p]);
+ remove(part);
+ }
+ if (m->jns_sha) {
+ char *argv[] = {"jns_planes", flat, out, NULL};
+ printf("== cutting the experts into bit planes (jns_planes)\n");
+ fflush(stdout);
+ if (get_run("jns_planes", argv) != 0 ||
+ check(out, m->jns_sha, "jns_planes") != 0)
+ return 1;
+ remove(flat);
+ }
+ }
+ if (m->extra != EXTRA_NONE && !no_mtp) {
+ char xout[4096];
+ snprintf(xout, sizeof(xout), "%s/%s", dir, m->extra_out);
+ if (exists(xout) && get_same(xout, m->extra_sha) == 1) {
+ printf("%s: already here, fingerprint checked\n", xout);
+ } else if (m->extra == EXTRA_GGUF) {
+ char src[4096];
+ printf("== its assistant\n");
+ if (get_fetch(m->repo, m->extra_src, dir, m->extra_src_sha) != 0)
+ return 1;
+ get_local(src, sizeof(src), dir, m->extra_src);
+ if (convert(src, xout) != 0 ||
+ check(xout, m->extra_sha, "gguf2jns (assistant)") != 0)
+ return 1;
+ if (!keep)
+ remove(src);
+ } else {
+ char *argv[] = {"hf2jns_mtp", out, (char *)m->extra_src, xout,
+ NULL};
+ printf("== its MTP block, from %s (hf2jns_mtp)\n", m->extra_src);
+ fflush(stdout);
+ if (get_run("hf2jns_mtp", argv) != 0 ||
+ check(xout, m->extra_sha, "hf2jns_mtp") != 0)
+ return 1;
+ }
+ }
+ printf("\nReady: %s\nChat with it: janas-chat %s\n", out, out);
+ return 0;
+}
+
+/* hf:<owner>/<repo>/<path>: any GGUF, checked against the SHA-256 Hugging
+ Face lists, converted in one step */
+static int from_hf(const char *spec, const char *dir, int keep)
+{
+ char repo[512];
+ const char *s = spec + 3, *a = strchr(s, '/'),
+ *b = a ? strchr(a + 1, '/') : NULL;
+ size_t n = b ? (size_t)(b - s) : 0;
+ if (!b || n >= sizeof(repo) || !b[1]) {
+ usage();
+ return 2;
+ }
+ memcpy(repo, s, n);
+ repo[n] = 0;
+ const char *path = b + 1;
+ char local[4096], out[4200];
+ get_local(local, sizeof(local), dir, path);
+ snprintf(out, sizeof(out), "%s", local);
+ char *dot = strrchr(out, '.');
+ if (dot && strcmp(dot, ".gguf") == 0)
+ *dot = 0;
+ for (char *c = strrchr(out, '/') ? strrchr(out, '/') + 1 : out; *c; c++)
+ if (*c >= 'A' && *c <= 'Z')
+ *c = (char)(*c - 'A' + 'a');
+ strcat(out, ".jns");
+ printf("== %s: downloading (not in the catalog: checked against the "
+ "SHA-256 Hugging Face lists)\n",
+ path);
+ if (get_fetch(repo, path, dir, NULL) != 0 || convert(local, out) != 0)
+ return 1;
+ if (!keep)
+ remove(local);
+ printf("\nReady: %s (no fingerprint to compare: not in MODELS.md)\n"
+ "Chat with it: janas-chat %s\n",
+ out, out);
+ return 0;
+}
+
+int main(int argc, char **argv)
+{
+ if (argc < 2) {
+ usage();
+ return 2;
+ }
+ const char *dir = "models";
+ int keep = 0, no_mtp = 0;
+ for (int i = 2; i < argc; i++) {
+ if (strcmp(argv[i], "--dir") == 0 && i + 1 < argc)
+ dir = argv[++i];
+ else if (strcmp(argv[i], "--keep") == 0)
+ keep = 1;
+ else if (strcmp(argv[i], "--no-mtp") == 0)
+ no_mtp = 1;
+ else {
+ usage();
+ return 2;
+ }
+ }
+ if (strcmp(argv[1], "list") == 0) {
+ list();
+ return 0;
+ }
+ if (strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "--help") == 0) {
+ usage();
+ return 0;
+ }
+ mkdir(dir, 0755);
+ if (strncmp(argv[1], "hf:", 3) == 0)
+ return from_hf(argv[1], dir, keep);
+ const struct get_model *m = get_find(argv[1]);
+ if (!m) {
+ fprintf(stderr, "janas-get: no model called %s (janas-get list)\n",
+ argv[1]);
+ return 2;
+ }
+ return from_catalog(m, dir, keep, no_mtp);
+}
diff --git a/src/get/steps.c b/src/get/steps.c
new file mode 100644
index 0000000..cb68297
--- /dev/null
+++ b/src/get/steps.c
@@ -0,0 +1,129 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later
+ Copyright (C) 2026 Maurizio Cammalleri */
+/*
+ * steps.c - janas-get's steps (see steps.h). The converters are the
+ * programs of the same build, run as they are: what they write is what
+ * MODELS.md fingerprints, whoever runs them.
+ */
+#define _GNU_SOURCE
+#include "steps.h"
+
+#include <spawn.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/wait.h>
+#include <time.h>
+#include <unistd.h>
+
+#include "common/hf_fetch.h"
+#include "common/sha256.h"
+
+extern char **environ;
+
+void get_local(char *out, size_t len, const char *dir, const char *path)
+{
+ const char *base = strrchr(path, '/');
+ snprintf(out, len, "%s/%s", dir, base ? base + 1 : path);
+}
+
+static double now(void)
+{
+ struct timespec t;
+ clock_gettime(CLOCK_MONOTONIC, &t);
+ return (double)t.tv_sec + 1e-9 * (double)t.tv_nsec;
+}
+
+struct bar {
+ double t0;
+ uint64_t from;
+};
+
+static void progress(uint64_t done, uint64_t size, void *arg)
+{
+ struct bar *b = arg;
+ double dt = now() - b->t0;
+ double mbs = dt > 0 ? (double)(done - b->from) / dt / 1e6 : 0;
+ fprintf(stderr, "\r %.2f of %.2f GB (%.0f%%), %.0f MB/s ",
+ (double)done / 1e9, (double)size / 1e9,
+ size ? 100.0 * (double)done / (double)size : 100.0, mbs);
+ if (done >= size)
+ fputc('\n', stderr);
+}
+
+int get_same(const char *file, const char *want)
+{
+ char got[65];
+ if (janas_sha256_file(file, got) != 0)
+ return -1;
+ return strcmp(got, want) == 0;
+}
+
+int get_fetch(const char *repo, const char *path, const char *dir,
+ const char *want)
+{
+ char local[4096], err[512], listed[65], got[65];
+ get_local(local, sizeof(local), dir, path);
+ uint64_t size = 0;
+ if (janas_hf_info(repo, NULL, path, &size, listed, err, sizeof(err)) != 0) {
+ fprintf(stderr, "janas-get: %s\n", err);
+ return -1;
+ }
+ /* the repository may have been changed since MODELS.md was written:
+ then its file is not the one the fingerprints are of */
+ if (want && listed[0] && strcmp(listed, want) != 0) {
+ fprintf(stderr,
+ "janas-get: %s has changed on Hugging Face since MODELS.md "
+ "was written (SHA-256 %s, not %s); not downloaded\n",
+ path, listed, want);
+ return -1;
+ }
+ printf(" %s from %s, %.2f GB\n", path, repo, (double)size / 1e9);
+ fflush(stdout);
+ struct bar b = {.t0 = now()};
+ FILE *f = fopen(local, "rb");
+ if (f) {
+ fseek(f, 0, SEEK_END);
+ long at = ftell(f);
+ b.from = at > 0 ? (uint64_t)at : 0;
+ fclose(f);
+ }
+ if (janas_hf_download(repo, NULL, path, local, size, got, progress, &b, err,
+ sizeof(err)) != 0) {
+ fprintf(stderr, "\njanas-get: %s (run again to resume)\n", err);
+ return -1;
+ }
+ const char *expect = want ? want : listed;
+ if (expect[0] && strcmp(got, expect) != 0) {
+ fprintf(stderr,
+ "janas-get: %s is not the file it should be (SHA-256 %s, "
+ "not %s); deleted\n",
+ local, got, expect);
+ remove(local);
+ return -1;
+ }
+ printf(" SHA-256 checked%s\n", want ? " against MODELS.md" : "");
+ return 0;
+}
+
+int get_run(const char *tool, char *const argv[])
+{
+ char self[4096], path[4200];
+ ssize_t n = readlink("/proc/self/exe", self, sizeof(self) - 1);
+ if (n <= 0)
+ return -1;
+ self[n] = 0;
+ char *slash = strrchr(self, '/');
+ if (slash)
+ *slash = 0;
+ snprintf(path, sizeof(path), "%s/%s", self, tool);
+ pid_t pid;
+ if (posix_spawn(&pid, path, NULL, NULL, argv, environ) != 0) {
+ fprintf(stderr, "janas-get: cannot run %s\n", path);
+ return -1;
+ }
+ int st;
+ if (waitpid(pid, &st, 0) != pid)
+ return -1;
+ return WIFEXITED(st) ? WEXITSTATUS(st) : -1;
+}
diff --git a/src/get/steps.h b/src/get/steps.h
new file mode 100644
index 0000000..792bad4
--- /dev/null
+++ b/src/get/steps.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later
+ Copyright (C) 2026 Maurizio Cammalleri */
+/*
+ * steps.h - what janas-get does to a file: download it and check it, run a
+ * converter beside janas-get, check what it wrote.
+ */
+#ifndef JANAS_GET_STEPS_H
+#define JANAS_GET_STEPS_H
+
+#include <stddef.h>
+
+/* dir/the last part of path */
+void get_local(char *out, size_t len, const char *dir, const char *path);
+
+/* File path of repo downloaded into dir (resumed if part of it is there),
+ its SHA-256 checked against want (NULL: the one Hugging Face lists). 0,
+ or -1 having said why. */
+int get_fetch(const char *repo, const char *path, const char *dir,
+ const char *want);
+
+/* 1 when file's SHA-256 is want; 0 when it is another, -1 unreadable. */
+int get_same(const char *file, const char *want);
+
+/* A converter of the same build (gguf2jns, jns_planes, hf2jns_mtp) run on
+ argv (argv[0] its name, NULL-terminated). Its exit status, or -1. */
+int get_run(const char *tool, char *const argv[]);
+
+#endif
diff --git a/tests/test_get_catalog.c b/tests/test_get_catalog.c
new file mode 100644
index 0000000..69e98cf
--- /dev/null
+++ b/tests/test_get_catalog.c
@@ -0,0 +1,83 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later
+ Copyright (C) 2026 Maurizio Cammalleri */
+/*
+ * test_get_catalog.c - janas-get's catalog against MODELS.md: every
+ * SHA-256 in it 64 hex digits and written in MODELS.md, every name unique.
+ * A digest copied by hand one digit short would otherwise turn good
+ * downloads away (it happened while writing the catalog).
+ */
+#include <ctype.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "../src/get/catalog.c"
+
+static int failures;
+
+static char *slurp(const char *path)
+{
+ FILE *f = fopen(path, "rb");
+ if (!f)
+ return NULL;
+ fseek(f, 0, SEEK_END);
+ long n = ftell(f);
+ fseek(f, 0, SEEK_SET);
+ char *p = malloc((size_t)n + 1);
+ if (p && fread(p, 1, (size_t)n, f) != (size_t)n) {
+ free(p);
+ p = NULL;
+ }
+ if (p)
+ p[n] = 0;
+ fclose(f);
+ return p;
+}
+
+static void digest(const char *models, const char *who, const char *d)
+{
+ if (!d)
+ return;
+ int hex = strlen(d) == 64;
+ for (const char *c = d; hex && *c; c++)
+ hex = isxdigit((unsigned char)*c) && !isupper((unsigned char)*c);
+ if (!hex) {
+ printf("FAIL %s: %s is not 64 hex digits\n", who, d);
+ failures++;
+ } else if (!strstr(models, d)) {
+ printf("FAIL %s: %s is not in MODELS.md\n", who, d);
+ failures++;
+ }
+}
+
+int main(void)
+{
+ char *models = slurp("MODELS.md");
+ if (!models) {
+ printf("test_get_catalog: MODELS.md not found (run from the "
+ "repository's root)\n");
+ return 1;
+ }
+ for (int i = 0; i < get_catalog_n; i++) {
+ const struct get_model *m = &get_catalog[i];
+ for (int p = 0; p < GET_MAX_PARTS && m->parts[p]; p++)
+ digest(models, m->name, m->part_sha[p]);
+ digest(models, m->name, m->flat_sha);
+ digest(models, m->name, m->jns_sha);
+ digest(models, m->name, m->extra_src_sha);
+ digest(models, m->name, m->extra_sha);
+ if (!m->flat_sha || (m->extra != EXTRA_NONE && !m->extra_sha)) {
+ printf("FAIL %s: a fingerprint missing\n", m->name);
+ failures++;
+ }
+ for (int k = 0; k < i; k++)
+ if (strcmp(get_catalog[k].name, m->name) == 0) {
+ printf("FAIL %s: twice in the catalog\n", m->name);
+ failures++;
+ }
+ }
+ free(models);
+ printf("test_get_catalog: %s (%d models)\n", failures ? "FAILED" : "ok",
+ get_catalog_n);
+ return failures ? 1 : 0;
+}
diff --git a/tests/test_sha256.c b/tests/test_sha256.c
new file mode 100644
index 0000000..184bb8e
--- /dev/null
+++ b/tests/test_sha256.c
@@ -0,0 +1,78 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later
+ Copyright (C) 2026 Maurizio Cammalleri */
+/*
+ * test_sha256.c - SHA-256 against the vectors of FIPS 180-4 and, over
+ * lengths across the block boundaries fed in uneven pieces, the portable
+ * compression function against the one on the SHA extensions (the same
+ * digest by both, JANAS_SHA256_C=1 forcing the portable one).
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "common/sha256.h"
+
+static int failures;
+
+static void hex_of(const void *p, size_t n, size_t piece, char out[65])
+{
+ struct janas_sha256 s;
+ janas_sha256_init(&s);
+ const unsigned char *c = p;
+ for (size_t at = 0; at < n; at += piece)
+ janas_sha256_update(&s, c + at, n - at < piece ? n - at : piece);
+ janas_sha256_hex(&s, out);
+}
+
+static void check(const char *what, const void *p, size_t n, const char *want)
+{
+ char got[65];
+ hex_of(p, n, n ? n : 1, got);
+ if (strcmp(got, want) != 0) {
+ printf("FAIL %s: %s, want %s\n", what, got, want);
+ failures++;
+ }
+}
+
+int main(void)
+{
+ check("empty", "", 0,
+ "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
+ check("abc", "abc", 3,
+ "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");
+ const char *two =
+ "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq";
+ check("two blocks", two, strlen(two),
+ "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1");
+ size_t n = 1000000;
+ char *a = malloc(n);
+ if (!a)
+ return 1;
+ memset(a, 'a', n);
+ check("a million a", a, n,
+ "cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0");
+ /* uneven pieces: the digest does not depend on how the data comes */
+ char whole[65], pieces[65];
+ unsigned x = 12345;
+ for (size_t i = 0; i < n; i++)
+ a[i] = (char)((x = x * 1103515245u + 12345u) >> 16);
+ for (size_t len = 0; len < 300; len += 7) {
+ hex_of(a, len, len ? len : 1, whole);
+ hex_of(a, len, 13, pieces);
+ if (strcmp(whole, pieces) != 0) {
+ printf("FAIL length %zu: whole %s, in pieces %s\n", len, whole,
+ pieces);
+ failures++;
+ }
+ }
+ hex_of(a, n, n, whole);
+ hex_of(a, n, 4099, pieces);
+ if (strcmp(whole, pieces) != 0) {
+ printf("FAIL a million bytes in pieces\n");
+ failures++;
+ }
+ free(a);
+ printf("test_sha256: %s%s\n", failures ? "FAILED" : "ok",
+ getenv("JANAS_SHA256_C") ? " (portable)" : "");
+ return failures ? 1 : 0;
+}
diff --git a/tools/janas-try.sh b/tools/janas-try.sh
index 401bfd9..5866680 100755
--- a/tools/janas-try.sh
+++ b/tools/janas-try.sh
@@ -101,41 +101,27 @@ ask()
gib() { awk -v b="$1" 'BEGIN { printf "%.1f", b / 1073741824 }'; }
# ---------------------------------------------------------------- models
-# level: repository, file, GGUF sha256, converted (flat) sha256, with bit
-# planes sha256 - the fingerprints of MODELS.md; the conversion is
-# deterministic, so a different sum is a defect found
+# level: the model's name for janas-get (which downloads, checks and
+# converts it, with the fingerprints of MODELS.md), its repository and
+# GGUF (for the report), the file that runs and its prediction file ("-":
+# none, or inside it), the tools that made it, then download, disk while
+# converting and the memory of the smallest machine, in bytes and GB
model_of()
{
case "$1" in
- quick) echo "Qwen/Qwen3-4B-GGUF Qwen3-4B-Q4_K_M.gguf \
-7485fe6f11af29433bc51cab58009521f205840f5b4ae3a32fa7f92e8534fdf5 \
-e8ed44bdd3c612ce3ec1204a4b9db671115efab4a2ada384c70cf6492d430291 \
-1cbd8ebfdf14aee05777f5278e655382f040cdedc21d282e93806fc8cf279fc5 \
-qwen3-4b-q4km 2684354560 5368709120 8" ;;
- medium) echo "bartowski/Qwen_Qwen3.6-35B-A3B-GGUF Qwen_Qwen3.6-35B-A3B-Q4_K_M.gguf \
-b46fedd33e0bfb0cae308aa3c158d0a4b2c4a1d2185a1ed6f093cdaf39064772 \
-31772037300f332d712f4ef8dd5e7251c7cd76cf6b9dab33b089394b7c079fb2 \
-8feaeb66d93593b9876b564ca3878f2a9252e46b8d4e2d06caa4e19d43dc6464 \
-qwen3.6-35b-a3b-q4km 23622320128 48318382080 16" ;;
- full) echo "Qwen/Qwen3-Next-80B-A3B-Instruct-GGUF Qwen3-Next-80B-A3B-Instruct-Q4_K_M.gguf \
-d103b2733ec1012a52d01edda66b7e5c24ae50508c9f99f5297ea459ef3c061a \
-fa9a50dd910de4e78064ecff17aef43286694277eea3a89a57e9921ca2bfd2da \
-fa3255c73106391e9cb5a97efd30f77a928fb4518d40060f795a8cff6656daed \
-qwen3-next-80b-a3b-q4km 55834574848 107374182400 16" ;;
- gemma) echo "unsloth/gemma-4-E4B-it-GGUF gemma-4-E4B-it-Q4_K_M.gguf \
-85a896a047553e842f25297ee5b031d64ff30147d9c4af17b1e4b394cd1fab87 \
-7ec828e9bdc1ada2eda907e93b87072f3180f2e8cba5ccedcd360d1a53858675 \
-- gemma-4-e4b-it-q4km 5476083302 11811160064 8" ;;
+ quick) echo "qwen3-4b Qwen/Qwen3-4B-GGUF Qwen3-4B-Q4_K_M.gguf \
+qwen3-4b-q4km.jns - planes 2684354560 5368709120 8" ;;
+ medium) echo "qwen3.6-35b-a3b bartowski/Qwen_Qwen3.6-35B-A3B-GGUF \
+Qwen_Qwen3.6-35B-A3B-Q4_K_M.gguf qwen3.6-35b-a3b-q4km.jns - planes \
+23622320128 48318382080 16" ;;
+ full) echo "qwen3-next-80b-a3b Qwen/Qwen3-Next-80B-A3B-Instruct-GGUF \
+Qwen3-Next-80B-A3B-Instruct-Q4_K_M.gguf qwen3-next-80b-a3b-q4km.jns \
+qwen3-next-80b-a3b-mtp-q4k.jns planes 55834574848 107374182400 16" ;;
+ gemma) echo "gemma-4-e4b unsloth/gemma-4-E4B-it-GGUF \
+gemma-4-E4B-it-Q4_K_M.gguf gemma-4-e4b-it-q4km.jns gemma-4-e4b-it-mtp.jns \
+one 5476083302 11811160064 8" ;;
esac
}
-# Gemma 4's assistant (its MTP file), converted by gguf2jns as the model
-GEMMA_MTP_FILE=mtp-gemma-4-E4B-it.gguf
-GEMMA_MTP_SUM=b6a723115efa510d3b3215db1e26790dae84cd08c2134a764f3d194f1f0c3376
-GEMMA_MTP_OUT_SUM=7c6fdb74ba1eb11f9d47b15097a09cce0142aea380a41f1986ba7b8ec4785b53
-MTP_REPO=Qwen/Qwen3-Next-80B-A3B-Instruct
-MTP_FILE=model-00041-of-00041.safetensors
-MTP_SUM=1f3ac4d828f7e08dd14eb4dc6f282139ae1fa0d894e43f247da2539d2ef43826
-MTP_OUT_SUM=5c9bc4cb6f739193fac5e4daa30c922f73255acf41443ecfc0123ef084584559
# The answer to a fixed prompt at temperature 0, cut at 64 tokens, as a
# fingerprint of its text: the kernels give the same numbers bit for bit on
@@ -262,43 +248,12 @@ build()
return 1
}
-# ---------------------------------------------------------------- download
-download()
-{ # repository, file, sha256 -> $MODELS/file
- local repo=$1 file=$2 sum=$3 dst=$MODELS/$2
- if [ -f "$dst" ] && [ -f "$dst.ok" ]; then
- note "$file: already here, checked"
- return 0
- fi
- grep -qx "$dst" "$WORK/created" || echo "$dst" >> "$WORK/created"
- note "$file: downloading (it resumes if interrupted)"
- curl -fL -C - --retry 5 --retry-delay 5 -o "$dst" \
- "https://huggingface.co/$repo/resolve/main/$file" || {
- fail "the download of $file failed; run again to resume"
- return 1
- }
- note "$file: checking its SHA-256"
- if [ "$(sha256sum "$dst" | cut -d' ' -f1)" != "$sum" ]; then
- fail "$file is not the file it should be (SHA-256 differs); deleted"
- rm -f "$dst"
- return 1
- fi
- touch "$dst.ok"
-}
-
-fingerprint_ok()
-{ # file, sha256: 0 when it matches; the result goes to the report
- local got
- got=$(sha256sum "$1" | cut -d' ' -f1)
- [ "$got" = "$2" ]
-}
-
# ---------------------------------------------------------------- a level
level()
{
local lv=$1
- read -r repo gguf gsum fsum psum name dl disk ram <<<"$(model_of "$lv")"
- local flat=$MODELS/$name-flat.jns jns=$MODELS/$name.jns converted=0
+ read -r getname repo gguf jnsn mtpn steps dl disk ram <<<"$(model_of "$lv")"
+ local jns=$MODELS/$jnsn converted=0
say "Level $lv: ${gguf%.gguf}"
machine
# a machine sold with 16 GB shows less (the kernel and an integrated GPU
@@ -318,67 +273,45 @@ level()
echo "$lv declined" >> "$WORK/results"
return 0
}
- printf '%s\n' "$flat" "$jns" >> "$WORK/created"
- download "$repo" "$gguf" "$gsum" || return 1
- # one step where there are no planes to cut (psum "-"): the file
- # gguf2jns writes is the one that runs
- local out=$flat
- [ "$psum" = "-" ] && out=$jns
- note "converting (gguf2jns)"
- "$BIN/gguf2jns" "$MODELS/$gguf" "$out" >>"$LOG" 2>&1 || {
- fail "gguf2jns failed: $LOG"; echo "$lv conversion FAILED" >> "$WORK/results"; return 1; }
- fingerprint_ok "$out" "$fsum" && echo "$lv gguf2jns fingerprint ok" >> "$WORK/results" ||
- echo "$lv gguf2jns fingerprint DIFFERS" >> "$WORK/results"
- if [ "$KEEP" = 0 ] && ask "Delete the downloaded GGUF ($(gib "$(stat -c %s "$MODELS/$gguf")") GiB)? It is not needed any more"; then
- rm -f "$MODELS/$gguf" "$MODELS/$gguf.ok"
- fi
- if [ "$psum" != "-" ]; then
- note "cutting the experts into bit planes (jns_planes)"
- "$BIN/jns_planes" "$flat" "$jns" >>"$LOG" 2>&1 || {
- fail "jns_planes failed: $LOG"; echo "$lv planes FAILED" >> "$WORK/results"; return 1; }
- fingerprint_ok "$jns" "$psum" && echo "$lv jns_planes fingerprint ok" >> "$WORK/results" ||
- echo "$lv jns_planes fingerprint DIFFERS" >> "$WORK/results"
- rm -f "$flat"
- fi
converted=1
fi
- local mtp=()
- if [ "$lv" = full ]; then
- local mjns=$MODELS/$name-mtp.jns
- if ! [ -f "$mjns" ]; then
- echo "$mjns" >> "$WORK/created"
- download "$MTP_REPO" "$MTP_FILE" "$MTP_SUM" || return 1
- "$BIN/hf2jns_mtp" "$jns" "$MODELS/$MTP_FILE" "$mjns" >>"$LOG" 2>&1 || {
- fail "hf2jns_mtp failed: $LOG"; echo "$lv mtp FAILED" >> "$WORK/results"; return 1; }
- fingerprint_ok "$mjns" "$MTP_OUT_SUM" && echo "$lv hf2jns_mtp fingerprint ok" >> "$WORK/results" ||
- echo "$lv hf2jns_mtp fingerprint DIFFERS" >> "$WORK/results"
- [ "$KEEP" = 1 ] || rm -f "$MODELS/$MTP_FILE" "$MODELS/$MTP_FILE.ok"
- fi
- mtp=(--mtp "$mjns")
- fi
- if [ "$lv" = gemma ]; then
- local mjns=$MODELS/$name-mtp.jns
- if ! [ -f "$mjns" ]; then
- echo "$mjns" >> "$WORK/created"
- download "${repo}" "$GEMMA_MTP_FILE" "$GEMMA_MTP_SUM" || return 1
- "$BIN/gguf2jns" "$MODELS/$GEMMA_MTP_FILE" "$mjns" >>"$LOG" 2>&1 || {
- fail "gguf2jns failed on the assistant: $LOG"; echo "$lv assistant FAILED" >> "$WORK/results"; return 1; }
- fingerprint_ok "$mjns" "$GEMMA_MTP_OUT_SUM" && echo "$lv assistant fingerprint ok" >> "$WORK/results" ||
- echo "$lv assistant fingerprint DIFFERS" >> "$WORK/results"
- [ "$KEEP" = 1 ] || rm -f "$MODELS/$GEMMA_MTP_FILE" "$MODELS/$GEMMA_MTP_FILE.ok"
- fi
- mtp=(--mtp "$mjns")
+ # janas-get downloads, checks the SHA-256 of every file against
+ # MODELS.md, converts, and makes the prediction file beside the model;
+ # what it says goes to the screen and, for the report, to a file
+ [ -f "$jns" ] || echo "$jns" >> "$WORK/created"
+ [ "$mtpn" = - ] || [ -f "$MODELS/$mtpn" ] || echo "$MODELS/$mtpn" >> "$WORK/created"
+ local keep=()
+ [ "$KEEP" = 1 ] && keep=(--keep)
+ note "janas-get $getname: the GGUF goes once converted${keep:+ (kept: --keep)}"
+ "$BIN/janas-get" "$getname" --dir "$MODELS" "${keep[@]}" 2>&1 | tee "$WORK/get-$lv.txt"
+ local rc=${PIPESTATUS[0]} got=$WORK/get-$lv.txt
+ local t
+ for t in "gguf2jns" "jns_planes" "gguf2jns (assistant)" "hf2jns_mtp"; do
+ local key=${t%% *}
+ [ "$t" = "gguf2jns (assistant)" ] && key=assistant
+ grep -qF " $t: fingerprint as in MODELS.md" "$got" &&
+ echo "$lv $key fingerprint ok" >> "$WORK/results"
+ grep -qF "$t wrote" "$got" &&
+ echo "$lv $key fingerprint DIFFERS" >> "$WORK/results"
+ done
+ if [ "$rc" != 0 ] || ! [ -f "$jns" ]; then
+ fail "janas-get failed: $got"
+ echo "$lv janas-get FAILED: $(grep -m1 'janas-get:' "$got")" >> "$WORK/results"
+ return 1
fi
+ cat "$got" >> "$LOG"
+ local mtp=()
+ [ "$mtpn" = - ] || mtp=(--mtp "$MODELS/$mtpn")
# what ran: Janas's own file, and where it came from
- local shown=${gguf%%-Q4_K_M.gguf} how="the SHA-256 of both checked against MODELS.md"
+ local shown=${gguf%%-Q4_K_M.gguf} how="the SHA-256 of every file checked against MODELS.md by janas-get"
shown=${shown#Qwen_}
- [ "$converted" = 1 ] || how="converted in an earlier run of this script"
+ [ "$converted" = 1 ] || how="converted in an earlier run, checked again by janas-get"
local tools="gguf2jns and jns_planes"
- [ "$psum" = "-" ] && tools="gguf2jns"
- local line="- Model ($lv): $shown, run as \`$(basename "$jns")\` - Janas's format (JNS), converted here by $tools from \`$gguf\` of \`$repo\`; $how"
- [ "$lv" = gemma ] && line="$line; with its assistant \`$(basename "$MODELS/$name-mtp.jns")\`, converted by gguf2jns from \`$GEMMA_MTP_FILE\` of \`$repo\`"
- [ "$lv" = full ] && line="$line; with the MTP block \`$(basename "$MODELS/$name-mtp.jns")\`, converted by hf2jns_mtp from \`$MTP_FILE\` of \`$MTP_REPO\`"
+ [ "$steps" = one ] && tools="gguf2jns"
+ local line="- Model ($lv): $shown, run as \`$jnsn\` - Janas's format (JNS), converted here by $tools from \`$gguf\` of \`$repo\`; $how"
+ [ "$lv" = gemma ] && line="$line; with its assistant \`$mtpn\`, converted by gguf2jns"
+ [ "$lv" = full ] && line="$line; with the MTP block \`$mtpn\`, made by hf2jns_mtp from the original checkpoint"
grep -qF "$line" "$WORK/models" 2>/dev/null || echo "$line" >> "$WORK/models"
# the answers: the same text as on every other machine