37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
Release 2026-09-27: janas-get, a model in one command
camauri4534903c229e
diff --git a/ChangeLog.md b/ChangeLog.mdindex 7eeee51..8bcdfc9 100644--- a/ChangeLog.md+++ b/ChangeLog.md@@ -2,6 +2,25 @@> Curated, user-facing summary of completed work, newest first.+## [2026-09-27] - janas-get: a model in one command++- **`janas-get`** downloads a model of its catalog (the thirteen of+ MODELS.md: Qwen3, Qwen3.5, Qwen3.6, Qwen3-Next, Coder-Next, the+ embedding model, Gemma 4) from Hugging Face, checks the SHA-256 of+ every file against MODELS.md, runs the converters of the same build+ (gguf2jns, jns_planes) and makes the prediction file beside the model -+ hf2jns_mtp over range requests, or Gemma 4's assistant - with its+ fingerprint too. Downloads resume; files already there are checked, not+ fetched again; the GGUF goes as soon as it is converted, so the disk+ holds two copies at the most. `janas-get hf:<owner>/<repo>/<file>` takes+ any GGUF, checked against the SHA-256 Hugging Face lists.+- Its own HTTPS download in pieces (the client of hf2jns_mtp) and its own+ SHA-256, on the x86 SHA extensions where the CPU has them (1.4 GB/s on+ the development laptop): no curl, no sha256sum.+- **`janas-try.sh` gets its models through it**: the catalog and the+ fingerprints are in one place. A test checks the catalog against+ MODELS.md, digit by digit.+## [2026-09-27] - janas-try tries Gemma- **`janas-try.sh --level gemma`**: Gemma-4-E4B and its assistant,diff --git a/README.md b/README.mdindex 4cf0ce1..1dfa4a1 100644--- a/README.md+++ b/README.md@@ -213,6 +213,25 @@ sudo pacman -S vulkan-headers glslang vulkan-tools # the same, Arch## Getting a model+**The short way: `janas-get`.** One command downloads a model from+Hugging Face, checks the SHA-256 of every file against [MODELS.md](MODELS.md),+converts it and makes its prediction file beside it (an MTP block, or Gemma+4's assistant), which the chat then finds by itself:++```sh+bin/x86_64-linux/janas-get list # the models it knows+bin/x86_64-linux/janas-get gemma-4-e4b # into models/+bin/x86_64-linux/janas-chat models/gemma-4-e4b-it-q4km.jns+```++A download stopped halfway resumes where it stopped, a file already there+is checked rather than fetched again, and the GGUF goes once converted+(`--keep` keeps it). Any other GGUF: `janas-get+hf:<owner>/<repo>/<file.gguf>`, checked against the SHA-256 Hugging Face+lists for it. The memory column of `janas-get list` is the smallest+machine each model is meant for. The rest of this section is the same by+hand.+Janas reads its own format, `.jns`, converted from a **Q4_K_M** GGUF (Q3_K,IQ3_S, IQ4_NL, IQ4_XS, Q4_0, Q4_1 and Q5_1 convert too, and so do unsloth's"UD" files made of them; IQ2 and IQ1 types, not yet). Keep bothdiff --git a/src/common/hf_download.c b/src/common/hf_download.cnew file mode 100644index 0000000..0ee15a1--- /dev/null+++ b/src/common/hf_download.c@@ -0,0 +1,186 @@+/* SPDX-License-Identifier: GPL-3.0-or-later+ Copyright (C) 2026 Maurizio Cammalleri */+/*+ * hf_download.c - whole files of a Hugging Face repository, for janas-get+ * (see hf_fetch.h): the size and SHA-256 Hugging Face lists for a file,+ * and a download in pieces that resumes and hashes as it goes.+ *+ * Pieces rather than one long transfer: each is a request of its own on+ * the address the redirect gave, so a broken connection costs one piece,+ * and the signed address of the CDN, if it expires in a long download, is+ * asked for again.+ */+#define _GNU_SOURCE+#include <fcntl.h>+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <sys/stat.h>+#include <unistd.h>++#include "common/hf_fetch.h"+#include "common/sha256.h"++#define PIECE ((int64_t)128 << 20)+#define TRIES 6++int janas_hf_info(const char *repo, const char *rev, const char *path,+ uint64_t *size, char sha[65], char *err, size_t err_len)+{+ /* the listing of the file's directory */+ char url[1536], dir[1024] = "";+ const char *slash = strrchr(path, '/');+ if (slash)+ snprintf(dir, sizeof(dir), "/%.*s", (int)(slash - path), path);+ int k = snprintf(url, sizeof(url),+ "https://huggingface.co/api/models/%s/tree/%s%s", repo,+ rev ? rev : "main", dir);+ if (k < 0 || (size_t)k >= sizeof(url)) {+ snprintf(err, err_len, "%s: a name too long", path);+ return -1;+ }+ struct janas_buf b = {0};+ if (janas_hf_get_url(url, NULL, NULL, &b, NULL, 0, err, err_len) != 0) {+ janas_buf_free(&b);+ return -1;+ }+ struct janas_json_doc *d = janas_json_parse(b.p, b.n, err, err_len);+ janas_buf_free(&b);+ if (!d)+ return -1;+ int found = -1;+ const struct janas_json *root = janas_json_root(d);+ for (const struct janas_json *e =+ root && root->type == JANAS_JSON_ARRAY ? root->child : NULL;+ e; e = e->next) {+ if (!janas_json_is(janas_json_get(e, "path"), path))+ continue;+ *size = (uint64_t)janas_json_num(janas_json_get(e, "size"), 0);+ const char *oid =+ janas_json_str(janas_json_get(janas_json_get(e, "lfs"), "oid"));+ snprintf(sha, 65, "%s", oid && strlen(oid) == 64 ? oid : "");+ found = 0;+ break;+ }+ janas_json_free(d);+ if (found != 0)+ snprintf(err, err_len, "%s: not in %s", path, repo);+ return found;+}++/* The address the file answers at once redirected */+static int resolve(const char *repo, const char *rev, const char *path,+ char *final, size_t len, char *err, size_t err_len)+{+ char url[1536];+ int k =+ snprintf(url, sizeof(url), "https://huggingface.co/%s/resolve/%s/%s",+ repo, rev ? rev : "main", path);+ if (k < 0 || (size_t)k >= sizeof(url)) {+ snprintf(err, err_len, "%s: a name too long", path);+ return -1;+ }+ struct janas_buf b = {0};+ int64_t a = 0, z = 1;+ int r = janas_hf_get_url(url, &a, &z, &b, final, len, err, err_len);+ janas_buf_free(&b);+ return r;+}++int janas_hf_download(const char *repo, const char *rev, const char *path,+ const char *out_path, uint64_t size, char sha[65],+ void (*progress)(uint64_t, uint64_t, void *), void *arg,+ char *err, size_t err_len)+{+ int fd = open(out_path, O_RDWR | O_CREAT | O_CLOEXEC, 0644);+ if (fd < 0) {+ snprintf(err, err_len, "cannot write %s", out_path);+ return -1;+ }+ struct stat st;+ uint64_t done = fstat(fd, &st) == 0 ? (uint64_t)st.st_size : 0;+ if (done > size) { /* not this file: start again */+ if (ftruncate(fd, 0) != 0) {+ close(fd);+ snprintf(err, err_len, "cannot truncate %s", out_path);+ return -1;+ }+ done = 0;+ }+ struct janas_sha256 h;+ janas_sha256_init(&h);+ struct janas_buf b = {0};+ /* what is already there: hashed as it is, then continued */+ if (done) {+ size_t cap = (size_t)8 << 20;+ char *buf = malloc(cap);+ uint64_t at = 0;+ while (buf && at < done) {+ size_t want = done - at < cap ? (size_t)(done - at) : cap;+ ssize_t r = pread(fd, buf, want, (off_t)at);+ if (r <= 0)+ break;+ janas_sha256_update(&h, buf, (size_t)r);+ at += (uint64_t)r;+ }+ free(buf);+ if (at != done) {+ close(fd);+ snprintf(err, err_len, "cannot read %s", out_path);+ return -1;+ }+ if (progress)+ progress(done, size, arg);+ }+ char *final = malloc(8192);+ int ok = final != NULL, tries = 0;+ if (final)+ final[0] = 0;+ while (ok && done < size) {+ if (!final[0] &&+ resolve(repo, rev, path, final, 8192, err, err_len) != 0) {+ if (++tries >= TRIES)+ ok = 0;+ continue;+ }+ int64_t a = (int64_t)done, z = (int64_t)(size - done) < PIECE+ ? (int64_t)size+ : (int64_t)done + PIECE;+ b.n = 0;+ if (janas_hf_get_url(final, &a, &z, &b, NULL, 0, err, err_len) != 0 ||+ b.oom || (int64_t)b.n != z - a) {+ final[0] = 0; /* ask for the address again */+ if (++tries >= TRIES)+ ok = 0;+ continue;+ }+ tries = 0;+ size_t w = 0;+ while (w < b.n) {+ ssize_t r = pwrite(fd, b.p + w, b.n - w, (off_t)(done + w));+ if (r <= 0) {+ snprintf(err, err_len, "cannot write %s (disk full?)",+ out_path);+ ok = 0;+ break;+ }+ w += (size_t)r;+ }+ if (!ok)+ break;+ janas_sha256_update(&h, b.p, b.n);+ done += b.n;+ if (progress)+ progress(done, size, arg);+ }+ free(final);+ janas_buf_free(&b);+ if (close(fd) != 0 && ok) {+ snprintf(err, err_len, "cannot write %s", out_path);+ ok = 0;+ }+ if (!ok)+ return -1;+ janas_sha256_hex(&h, sha);+ return 0;+}diff --git a/src/common/hf_fetch.c b/src/common/hf_fetch.cindex bf9e78c..b832105 100644--- a/src/common/hf_fetch.c+++ b/src/common/hf_fetch.c@@ -116,6 +116,13 @@ static int get_url(const char *url, const int64_t *from, const int64_t *to,return -1;}+int janas_hf_get_url(const char *url, const int64_t *from, const int64_t *to,+ struct janas_buf *out, char *final, size_t final_len,+ char *err, size_t err_len)+{+ return get_url(url, from, to, out, final, final_len, err, err_len);+}+int janas_hf_get(const char *repo, const char *rev, const char *path,const int64_t *from, const int64_t *to, struct janas_buf *out,char *err, size_t err_len)diff --git a/src/common/hf_fetch.h b/src/common/hf_fetch.hindex 4e66f38..bfd129c 100644--- a/src/common/hf_fetch.h+++ b/src/common/hf_fetch.h@@ -24,6 +24,28 @@ int janas_hf_get(const char *repo, const char *rev, const char *path,const int64_t *from, const int64_t *to, struct janas_buf *out,char *err, size_t err_len);+/* The same by address (a file of the CDN, as a redirect gave it): with+ final, the address the request ended at after the redirects. */+int janas_hf_get_url(const char *url, const int64_t *from, const int64_t *to,+ struct janas_buf *out, char *final, size_t final_len,+ char *err, size_t err_len);++/* A file of repo at rev (NULL: main): its size and, for a file kept by+ LFS (weights), the SHA-256 Hugging Face gives for it (sha empty+ otherwise). 0, or -1 with the reason in err. */+int janas_hf_info(const char *repo, const char *rev, const char *path,+ uint64_t *size, char sha[65], char *err, size_t err_len);++/* Downloads file path of repo at rev to out_path, resuming a partial file+ (its bytes hashed first), in pieces, each tried again if the connection+ breaks; the SHA-256 of the whole written to sha. size: the file's, from+ janas_hf_info. progress(done, size, arg) after each piece (may be NULL).+ 0, or -1 with the reason in err. */+int janas_hf_download(const char *repo, const char *rev, const char *path,+ const char *out_path, uint64_t size, char sha[65],+ void (*progress)(uint64_t done, uint64_t size, void *arg),+ void *arg, char *err, size_t err_len);+/* The bf16 tensors of repo's checkpoint (one model.safetensors, or shardslisted by model.safetensors.index.json) whose names start with prefix,written to out_path as one safetensors file. The number of tensors, ordiff --git a/src/common/sha256.c b/src/common/sha256.cnew file mode 100644index 0000000..b4a5e59--- /dev/null+++ b/src/common/sha256.c@@ -0,0 +1,213 @@+/* SPDX-License-Identifier: GPL-3.0-or-later+ Copyright (C) 2026 Maurizio Cammalleri */+/*+ * sha256.c - SHA-256 (see sha256.h): the portable compression function of+ * FIPS 180-4, and one on the x86 SHA extensions (sha256rnds2, sha256msg1,+ * sha256msg2) chosen at run time, several times faster.+ */+#include "sha256.h"++#include <fcntl.h>+#include <stdlib.h>+#include <string.h>+#include <unistd.h>++static const uint32_t K[64] = {+ 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1,+ 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,+ 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786,+ 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,+ 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147,+ 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,+ 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b,+ 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,+ 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a,+ 0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,+ 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2};++static inline uint32_t ror(uint32_t x, int n)+{+ return (x >> n) | (x << (32 - n));+}++static void blocks_c(uint32_t *h, const uint8_t *p, size_t nb)+{+ for (; nb; nb--, p += 64) {+ uint32_t w[64];+ for (int i = 0; i < 16; i++)+ w[i] = (uint32_t)p[4 * i] << 24 | (uint32_t)p[4 * i + 1] << 16 |+ (uint32_t)p[4 * i + 2] << 8 | p[4 * i + 3];+ for (int i = 16; i < 64; i++) {+ uint32_t s0 =+ ror(w[i - 15], 7) ^ ror(w[i - 15], 18) ^ (w[i - 15] >> 3);+ uint32_t s1 =+ ror(w[i - 2], 17) ^ ror(w[i - 2], 19) ^ (w[i - 2] >> 10);+ w[i] = w[i - 16] + s0 + w[i - 7] + s1;+ }+ uint32_t a = h[0], b = h[1], c = h[2], d = h[3], e = h[4], f = h[5],+ g = h[6], k = h[7];+ for (int i = 0; i < 64; i++) {+ uint32_t t1 = k + (ror(e, 6) ^ ror(e, 11) ^ ror(e, 25)) ++ ((e & f) ^ (~e & g)) + K[i] + w[i];+ uint32_t t2 = (ror(a, 2) ^ ror(a, 13) ^ ror(a, 22)) ++ ((a & b) ^ (a & c) ^ (b & c));+ k = g;+ g = f;+ f = e;+ e = d + t1;+ d = c;+ c = b;+ b = a;+ a = t1 + t2;+ }+ h[0] += a;+ h[1] += b;+ h[2] += c;+ h[3] += d;+ h[4] += e;+ h[5] += f;+ h[6] += g;+ h[7] += k;+ }+}++#if defined(__x86_64__)+#include <immintrin.h>++/* The x86 SHA extensions: the state as (ABEF, CDGH), four rounds per+ sha256rnds2 pair, the schedule by sha256msg1 and sha256msg2. */+__attribute__((target("sha,sse4.1"))) static void+blocks_ni(uint32_t *h, const uint8_t *p, size_t nb)+{+ const __m128i bswap =+ _mm_set_epi64x(0x0c0d0e0f08090a0bll, 0x0405060700010203ll);+ __m128i tmp = _mm_loadu_si128((const __m128i *)h);+ __m128i st1 = _mm_loadu_si128((const __m128i *)(h + 4));+ tmp = _mm_shuffle_epi32(tmp, 0xb1); /* CDAB */+ st1 = _mm_shuffle_epi32(st1, 0x1b); /* EFGH */+ __m128i st0 = _mm_alignr_epi8(tmp, st1, 8); /* ABEF */+ st1 = _mm_blend_epi16(st1, tmp, 0xf0); /* CDGH */+ for (; nb; nb--, p += 64) {+ __m128i a0 = st0, c0 = st1, msg, m[4];+ for (int i = 0; i < 4; i++)+ m[i] = _mm_shuffle_epi8(+ _mm_loadu_si128((const __m128i *)(p + 16 * i)), bswap);+ for (int r = 0; r < 16; r++) {+ __m128i w = m[r & 3];+ if (r >= 4) {+ /* the next four words of the schedule, into m[r & 3] */+ __m128i t = _mm_alignr_epi8(m[(r - 1) & 3], m[(r - 2) & 3], 4);+ w = _mm_add_epi32(+ _mm_sha256msg1_epu32(m[r & 3], m[(r + 1) & 3]), t);+ w = _mm_sha256msg2_epu32(w, m[(r - 1) & 3]);+ m[r & 3] = w;+ }+ msg =+ _mm_add_epi32(w, _mm_loadu_si128((const __m128i *)(K + 4 * r)));+ st1 = _mm_sha256rnds2_epu32(st1, st0, msg);+ msg = _mm_shuffle_epi32(msg, 0x0e);+ st0 = _mm_sha256rnds2_epu32(st0, st1, msg);+ }+ st0 = _mm_add_epi32(st0, a0);+ st1 = _mm_add_epi32(st1, c0);+ }+ tmp = _mm_shuffle_epi32(st0, 0x1b); /* FEBA */+ st1 = _mm_shuffle_epi32(st1, 0xb1); /* DCHG */+ st0 = _mm_blend_epi16(tmp, st1, 0xf0); /* DCBA */+ st1 = _mm_alignr_epi8(st1, tmp, 8); /* ABEF -> HGFE */+ _mm_storeu_si128((__m128i *)h, st0);+ _mm_storeu_si128((__m128i *)(h + 4), st1);+}+#endif++static void blocks(uint32_t *h, const uint8_t *p, size_t nb)+{+#if defined(__x86_64__)+ static int ni = -1;+ if (ni < 0)+ ni = __builtin_cpu_supports("sha") && __builtin_cpu_supports("sse4.1");+ if (ni && !getenv("JANAS_SHA256_C")) {+ blocks_ni(h, p, nb);+ return;+ }+#endif+ blocks_c(h, p, nb);+}++void janas_sha256_init(struct janas_sha256 *s)+{+ static const uint32_t h0[8] = {0x6a09e667, 0xbb67ae85, 0x3c6ef372,+ 0xa54ff53a, 0x510e527f, 0x9b05688c,+ 0x1f83d9ab, 0x5be0cd19};+ memcpy(s->h, h0, sizeof(h0));+ s->len = 0;+ s->n = 0;+}++void janas_sha256_update(struct janas_sha256 *s, const void *data, size_t n)+{+ const uint8_t *p = data;+ s->len += n;+ if (s->n) {+ size_t k = 64 - s->n < n ? 64 - s->n : n;+ memcpy(s->buf + s->n, p, k);+ s->n += k;+ p += k;+ n -= k;+ if (s->n < 64)+ return;+ blocks(s->h, s->buf, 1);+ s->n = 0;+ }+ if (n >= 64) {+ blocks(s->h, p, n / 64);+ p += n / 64 * 64;+ n %= 64;+ }+ memcpy(s->buf, p, n);+ s->n = n;+}++void janas_sha256_hex(struct janas_sha256 *s, char out[65])+{+ uint64_t bits = s->len * 8;+ uint8_t pad[72] = {0x80};+ size_t k = (s->n < 56 ? 56 : 120) - s->n;+ for (int i = 0; i < 8; i++)+ pad[k + i] = (uint8_t)(bits >> (56 - 8 * i));+ uint64_t len = s->len;+ janas_sha256_update(s, pad, k + 8);+ s->len = len;+ static const char hex[] = "0123456789abcdef";+ for (int i = 0; i < 8; i++)+ for (int j = 0; j < 4; j++) {+ uint8_t b = (uint8_t)(s->h[i] >> (24 - 8 * j));+ out[8 * i + 2 * j] = hex[b >> 4];+ out[8 * i + 2 * j + 1] = hex[b & 15];+ }+ out[64] = 0;+}++int janas_sha256_file(const char *path, char out[65])+{+ int fd = open(path, O_RDONLY | O_CLOEXEC);+ if (fd < 0)+ return -1;+ size_t cap = (size_t)8 << 20;+ uint8_t *buf = malloc(cap);+ if (!buf) {+ close(fd);+ return -1;+ }+ struct janas_sha256 s;+ janas_sha256_init(&s);+ ssize_t r;+ while ((r = read(fd, buf, cap)) > 0)+ janas_sha256_update(&s, buf, (size_t)r);+ free(buf);+ close(fd);+ if (r < 0)+ return -1;+ janas_sha256_hex(&s, out);+ return 0;+}diff --git a/src/common/sha256.h b/src/common/sha256.hnew file mode 100644index 0000000..b51459f--- /dev/null+++ b/src/common/sha256.h@@ -0,0 +1,28 @@+/* SPDX-License-Identifier: GPL-3.0-or-later+ Copyright (C) 2026 Maurizio Cammalleri */+/*+ * sha256.h - SHA-256 (FIPS 180-4), to check the files the tools download+ * and write against the fingerprints of MODELS.md. With the x86 SHA+ * extensions where the CPU has them: a converted model is up to 48 GB.+ */+#ifndef JANAS_COMMON_SHA256_H+#define JANAS_COMMON_SHA256_H++#include <stddef.h>+#include <stdint.h>++struct janas_sha256 {+ uint32_t h[8];+ uint64_t len; /* bytes hashed */+ uint8_t buf[64];+ size_t n; /* bytes waiting in buf */+};++void janas_sha256_init(struct janas_sha256 *s);+void janas_sha256_update(struct janas_sha256 *s, const void *data, size_t n);+/* The digest as 64 lowercase hex digits and a NUL. */+void janas_sha256_hex(struct janas_sha256 *s, char out[65]);+/* The SHA-256 of a whole file, as hex; 0, or -1 if it cannot be read. */+int janas_sha256_file(const char *path, char out[65]);++#endifdiff --git a/src/get/catalog.c b/src/get/catalog.cnew file mode 100644index 0000000..66a103e--- /dev/null+++ b/src/get/catalog.c@@ -0,0 +1,226 @@+/* SPDX-License-Identifier: GPL-3.0-or-later+ Copyright (C) 2026 Maurizio Cammalleri */+/*+ * catalog.c - the models janas-get knows (see catalog.h). Every SHA-256+ * here is one of MODELS.md; a model added here is added there too, and to+ * janas-try.sh's levels through this tool.+ */+#include <string.h>++#include "catalog.h"++const struct get_model get_catalog[] = {+ {.name = "qwen3-4b",+ .what = "Qwen3-4B, dense - the quick start",+ .repo = "Qwen/Qwen3-4B-GGUF",+ .parts = {"Qwen3-4B-Q4_K_M.gguf"},+ .part_sha =+ {"7485fe6f11af29433bc51cab58009521f205840f5b4ae3a32fa7f92e8534fdf5"},+ .gguf_gb = 2.5,+ .out = "qwen3-4b-q4km.jns",+ .flat_sha =+ "e8ed44bdd3c612ce3ec1204a4b9db671115efab4a2ada384c70cf6492d430291",+ .jns_sha =+ "1cbd8ebfdf14aee05777f5278e655382f040cdedc21d282e93806fc8cf279fc5",+ .ram_gb = 8},+ {.name = "qwen3.5-0.8b",+ .what = "Qwen3.5-0.8B, dense",+ .repo = "unsloth/Qwen3.5-0.8B-GGUF",+ .parts = {"Qwen3.5-0.8B-Q4_K_M.gguf"},+ .part_sha =+ {"bd258782e35f7f458f8aced1adc053e6e92e89bc735ba3be89d38a06121dc517"},+ .gguf_gb = 0.53,+ .out = "qwen3.5-0.8b-q4km.jns",+ .flat_sha =+ "75a3b62c4a10cad87a7ce64bf89485cfcc564fa949f2c5af8c7051615dc26124",+ .ram_gb = 2},+ {.name = "qwen3.5-2b",+ .what = "Qwen3.5-2B, dense, with its MTP block",+ .repo = "unsloth/Qwen3.5-2B-GGUF",+ .parts = {"Qwen3.5-2B-Q4_K_M.gguf"},+ .part_sha =+ {"aaf42c8b7c3cab2bf3d69c355048d4a0ee9973d48f16c731c0520ee914699223"},+ .gguf_gb = 1.28,+ .out = "qwen3.5-2b-q4km.jns",+ .flat_sha =+ "286118ea3ad1903890a593582e22bed82859bf7c38b0ff9ed043050e5bd572a6",+ .extra = EXTRA_HF2JNS,+ .extra_src = "hf:Qwen/Qwen3.5-2B@15852e8c16360a2fea060d615a32b45270f8a8fc",+ .extra_out = "qwen3.5-2b-mtp-q4k.jns",+ .extra_sha =+ "19c2d7eba1800acb10a06d7b38709ec92776e852cb2f25032ddd8977d17f2b05",+ .ram_gb = 4},+ {.name = "qwen3.5-9b",+ .what = "Qwen3.5-9B, dense, with its MTP block",+ .repo = "unsloth/Qwen3.5-9B-GGUF",+ .parts = {"Qwen3.5-9B-Q4_K_M.gguf"},+ .part_sha =+ {"03b74727a860a56338e042c4420bb3f04b2fec5734175f4cb9fa853daf52b7e8"},+ .gguf_gb = 5.68,+ .out = "qwen3.5-9b-q4km.jns",+ .flat_sha =+ "e40ec6c58ef2390c7a4ce8c9b7b6b384ed4e55c9980f646d1c98e83e2c018115",+ .extra = EXTRA_HF2JNS,+ .extra_src = "hf:Qwen/Qwen3.5-9B@c202236235762e1c871ad0ccb60c8ee5ba337b9a",+ .extra_out = "qwen3.5-9b-mtp-q4k.jns",+ .extra_sha =+ "67f31f57dbc78ca8c02ef5e3aa137c5df3836a3decda20b6ea057d004089c361",+ .ram_gb = 8},+ {.name = "qwen3-30b-a3b",+ .what = "Qwen3-30B-A3B, mixture of experts",+ .repo = "Qwen/Qwen3-30B-A3B-GGUF",+ .parts = {"Qwen3-30B-A3B-Q4_K_M.gguf"},+ .part_sha =+ {"0d003f6662faee786ed5da3e31b29c978de5ae5d275c8794c606a7f3c01aa8f5"},+ .gguf_gb = 18.6,+ .out = "qwen3-30b-a3b-q4km.jns",+ .flat_sha =+ "6906de51f2923b3be3a83eb040eaaeb090cba3389e9217648abe08bd678376e9",+ .jns_sha =+ "3fa0185b7eecd43e222d420f68aae28a86b3b90392fd7e4052b5cd337318e895",+ .ram_gb = 16},+ {.name = "qwen3.6-35b-a3b",+ .what = "Qwen3.6-35B-A3B, mixture of experts, MTP inside",+ .repo = "bartowski/Qwen_Qwen3.6-35B-A3B-GGUF",+ .parts = {"Qwen_Qwen3.6-35B-A3B-Q4_K_M.gguf"},+ .part_sha =+ {"b46fedd33e0bfb0cae308aa3c158d0a4b2c4a1d2185a1ed6f093cdaf39064772"},+ .gguf_gb = 22.3,+ .out = "qwen3.6-35b-a3b-q4km.jns",+ .flat_sha =+ "31772037300f332d712f4ef8dd5e7251c7cd76cf6b9dab33b089394b7c079fb2",+ .jns_sha =+ "8feaeb66d93593b9876b564ca3878f2a9252e46b8d4e2d06caa4e19d43dc6464",+ .ram_gb = 16},+ {.name = "qwen3-next-80b-a3b",+ .what = "Qwen3-Next-80B-A3B-Instruct, with its MTP block",+ .repo = "Qwen/Qwen3-Next-80B-A3B-Instruct-GGUF",+ .parts = {"Qwen3-Next-80B-A3B-Instruct-Q4_K_M.gguf"},+ .part_sha =+ {"d103b2733ec1012a52d01edda66b7e5c24ae50508c9f99f5297ea459ef3c061a"},+ .gguf_gb = 48.4,+ .out = "qwen3-next-80b-a3b-q4km.jns",+ .flat_sha =+ "fa9a50dd910de4e78064ecff17aef43286694277eea3a89a57e9921ca2bfd2da",+ .jns_sha =+ "fa3255c73106391e9cb5a97efd30f77a928fb4518d40060f795a8cff6656daed",+ .extra = EXTRA_HF2JNS,+ .extra_src = "hf:Qwen/Qwen3-Next-80B-A3B-Instruct",+ .extra_out = "qwen3-next-80b-a3b-mtp-q4k.jns",+ .extra_sha =+ "5c9bc4cb6f739193fac5e4daa30c922f73255acf41443ecfc0123ef084584559",+ .ram_gb = 32},+ {.name = "qwen3-coder-next",+ .what = "Qwen3-Coder-Next, mixture of experts, for code",+ .repo = "Qwen/Qwen3-Coder-Next-GGUF",+ .parts =+ {"Qwen3-Coder-Next-Q4_K_M/Qwen3-Coder-Next-Q4_K_M-00001-of-00004.gguf",+ "Qwen3-Coder-Next-Q4_K_M/Qwen3-Coder-Next-Q4_K_M-00002-of-00004.gguf",+ "Qwen3-Coder-Next-Q4_K_M/Qwen3-Coder-Next-Q4_K_M-00003-of-00004.gguf",+ "Qwen3-Coder-Next-Q4_K_M/Qwen3-Coder-Next-Q4_K_M-00004-of-00004.gguf"},+ .part_sha =+ {"6bcfc9f9c37901eeb92172e2ab871224dab36a453d263bcb2547f737409534da",+ "817def0691ee9d08bf3dc4444be7aed29c9e52091e8fa9d97901ce7e7f6f01d3",+ "23aa634d47dca9b4ca3ea249384e6f01951b24c83cdc076f37f6f43d6c99883f",+ "249c768cc5f130dc731567d6edcbdacc48e14dec9e02c5dbe2b2185d2c5bdb2b"},+ .gguf_gb = 48.4,+ .out = "qwen3-coder-next-q4km.jns",+ .flat_sha =+ "5912d964247002473eff1811058b4ffadf7b9bb47ae336c486cdf2964de68382",+ .jns_sha =+ "0276b0697a05075fec8ab076a757205537804ff41dd7e9dadace7c6993763563",+ .ram_gb = 32},+ {.name = "qwen3-embedding-0.6b",+ .what = "Qwen3-Embedding-0.6B, for janas-server --embedding-model",+ .repo = "Qwen/Qwen3-Embedding-0.6B-GGUF",+ .parts = {"Qwen3-Embedding-0.6B-Q8_0.gguf"},+ .part_sha =+ {"06507c7b42688469c4e7298b0a1e16deff06caf291cf0a5b278c308249c3e439"},+ .gguf_gb = 0.64,+ .out = "qwen3-embedding-0.6b-q8.jns",+ .flat_sha =+ "1e8eb8cbeba7a1a657354d259c5730bb79f6087160a9ca8da901bd08bdbe943f",+ .ram_gb = 2},+ {.name = "gemma-4-e2b",+ .what = "Gemma-4-E2B-it, with its assistant",+ .repo = "unsloth/gemma-4-E2B-it-GGUF",+ .parts = {"gemma-4-E2B-it-Q4_K_M.gguf"},+ .part_sha =+ {"740185b21d22ceb83a11c3aa62ad5842ef32c70f6096d756bbee85a1e4ec34b8"},+ .gguf_gb = 3.1,+ .out = "gemma-4-e2b-it-q4km.jns",+ .flat_sha =+ "1f18e08878df41f377d563b38048d41841c7973ddea845cd9672fcf23cbd864b",+ .extra = EXTRA_GGUF,+ .extra_src = "mtp-gemma-4-E2B-it.gguf",+ .extra_src_sha =+ "9eba819938efccfd6044f8af84e3bbfddc639a2bcf32ebc36420e6a649191919",+ .extra_out = "gemma-4-e2b-it-mtp.jns",+ .extra_sha =+ "79df1f3c817542b3867a0940b1a5b7330d478bddb4102ec90ff817334877cd62",+ .ram_gb = 4},+ {.name = "gemma-4-e4b",+ .what = "Gemma-4-E4B-it, with its assistant",+ .repo = "unsloth/gemma-4-E4B-it-GGUF",+ .parts = {"gemma-4-E4B-it-Q4_K_M.gguf"},+ .part_sha =+ {"85a896a047553e842f25297ee5b031d64ff30147d9c4af17b1e4b394cd1fab87"},+ .gguf_gb = 5.0,+ .out = "gemma-4-e4b-it-q4km.jns",+ .flat_sha =+ "7ec828e9bdc1ada2eda907e93b87072f3180f2e8cba5ccedcd360d1a53858675",+ .extra = EXTRA_GGUF,+ .extra_src = "mtp-gemma-4-E4B-it.gguf",+ .extra_src_sha =+ "b6a723115efa510d3b3215db1e26790dae84cd08c2134a764f3d194f1f0c3376",+ .extra_out = "gemma-4-e4b-it-mtp.jns",+ .extra_sha =+ "7c6fdb74ba1eb11f9d47b15097a09cce0142aea380a41f1986ba7b8ec4785b53",+ .ram_gb = 8},+ {.name = "gemma-4-12b",+ .what = "Gemma-4-12B-it, dense, with its assistant",+ .repo = "unsloth/gemma-4-12b-it-GGUF",+ .parts = {"gemma-4-12b-it-Q4_K_M.gguf"},+ .part_sha =+ {"0a270ec9fe6b34f4a0d33992b6135117b484ebc4766ab76b51d4ae8c457e4c42"},+ .gguf_gb = 7.1,+ .out = "gemma-4-12b-it-q4km.jns",+ .flat_sha =+ "cae09e04598891dea6e9cd93fb496fbde08d822ac35de4ff45dbfab1ea25c4dd",+ .extra = EXTRA_GGUF,+ .extra_src = "mtp-gemma-4-12b-it.gguf",+ .extra_src_sha =+ "145db9094bc0f85f1701e255a2ed216dcc9800fc8bc8631ad00905b456bd451b",+ .extra_out = "gemma-4-12b-it-mtp.jns",+ .extra_sha =+ "35e640f20dd74e862df9c1d949ad4edca9f58beef0cb29e99cccc6c99297826f",+ .ram_gb = 16},+ {.name = "gemma-4-26b-a4b",+ .what = "Gemma-4-26B-A4B-it, mixture of experts, with its assistant",+ .repo = "unsloth/gemma-4-26B-A4B-it-GGUF",+ .parts = {"gemma-4-26B-A4B-it-UD-Q4_K_M.gguf"},+ .part_sha =+ {"f2c28b3dc4776931ac6f879e11f203dec637ea0f14267a86ec8f6165f63f293f"},+ .gguf_gb = 16.9,+ .out = "gemma-4-26b-a4b-it-udq4km.jns",+ .flat_sha =+ "35c3845bbc21de52bf630d3c7455b4bb5379c34487b5925e020ec2e573931666",+ .extra = EXTRA_GGUF,+ .extra_src = "mtp-gemma-4-26B-A4B-it.gguf",+ .extra_src_sha =+ "6326fb9f5e487aa8dcdd313a091e3c67724cb2a666ec3b7d2895b5b26d93ed1b",+ .extra_out = "gemma-4-26b-it-mtp.jns",+ .extra_sha =+ "6ba24ee91fe6fe904b29919acbb654a66d9caa0aba67db1df9f921a76e5bcedf",+ .ram_gb = 16},+};++const int get_catalog_n = (int)(sizeof(get_catalog) / sizeof(get_catalog[0]));++const struct get_model *get_find(const char *name)+{+ for (int i = 0; i < get_catalog_n; i++)+ if (strcmp(get_catalog[i].name, name) == 0)+ return &get_catalog[i];+ return NULL;+}diff --git a/src/get/catalog.h b/src/get/catalog.hnew file mode 100644index 0000000..8de7fc6--- /dev/null+++ b/src/get/catalog.h@@ -0,0 +1,44 @@+/* SPDX-License-Identifier: GPL-3.0-or-later+ Copyright (C) 2026 Maurizio Cammalleri */+/*+ * catalog.h - the models janas-get knows: where each comes from, and the+ * SHA-256 of every file on the way, as MODELS.md lists them.+ */+#ifndef JANAS_GET_CATALOG_H+#define JANAS_GET_CATALOG_H++#define GET_MAX_PARTS 4++/* how the model's multi-token prediction file is made, if it has one */+enum get_extra {+ EXTRA_NONE,+ EXTRA_GGUF, /* a GGUF of its own (Gemma 4's assistant), gguf2jns */+ EXTRA_HF2JNS /* the block of the original checkpoint, hf2jns_mtp hf: */+};++struct get_model {+ const char *name; /* what janas-get takes */+ const char *what; /* a line for the list */+ const char *repo; /* the GGUF's repository */+ const char *parts[GET_MAX_PARTS]; /* its file(s), the first given to+ gguf2jns */+ const char *part_sha[GET_MAX_PARTS];+ double gguf_gb; /* download */+ const char *out; /* the .jns, beside the downloads */+ const char *flat_sha; /* what gguf2jns writes (the .jns itself when+ there are no planes) */+ const char *jns_sha; /* what jns_planes writes, NULL: one step */+ enum get_extra extra;+ const char *extra_src; /* GGUF file of repo, or hf:repo[@rev] */+ const char *extra_src_sha; /* of that GGUF (EXTRA_GGUF) */+ const char *extra_out;+ const char *extra_sha;+ int ram_gb; /* the smallest machine it is meant for */+};++extern const struct get_model get_catalog[];+extern const int get_catalog_n;++const struct get_model *get_find(const char *name);++#endifdiff --git a/src/get/main.c b/src/get/main.cnew file mode 100644index 0000000..e1c98de--- /dev/null+++ b/src/get/main.c@@ -0,0 +1,240 @@+/* SPDX-License-Identifier: GPL-3.0-or-later+ Copyright (C) 2026 Maurizio Cammalleri */+/*+ * janas-get - a model ready for Janas in one command: downloaded from+ * Hugging Face, checked, converted, its prediction file made beside it,+ * every file compared with the fingerprints of MODELS.md.+ *+ * janas-get list+ * janas-get <model> [--dir DIR] [--keep] [--no-mtp]+ * janas-get hf:<owner>/<repo>/<file.gguf> [--dir DIR] [--keep]+ *+ * A download stopped halfway resumes where it stopped; a file already+ * there is checked, not fetched again. The GGUF goes once converted,+ * unless --keep.+ */+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <sys/stat.h>+#include <sys/statvfs.h>++#include "catalog.h"+#include "common/sysinfo.h"+#include "steps.h"++static void usage(void)+{+ fprintf(stderr,+ "usage: janas-get list\n"+ " janas-get <model> [--dir DIR] [--keep] [--no-mtp]\n"+ " janas-get hf:<owner>/<repo>/<file.gguf> [--dir DIR] "+ "[--keep]\n"+ " --dir DIR where the models go (default: models)\n"+ " --keep keep the downloaded GGUF once converted\n"+ " --no-mtp without the model's prediction file\n");+}++static void list(void)+{+ printf("%-22s %8s %7s %s\n", "model", "download", "memory", "");+ for (int i = 0; i < get_catalog_n; i++) {+ const struct get_model *m = &get_catalog[i];+ printf("%-22s %6.1f GB %4d GB %s\n", m->name, m->gguf_gb, m->ram_gb,+ m->what);+ }+ printf("\nFingerprints and licences: MODELS.md. Any other GGUF: "+ "janas-get hf:<owner>/<repo>/<file.gguf>\n");+}++static int exists(const char *path)+{+ struct stat st;+ return stat(path, &st) == 0;+}++/* a converter's output compared with MODELS.md; 0 when it matches */+static int check(const char *file, const char *want, const char *who)+{+ int r = get_same(file, want);+ if (r == 1) {+ printf(" %s: fingerprint as in MODELS.md\n", who);+ return 0;+ }+ fprintf(stderr,+ "janas-get: %s wrote %s, and its SHA-256 is not the one of "+ "MODELS.md: please report it (an issue on GitHub)\n",+ who, file);+ return -1;+}++/* gguf2jns from the first part into out */+static int convert(const char *first, const char *out)+{+ char *argv[] = {"gguf2jns", (char *)first, (char *)out, NULL};+ printf("== converting (gguf2jns)\n");+ fflush(stdout);+ return get_run("gguf2jns", argv) == 0 ? 0 : -1;+}++static int from_catalog(const struct get_model *m, const char *dir, int keep,+ int no_mtp)+{+ char out[4096], first[4096], flat[4200];+ snprintf(out, sizeof(out), "%s/%s", dir, m->out);+ get_local(first, sizeof(first), dir, m->parts[0]);+ if (m->ram_gb &&+ janas_mem_total() < (uint64_t)m->ram_gb * (1ull << 30) * 85 / 100)+ printf("note: %s is meant for machines with %d GB of memory or "+ "more; this one has %.1f GB\n",+ m->name, m->ram_gb, (double)janas_mem_total() / (1ull << 30));+ int have = exists(out) &&+ get_same(out, m->jns_sha ? m->jns_sha : m->flat_sha) == 1;+ if (have) {+ printf("%s: already here, fingerprint checked\n", out);+ } else {+ /* room: two copies at the most - the GGUF and what gguf2jns+ writes, then (the GGUF gone) that and what jns_planes writes */+ struct statvfs vf;+ double need = m->gguf_gb * 2.0 * 1e9;+ if (statvfs(dir, &vf) == 0 &&+ (double)vf.f_bavail * (double)vf.f_frsize < need) {+ fprintf(stderr,+ "janas-get: %s needs about %.0f GB free in %s while it "+ "works; there are %.0f\n",+ m->name, need / 1e9, dir,+ (double)vf.f_bavail * (double)vf.f_frsize / 1e9);+ return 1;+ }+ printf("== %s: downloading\n", m->name);+ for (int p = 0; p < GET_MAX_PARTS && m->parts[p]; p++)+ if (get_fetch(m->repo, m->parts[p], dir, m->part_sha[p]) != 0)+ return 1;+ snprintf(flat, sizeof(flat), "%s.flat", out);+ const char *conv = m->jns_sha ? flat : out;+ if (convert(first, conv) != 0 ||+ check(conv, m->flat_sha, "gguf2jns") != 0)+ return 1;+ if (!keep)+ for (int p = 0; p < GET_MAX_PARTS && m->parts[p]; p++) {+ char part[4096];+ get_local(part, sizeof(part), dir, m->parts[p]);+ remove(part);+ }+ if (m->jns_sha) {+ char *argv[] = {"jns_planes", flat, out, NULL};+ printf("== cutting the experts into bit planes (jns_planes)\n");+ fflush(stdout);+ if (get_run("jns_planes", argv) != 0 ||+ check(out, m->jns_sha, "jns_planes") != 0)+ return 1;+ remove(flat);+ }+ }+ if (m->extra != EXTRA_NONE && !no_mtp) {+ char xout[4096];+ snprintf(xout, sizeof(xout), "%s/%s", dir, m->extra_out);+ if (exists(xout) && get_same(xout, m->extra_sha) == 1) {+ printf("%s: already here, fingerprint checked\n", xout);+ } else if (m->extra == EXTRA_GGUF) {+ char src[4096];+ printf("== its assistant\n");+ if (get_fetch(m->repo, m->extra_src, dir, m->extra_src_sha) != 0)+ return 1;+ get_local(src, sizeof(src), dir, m->extra_src);+ if (convert(src, xout) != 0 ||+ check(xout, m->extra_sha, "gguf2jns (assistant)") != 0)+ return 1;+ if (!keep)+ remove(src);+ } else {+ char *argv[] = {"hf2jns_mtp", out, (char *)m->extra_src, xout,+ NULL};+ printf("== its MTP block, from %s (hf2jns_mtp)\n", m->extra_src);+ fflush(stdout);+ if (get_run("hf2jns_mtp", argv) != 0 ||+ check(xout, m->extra_sha, "hf2jns_mtp") != 0)+ return 1;+ }+ }+ printf("\nReady: %s\nChat with it: janas-chat %s\n", out, out);+ return 0;+}++/* hf:<owner>/<repo>/<path>: any GGUF, checked against the SHA-256 Hugging+ Face lists, converted in one step */+static int from_hf(const char *spec, const char *dir, int keep)+{+ char repo[512];+ const char *s = spec + 3, *a = strchr(s, '/'),+ *b = a ? strchr(a + 1, '/') : NULL;+ size_t n = b ? (size_t)(b - s) : 0;+ if (!b || n >= sizeof(repo) || !b[1]) {+ usage();+ return 2;+ }+ memcpy(repo, s, n);+ repo[n] = 0;+ const char *path = b + 1;+ char local[4096], out[4200];+ get_local(local, sizeof(local), dir, path);+ snprintf(out, sizeof(out), "%s", local);+ char *dot = strrchr(out, '.');+ if (dot && strcmp(dot, ".gguf") == 0)+ *dot = 0;+ for (char *c = strrchr(out, '/') ? strrchr(out, '/') + 1 : out; *c; c++)+ if (*c >= 'A' && *c <= 'Z')+ *c = (char)(*c - 'A' + 'a');+ strcat(out, ".jns");+ printf("== %s: downloading (not in the catalog: checked against the "+ "SHA-256 Hugging Face lists)\n",+ path);+ if (get_fetch(repo, path, dir, NULL) != 0 || convert(local, out) != 0)+ return 1;+ if (!keep)+ remove(local);+ printf("\nReady: %s (no fingerprint to compare: not in MODELS.md)\n"+ "Chat with it: janas-chat %s\n",+ out, out);+ return 0;+}++int main(int argc, char **argv)+{+ if (argc < 2) {+ usage();+ return 2;+ }+ const char *dir = "models";+ int keep = 0, no_mtp = 0;+ for (int i = 2; i < argc; i++) {+ if (strcmp(argv[i], "--dir") == 0 && i + 1 < argc)+ dir = argv[++i];+ else if (strcmp(argv[i], "--keep") == 0)+ keep = 1;+ else if (strcmp(argv[i], "--no-mtp") == 0)+ no_mtp = 1;+ else {+ usage();+ return 2;+ }+ }+ if (strcmp(argv[1], "list") == 0) {+ list();+ return 0;+ }+ if (strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "--help") == 0) {+ usage();+ return 0;+ }+ mkdir(dir, 0755);+ if (strncmp(argv[1], "hf:", 3) == 0)+ return from_hf(argv[1], dir, keep);+ const struct get_model *m = get_find(argv[1]);+ if (!m) {+ fprintf(stderr, "janas-get: no model called %s (janas-get list)\n",+ argv[1]);+ return 2;+ }+ return from_catalog(m, dir, keep, no_mtp);+}diff --git a/src/get/steps.c b/src/get/steps.cnew file mode 100644index 0000000..cb68297--- /dev/null+++ b/src/get/steps.c@@ -0,0 +1,129 @@+/* SPDX-License-Identifier: GPL-3.0-or-later+ Copyright (C) 2026 Maurizio Cammalleri */+/*+ * steps.c - janas-get's steps (see steps.h). The converters are the+ * programs of the same build, run as they are: what they write is what+ * MODELS.md fingerprints, whoever runs them.+ */+#define _GNU_SOURCE+#include "steps.h"++#include <spawn.h>+#include <stdint.h>+#include <stdio.h>+#include <string.h>+#include <sys/wait.h>+#include <time.h>+#include <unistd.h>++#include "common/hf_fetch.h"+#include "common/sha256.h"++extern char **environ;++void get_local(char *out, size_t len, const char *dir, const char *path)+{+ const char *base = strrchr(path, '/');+ snprintf(out, len, "%s/%s", dir, base ? base + 1 : path);+}++static double now(void)+{+ struct timespec t;+ clock_gettime(CLOCK_MONOTONIC, &t);+ return (double)t.tv_sec + 1e-9 * (double)t.tv_nsec;+}++struct bar {+ double t0;+ uint64_t from;+};++static void progress(uint64_t done, uint64_t size, void *arg)+{+ struct bar *b = arg;+ double dt = now() - b->t0;+ double mbs = dt > 0 ? (double)(done - b->from) / dt / 1e6 : 0;+ fprintf(stderr, "\r %.2f of %.2f GB (%.0f%%), %.0f MB/s ",+ (double)done / 1e9, (double)size / 1e9,+ size ? 100.0 * (double)done / (double)size : 100.0, mbs);+ if (done >= size)+ fputc('\n', stderr);+}++int get_same(const char *file, const char *want)+{+ char got[65];+ if (janas_sha256_file(file, got) != 0)+ return -1;+ return strcmp(got, want) == 0;+}++int get_fetch(const char *repo, const char *path, const char *dir,+ const char *want)+{+ char local[4096], err[512], listed[65], got[65];+ get_local(local, sizeof(local), dir, path);+ uint64_t size = 0;+ if (janas_hf_info(repo, NULL, path, &size, listed, err, sizeof(err)) != 0) {+ fprintf(stderr, "janas-get: %s\n", err);+ return -1;+ }+ /* the repository may have been changed since MODELS.md was written:+ then its file is not the one the fingerprints are of */+ if (want && listed[0] && strcmp(listed, want) != 0) {+ fprintf(stderr,+ "janas-get: %s has changed on Hugging Face since MODELS.md "+ "was written (SHA-256 %s, not %s); not downloaded\n",+ path, listed, want);+ return -1;+ }+ printf(" %s from %s, %.2f GB\n", path, repo, (double)size / 1e9);+ fflush(stdout);+ struct bar b = {.t0 = now()};+ FILE *f = fopen(local, "rb");+ if (f) {+ fseek(f, 0, SEEK_END);+ long at = ftell(f);+ b.from = at > 0 ? (uint64_t)at : 0;+ fclose(f);+ }+ if (janas_hf_download(repo, NULL, path, local, size, got, progress, &b, err,+ sizeof(err)) != 0) {+ fprintf(stderr, "\njanas-get: %s (run again to resume)\n", err);+ return -1;+ }+ const char *expect = want ? want : listed;+ if (expect[0] && strcmp(got, expect) != 0) {+ fprintf(stderr,+ "janas-get: %s is not the file it should be (SHA-256 %s, "+ "not %s); deleted\n",+ local, got, expect);+ remove(local);+ return -1;+ }+ printf(" SHA-256 checked%s\n", want ? " against MODELS.md" : "");+ return 0;+}++int get_run(const char *tool, char *const argv[])+{+ char self[4096], path[4200];+ ssize_t n = readlink("/proc/self/exe", self, sizeof(self) - 1);+ if (n <= 0)+ return -1;+ self[n] = 0;+ char *slash = strrchr(self, '/');+ if (slash)+ *slash = 0;+ snprintf(path, sizeof(path), "%s/%s", self, tool);+ pid_t pid;+ if (posix_spawn(&pid, path, NULL, NULL, argv, environ) != 0) {+ fprintf(stderr, "janas-get: cannot run %s\n", path);+ return -1;+ }+ int st;+ if (waitpid(pid, &st, 0) != pid)+ return -1;+ return WIFEXITED(st) ? WEXITSTATUS(st) : -1;+}diff --git a/src/get/steps.h b/src/get/steps.hnew file mode 100644index 0000000..792bad4--- /dev/null+++ b/src/get/steps.h@@ -0,0 +1,28 @@+/* SPDX-License-Identifier: GPL-3.0-or-later+ Copyright (C) 2026 Maurizio Cammalleri */+/*+ * steps.h - what janas-get does to a file: download it and check it, run a+ * converter beside janas-get, check what it wrote.+ */+#ifndef JANAS_GET_STEPS_H+#define JANAS_GET_STEPS_H++#include <stddef.h>++/* dir/the last part of path */+void get_local(char *out, size_t len, const char *dir, const char *path);++/* File path of repo downloaded into dir (resumed if part of it is there),+ its SHA-256 checked against want (NULL: the one Hugging Face lists). 0,+ or -1 having said why. */+int get_fetch(const char *repo, const char *path, const char *dir,+ const char *want);++/* 1 when file's SHA-256 is want; 0 when it is another, -1 unreadable. */+int get_same(const char *file, const char *want);++/* A converter of the same build (gguf2jns, jns_planes, hf2jns_mtp) run on+ argv (argv[0] its name, NULL-terminated). Its exit status, or -1. */+int get_run(const char *tool, char *const argv[]);++#endifdiff --git a/tests/test_get_catalog.c b/tests/test_get_catalog.cnew file mode 100644index 0000000..69e98cf--- /dev/null+++ b/tests/test_get_catalog.c@@ -0,0 +1,83 @@+/* SPDX-License-Identifier: GPL-3.0-or-later+ Copyright (C) 2026 Maurizio Cammalleri */+/*+ * test_get_catalog.c - janas-get's catalog against MODELS.md: every+ * SHA-256 in it 64 hex digits and written in MODELS.md, every name unique.+ * A digest copied by hand one digit short would otherwise turn good+ * downloads away (it happened while writing the catalog).+ */+#include <ctype.h>+#include <stdio.h>+#include <stdlib.h>+#include <string.h>++#include "../src/get/catalog.c"++static int failures;++static char *slurp(const char *path)+{+ FILE *f = fopen(path, "rb");+ if (!f)+ return NULL;+ fseek(f, 0, SEEK_END);+ long n = ftell(f);+ fseek(f, 0, SEEK_SET);+ char *p = malloc((size_t)n + 1);+ if (p && fread(p, 1, (size_t)n, f) != (size_t)n) {+ free(p);+ p = NULL;+ }+ if (p)+ p[n] = 0;+ fclose(f);+ return p;+}++static void digest(const char *models, const char *who, const char *d)+{+ if (!d)+ return;+ int hex = strlen(d) == 64;+ for (const char *c = d; hex && *c; c++)+ hex = isxdigit((unsigned char)*c) && !isupper((unsigned char)*c);+ if (!hex) {+ printf("FAIL %s: %s is not 64 hex digits\n", who, d);+ failures++;+ } else if (!strstr(models, d)) {+ printf("FAIL %s: %s is not in MODELS.md\n", who, d);+ failures++;+ }+}++int main(void)+{+ char *models = slurp("MODELS.md");+ if (!models) {+ printf("test_get_catalog: MODELS.md not found (run from the "+ "repository's root)\n");+ return 1;+ }+ for (int i = 0; i < get_catalog_n; i++) {+ const struct get_model *m = &get_catalog[i];+ for (int p = 0; p < GET_MAX_PARTS && m->parts[p]; p++)+ digest(models, m->name, m->part_sha[p]);+ digest(models, m->name, m->flat_sha);+ digest(models, m->name, m->jns_sha);+ digest(models, m->name, m->extra_src_sha);+ digest(models, m->name, m->extra_sha);+ if (!m->flat_sha || (m->extra != EXTRA_NONE && !m->extra_sha)) {+ printf("FAIL %s: a fingerprint missing\n", m->name);+ failures++;+ }+ for (int k = 0; k < i; k++)+ if (strcmp(get_catalog[k].name, m->name) == 0) {+ printf("FAIL %s: twice in the catalog\n", m->name);+ failures++;+ }+ }+ free(models);+ printf("test_get_catalog: %s (%d models)\n", failures ? "FAILED" : "ok",+ get_catalog_n);+ return failures ? 1 : 0;+}diff --git a/tests/test_sha256.c b/tests/test_sha256.cnew file mode 100644index 0000000..184bb8e--- /dev/null+++ b/tests/test_sha256.c@@ -0,0 +1,78 @@+/* SPDX-License-Identifier: GPL-3.0-or-later+ Copyright (C) 2026 Maurizio Cammalleri */+/*+ * test_sha256.c - SHA-256 against the vectors of FIPS 180-4 and, over+ * lengths across the block boundaries fed in uneven pieces, the portable+ * compression function against the one on the SHA extensions (the same+ * digest by both, JANAS_SHA256_C=1 forcing the portable one).+ */+#include <stdio.h>+#include <stdlib.h>+#include <string.h>++#include "common/sha256.h"++static int failures;++static void hex_of(const void *p, size_t n, size_t piece, char out[65])+{+ struct janas_sha256 s;+ janas_sha256_init(&s);+ const unsigned char *c = p;+ for (size_t at = 0; at < n; at += piece)+ janas_sha256_update(&s, c + at, n - at < piece ? n - at : piece);+ janas_sha256_hex(&s, out);+}++static void check(const char *what, const void *p, size_t n, const char *want)+{+ char got[65];+ hex_of(p, n, n ? n : 1, got);+ if (strcmp(got, want) != 0) {+ printf("FAIL %s: %s, want %s\n", what, got, want);+ failures++;+ }+}++int main(void)+{+ check("empty", "", 0,+ "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");+ check("abc", "abc", 3,+ "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");+ const char *two =+ "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq";+ check("two blocks", two, strlen(two),+ "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1");+ size_t n = 1000000;+ char *a = malloc(n);+ if (!a)+ return 1;+ memset(a, 'a', n);+ check("a million a", a, n,+ "cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0");+ /* uneven pieces: the digest does not depend on how the data comes */+ char whole[65], pieces[65];+ unsigned x = 12345;+ for (size_t i = 0; i < n; i++)+ a[i] = (char)((x = x * 1103515245u + 12345u) >> 16);+ for (size_t len = 0; len < 300; len += 7) {+ hex_of(a, len, len ? len : 1, whole);+ hex_of(a, len, 13, pieces);+ if (strcmp(whole, pieces) != 0) {+ printf("FAIL length %zu: whole %s, in pieces %s\n", len, whole,+ pieces);+ failures++;+ }+ }+ hex_of(a, n, n, whole);+ hex_of(a, n, 4099, pieces);+ if (strcmp(whole, pieces) != 0) {+ printf("FAIL a million bytes in pieces\n");+ failures++;+ }+ free(a);+ printf("test_sha256: %s%s\n", failures ? "FAILED" : "ok",+ getenv("JANAS_SHA256_C") ? " (portable)" : "");+ return failures ? 1 : 0;+}diff --git a/tools/janas-try.sh b/tools/janas-try.shindex 401bfd9..5866680 100755--- a/tools/janas-try.sh+++ b/tools/janas-try.sh@@ -101,41 +101,27 @@ ask()gib() { awk -v b="$1" 'BEGIN { printf "%.1f", b / 1073741824 }'; }# ---------------------------------------------------------------- models-# level: repository, file, GGUF sha256, converted (flat) sha256, with bit-# planes sha256 - the fingerprints of MODELS.md; the conversion is-# deterministic, so a different sum is a defect found+# level: the model's name for janas-get (which downloads, checks and+# converts it, with the fingerprints of MODELS.md), its repository and+# GGUF (for the report), the file that runs and its prediction file ("-":+# none, or inside it), the tools that made it, then download, disk while+# converting and the memory of the smallest machine, in bytes and GBmodel_of(){case "$1" in- quick) echo "Qwen/Qwen3-4B-GGUF Qwen3-4B-Q4_K_M.gguf \-7485fe6f11af29433bc51cab58009521f205840f5b4ae3a32fa7f92e8534fdf5 \-e8ed44bdd3c612ce3ec1204a4b9db671115efab4a2ada384c70cf6492d430291 \-1cbd8ebfdf14aee05777f5278e655382f040cdedc21d282e93806fc8cf279fc5 \-qwen3-4b-q4km 2684354560 5368709120 8" ;;- medium) echo "bartowski/Qwen_Qwen3.6-35B-A3B-GGUF Qwen_Qwen3.6-35B-A3B-Q4_K_M.gguf \-b46fedd33e0bfb0cae308aa3c158d0a4b2c4a1d2185a1ed6f093cdaf39064772 \-31772037300f332d712f4ef8dd5e7251c7cd76cf6b9dab33b089394b7c079fb2 \-8feaeb66d93593b9876b564ca3878f2a9252e46b8d4e2d06caa4e19d43dc6464 \-qwen3.6-35b-a3b-q4km 23622320128 48318382080 16" ;;- full) echo "Qwen/Qwen3-Next-80B-A3B-Instruct-GGUF Qwen3-Next-80B-A3B-Instruct-Q4_K_M.gguf \-d103b2733ec1012a52d01edda66b7e5c24ae50508c9f99f5297ea459ef3c061a \-fa9a50dd910de4e78064ecff17aef43286694277eea3a89a57e9921ca2bfd2da \-fa3255c73106391e9cb5a97efd30f77a928fb4518d40060f795a8cff6656daed \-qwen3-next-80b-a3b-q4km 55834574848 107374182400 16" ;;- gemma) echo "unsloth/gemma-4-E4B-it-GGUF gemma-4-E4B-it-Q4_K_M.gguf \-85a896a047553e842f25297ee5b031d64ff30147d9c4af17b1e4b394cd1fab87 \-7ec828e9bdc1ada2eda907e93b87072f3180f2e8cba5ccedcd360d1a53858675 \-- gemma-4-e4b-it-q4km 5476083302 11811160064 8" ;;+ quick) echo "qwen3-4b Qwen/Qwen3-4B-GGUF Qwen3-4B-Q4_K_M.gguf \+qwen3-4b-q4km.jns - planes 2684354560 5368709120 8" ;;+ medium) echo "qwen3.6-35b-a3b bartowski/Qwen_Qwen3.6-35B-A3B-GGUF \+Qwen_Qwen3.6-35B-A3B-Q4_K_M.gguf qwen3.6-35b-a3b-q4km.jns - planes \+23622320128 48318382080 16" ;;+ full) echo "qwen3-next-80b-a3b Qwen/Qwen3-Next-80B-A3B-Instruct-GGUF \+Qwen3-Next-80B-A3B-Instruct-Q4_K_M.gguf qwen3-next-80b-a3b-q4km.jns \+qwen3-next-80b-a3b-mtp-q4k.jns planes 55834574848 107374182400 16" ;;+ gemma) echo "gemma-4-e4b unsloth/gemma-4-E4B-it-GGUF \+gemma-4-E4B-it-Q4_K_M.gguf gemma-4-e4b-it-q4km.jns gemma-4-e4b-it-mtp.jns \+one 5476083302 11811160064 8" ;;esac}-# Gemma 4's assistant (its MTP file), converted by gguf2jns as the model-GEMMA_MTP_FILE=mtp-gemma-4-E4B-it.gguf-GEMMA_MTP_SUM=b6a723115efa510d3b3215db1e26790dae84cd08c2134a764f3d194f1f0c3376-GEMMA_MTP_OUT_SUM=7c6fdb74ba1eb11f9d47b15097a09cce0142aea380a41f1986ba7b8ec4785b53-MTP_REPO=Qwen/Qwen3-Next-80B-A3B-Instruct-MTP_FILE=model-00041-of-00041.safetensors-MTP_SUM=1f3ac4d828f7e08dd14eb4dc6f282139ae1fa0d894e43f247da2539d2ef43826-MTP_OUT_SUM=5c9bc4cb6f739193fac5e4daa30c922f73255acf41443ecfc0123ef084584559# The answer to a fixed prompt at temperature 0, cut at 64 tokens, as a# fingerprint of its text: the kernels give the same numbers bit for bit on@@ -262,43 +248,12 @@ build()return 1}-# ---------------------------------------------------------------- download-download()-{ # repository, file, sha256 -> $MODELS/file- local repo=$1 file=$2 sum=$3 dst=$MODELS/$2- if [ -f "$dst" ] && [ -f "$dst.ok" ]; then- note "$file: already here, checked"- return 0- fi- grep -qx "$dst" "$WORK/created" || echo "$dst" >> "$WORK/created"- note "$file: downloading (it resumes if interrupted)"- curl -fL -C - --retry 5 --retry-delay 5 -o "$dst" \- "https://huggingface.co/$repo/resolve/main/$file" || {- fail "the download of $file failed; run again to resume"- return 1- }- note "$file: checking its SHA-256"- if [ "$(sha256sum "$dst" | cut -d' ' -f1)" != "$sum" ]; then- fail "$file is not the file it should be (SHA-256 differs); deleted"- rm -f "$dst"- return 1- fi- touch "$dst.ok"-}--fingerprint_ok()-{ # file, sha256: 0 when it matches; the result goes to the report- local got- got=$(sha256sum "$1" | cut -d' ' -f1)- [ "$got" = "$2" ]-}-# ---------------------------------------------------------------- a levellevel(){local lv=$1- read -r repo gguf gsum fsum psum name dl disk ram <<<"$(model_of "$lv")"- local flat=$MODELS/$name-flat.jns jns=$MODELS/$name.jns converted=0+ read -r getname repo gguf jnsn mtpn steps dl disk ram <<<"$(model_of "$lv")"+ local jns=$MODELS/$jnsn converted=0say "Level $lv: ${gguf%.gguf}"machine# a machine sold with 16 GB shows less (the kernel and an integrated GPU@@ -318,67 +273,45 @@ level()echo "$lv declined" >> "$WORK/results"return 0}- printf '%s\n' "$flat" "$jns" >> "$WORK/created"- download "$repo" "$gguf" "$gsum" || return 1- # one step where there are no planes to cut (psum "-"): the file- # gguf2jns writes is the one that runs- local out=$flat- [ "$psum" = "-" ] && out=$jns- note "converting (gguf2jns)"- "$BIN/gguf2jns" "$MODELS/$gguf" "$out" >>"$LOG" 2>&1 || {- fail "gguf2jns failed: $LOG"; echo "$lv conversion FAILED" >> "$WORK/results"; return 1; }- fingerprint_ok "$out" "$fsum" && echo "$lv gguf2jns fingerprint ok" >> "$WORK/results" ||- echo "$lv gguf2jns fingerprint DIFFERS" >> "$WORK/results"- if [ "$KEEP" = 0 ] && ask "Delete the downloaded GGUF ($(gib "$(stat -c %s "$MODELS/$gguf")") GiB)? It is not needed any more"; then- rm -f "$MODELS/$gguf" "$MODELS/$gguf.ok"- fi- if [ "$psum" != "-" ]; then- note "cutting the experts into bit planes (jns_planes)"- "$BIN/jns_planes" "$flat" "$jns" >>"$LOG" 2>&1 || {- fail "jns_planes failed: $LOG"; echo "$lv planes FAILED" >> "$WORK/results"; return 1; }- fingerprint_ok "$jns" "$psum" && echo "$lv jns_planes fingerprint ok" >> "$WORK/results" ||- echo "$lv jns_planes fingerprint DIFFERS" >> "$WORK/results"- rm -f "$flat"- ficonverted=1fi- local mtp=()- if [ "$lv" = full ]; then- local mjns=$MODELS/$name-mtp.jns- if ! [ -f "$mjns" ]; then- echo "$mjns" >> "$WORK/created"- download "$MTP_REPO" "$MTP_FILE" "$MTP_SUM" || return 1- "$BIN/hf2jns_mtp" "$jns" "$MODELS/$MTP_FILE" "$mjns" >>"$LOG" 2>&1 || {- fail "hf2jns_mtp failed: $LOG"; echo "$lv mtp FAILED" >> "$WORK/results"; return 1; }- fingerprint_ok "$mjns" "$MTP_OUT_SUM" && echo "$lv hf2jns_mtp fingerprint ok" >> "$WORK/results" ||- echo "$lv hf2jns_mtp fingerprint DIFFERS" >> "$WORK/results"- [ "$KEEP" = 1 ] || rm -f "$MODELS/$MTP_FILE" "$MODELS/$MTP_FILE.ok"- fi- mtp=(--mtp "$mjns")- fi- if [ "$lv" = gemma ]; then- local mjns=$MODELS/$name-mtp.jns- if ! [ -f "$mjns" ]; then- echo "$mjns" >> "$WORK/created"- download "${repo}" "$GEMMA_MTP_FILE" "$GEMMA_MTP_SUM" || return 1- "$BIN/gguf2jns" "$MODELS/$GEMMA_MTP_FILE" "$mjns" >>"$LOG" 2>&1 || {- fail "gguf2jns failed on the assistant: $LOG"; echo "$lv assistant FAILED" >> "$WORK/results"; return 1; }- fingerprint_ok "$mjns" "$GEMMA_MTP_OUT_SUM" && echo "$lv assistant fingerprint ok" >> "$WORK/results" ||- echo "$lv assistant fingerprint DIFFERS" >> "$WORK/results"- [ "$KEEP" = 1 ] || rm -f "$MODELS/$GEMMA_MTP_FILE" "$MODELS/$GEMMA_MTP_FILE.ok"- fi- mtp=(--mtp "$mjns")+ # janas-get downloads, checks the SHA-256 of every file against+ # MODELS.md, converts, and makes the prediction file beside the model;+ # what it says goes to the screen and, for the report, to a file+ [ -f "$jns" ] || echo "$jns" >> "$WORK/created"+ [ "$mtpn" = - ] || [ -f "$MODELS/$mtpn" ] || echo "$MODELS/$mtpn" >> "$WORK/created"+ local keep=()+ [ "$KEEP" = 1 ] && keep=(--keep)+ note "janas-get $getname: the GGUF goes once converted${keep:+ (kept: --keep)}"+ "$BIN/janas-get" "$getname" --dir "$MODELS" "${keep[@]}" 2>&1 | tee "$WORK/get-$lv.txt"+ local rc=${PIPESTATUS[0]} got=$WORK/get-$lv.txt+ local t+ for t in "gguf2jns" "jns_planes" "gguf2jns (assistant)" "hf2jns_mtp"; do+ local key=${t%% *}+ [ "$t" = "gguf2jns (assistant)" ] && key=assistant+ grep -qF " $t: fingerprint as in MODELS.md" "$got" &&+ echo "$lv $key fingerprint ok" >> "$WORK/results"+ grep -qF "$t wrote" "$got" &&+ echo "$lv $key fingerprint DIFFERS" >> "$WORK/results"+ done+ if [ "$rc" != 0 ] || ! [ -f "$jns" ]; then+ fail "janas-get failed: $got"+ echo "$lv janas-get FAILED: $(grep -m1 'janas-get:' "$got")" >> "$WORK/results"+ return 1fi+ cat "$got" >> "$LOG"+ local mtp=()+ [ "$mtpn" = - ] || mtp=(--mtp "$MODELS/$mtpn")# what ran: Janas's own file, and where it came from- local shown=${gguf%%-Q4_K_M.gguf} how="the SHA-256 of both checked against MODELS.md"+ local shown=${gguf%%-Q4_K_M.gguf} how="the SHA-256 of every file checked against MODELS.md by janas-get"shown=${shown#Qwen_}- [ "$converted" = 1 ] || how="converted in an earlier run of this script"+ [ "$converted" = 1 ] || how="converted in an earlier run, checked again by janas-get"local tools="gguf2jns and jns_planes"- [ "$psum" = "-" ] && tools="gguf2jns"- local line="- Model ($lv): $shown, run as \`$(basename "$jns")\` - Janas's format (JNS), converted here by $tools from \`$gguf\` of \`$repo\`; $how"- [ "$lv" = gemma ] && line="$line; with its assistant \`$(basename "$MODELS/$name-mtp.jns")\`, converted by gguf2jns from \`$GEMMA_MTP_FILE\` of \`$repo\`"- [ "$lv" = full ] && line="$line; with the MTP block \`$(basename "$MODELS/$name-mtp.jns")\`, converted by hf2jns_mtp from \`$MTP_FILE\` of \`$MTP_REPO\`"+ [ "$steps" = one ] && tools="gguf2jns"+ local line="- Model ($lv): $shown, run as \`$jnsn\` - Janas's format (JNS), converted here by $tools from \`$gguf\` of \`$repo\`; $how"+ [ "$lv" = gemma ] && line="$line; with its assistant \`$mtpn\`, converted by gguf2jns"+ [ "$lv" = full ] && line="$line; with the MTP block \`$mtpn\`, made by hf2jns_mtp from the original checkpoint"grep -qF "$line" "$WORK/models" 2>/dev/null || echo "$line" >> "$WORK/models"# the answers: the same text as on every other machine