Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Invent tool for adding files to ZIP with alignment
Justine Tunneya459d435e57f
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mkindex a9ed9739..65fbb8d8 100644--- a/llamafile/BUILD.mk+++ b/llamafile/BUILD.mk@@ -11,5 +11,10 @@ LLAMAFILE_OBJS = \$(LLAMAFILE_SRCS:%.c=o/$(MODE)/%.o) \$(LLAMAFILE_FILES:%=o/$(MODE)/%.zip.o)+o/$(MODE)/llamafile/zipalign: \+ o/$(MODE)/llamafile/zipalign.o+.PHONY: o/$(MODE)/llamafile-o/$(MODE)/llamafile: $(LLAMAFILE_OBJS)+o/$(MODE)/llamafile: \+ $(LLAMAFILE_OBJS) \+ o/$(MODE)/llamafile/zipaligndiff --git a/llamafile/llamafile.c b/llamafile/llamafile.cindex 7b617821..28f9ed84 100644--- a/llamafile/llamafile.c+++ b/llamafile/llamafile.c@@ -128,7 +128,7 @@ struct llamafile *llamafile_open(const char *fname, const char *mode) {// the eocd (cdir) says where the central directory (cfile) array is located// we consistency check some legacy fields, to be extra sure that it is eocdunsigned cnt = 0;- for (int i = amt - Min(kZipCdirHdrMinSize, kZipCdir64LocatorSize); i; --i) {+ for (int i = amt - Min(kZipCdirHdrMinSize, kZipCdir64LocatorSize); i >= 0; --i) {uint32_t magic = ZIP_READ32(bufdata + i);if (magic == kZipCdir64LocatorMagic && i + kZipCdir64LocatorSize <= amt &&pread(fd, bufdata, kZipCdir64HdrMinSize,diff --git a/llamafile/zip.h b/llamafile/zip.hindex 34076540..e73f7d35 100644--- a/llamafile/zip.h+++ b/llamafile/zip.h@@ -118,46 +118,57 @@#define kZipCfileMagic "PK\001\002"-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__-#define ZIP_READ16(P) \- (__extension__({ \- uint16_t __x; \- __builtin_memcpy(&__x, P, 16 / 8); \- __x; \- }))-#define ZIP_READ32(P) \- (__extension__({ \- uint32_t __x; \- __builtin_memcpy(&__x, P, 32 / 8); \- __x; \- }))-#define ZIP_READ64(P) \- (__extension__({ \- uint64_t __x; \- __builtin_memcpy(&__x, P, 64 / 8); \- __x; \- }))+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__+#define ZIP_SWAP16_(P) __builtin_bswap16(P)+#define ZIP_SWAP32_(P) __builtin_bswap32(P)+#define ZIP_SWAP64_(P) __builtin_bswap64(P)#else-#define ZIP_READ16(P) \- (__extension__({ \- uint16_t __x; \- __builtin_memcpy(&__x, P, 16 / 8); \- __builtin_bswap16(__x); \- }))-#define ZIP_READ32(P) \- (__extension__({ \- uint32_t __x; \- __builtin_memcpy(&__x, P, 32 / 8); \- __builtin_bswap32(__x); \- }))-#define ZIP_READ64(P) \- (__extension__({ \- uint64_t __x; \- __builtin_memcpy(&__x, P, 64 / 8); \- __builtin_bswap64(__x); \- }))+#define ZIP_SWAP16_(P) (P)+#define ZIP_SWAP32_(P) (P)+#define ZIP_SWAP64_(P) (P)#endif+#define ZIP_READ16(P) \+ (__extension__({ \+ uint16_t __x; \+ __builtin_memcpy(&__x, P, 16 / 8); \+ ZIP_SWAP16_(__x); \+ }))+#define ZIP_READ32(P) \+ (__extension__({ \+ uint32_t __x; \+ __builtin_memcpy(&__x, P, 32 / 8); \+ ZIP_SWAP32_(__x); \+ }))+#define ZIP_READ64(P) \+ (__extension__({ \+ uint64_t __x; \+ __builtin_memcpy(&__x, P, 64 / 8); \+ ZIP_SWAP64_(__x); \+ }))++#define ZIP_WRITE16(P, X) \+ (__extension__({ \+ __typeof__(&(P)[0]) __p = (P); \+ uint16_t __x = ZIP_SWAP16_(X); \+ __builtin_memcpy(__p, &__x, 16 / 8); \+ __p + 16 / 8; \+ }))+#define ZIP_WRITE32(P, X) \+ (__extension__({ \+ __typeof__(&(P)[0]) __p = (P); \+ uint32_t __x = ZIP_SWAP32_(X); \+ __builtin_memcpy(__p, &__x, 32 / 8); \+ __p + 32 / 8; \+ }))+#define ZIP_WRITE64(P, X) \+ (__extension__({ \+ __typeof__(&(P)[0]) __p = (P); \+ uint64_t __x = ZIP_SWAP64_(X); \+ __builtin_memcpy(__p, &__x, 64 / 8); \+ __p + 64 / 8; \+ }))+/* end of central directory record */#define ZIP_CDIR_MAGIC(P) ZIP_READ32(P)#define ZIP_CDIR_DISK(P) ZIP_READ16((P) + kZipCdirDiskOffset)diff --git a/llamafile/zipalign.c b/llamafile/zipalign.cnew file mode 100644index 00000000..fca06435--- /dev/null+++ b/llamafile/zipalign.c@@ -0,0 +1,396 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set net ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2023 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include <time.h>+#include <fcntl.h>+#include <cosmo.h>+#include <stdio.h>+#include <limits.h>+#include <assert.h>+#include <stdint.h>+#include <stdlib.h>+#include <libgen.h>+#include <sys/uio.h>+#include <third_party/zlib/zlib.h>+#include <third_party/getopt/cosmo.h>+#include "zip.h"++#define USAGE \+ " ZIP FILE...\n\+\n\+SYNOPSIS\n\+\n\+ Adds aligned uncompressed files to PKZIP archives.\n\+\n\+FLAGS\n\+\n\+ -h help\n\+ -N nondeterministic mode\n\+ -a INT alignment (default 65536)\n\+ -j strip directory components\n\+\n"++#define Min(a, b) ((a) < (b) ? (a) : (b))+#define DOS_DATE(YEAR, MONTH_IDX1, DAY_IDX1) \+ (((YEAR)-1980) << 9 | (MONTH_IDX1) << 5 | (DAY_IDX1))+#define DOS_TIME(HOUR, MINUTE, SECOND) \+ ((HOUR) << 11 | (MINUTE) << 5 | (SECOND) >> 1)++static const char *prog;+static int alignment = 65536;+static bool nondeterministic;+static int strip_directory_components;++static wontreturn void Die(const char *thing, const char *reason) {+ tinyprint(2, thing, ": ", reason, "\n", NULL);+ exit(1);+}++static wontreturn void DieSys(const char *thing) {+ perror(thing);+ exit(1);+}++static wontreturn void DieOom(void) {+ Die("apelink", "out of memory");+}++static void *Malloc(size_t n) {+ void *p;+ if (!(p = malloc(n))) DieOom();+ return p;+}++static char *StrDup(const char *s) {+ char *p;+ if (!(p = strdup(s))) DieOom();+ return p;+}++static void *Realloc(void *p, size_t n) {+ if (!(p = realloc(p, n))) DieOom();+ return p;+}++static wontreturn void PrintUsage(int fd, int rc) {+ tinyprint(fd, "USAGE\n\n ", prog, USAGE, NULL);+ exit(rc);+}++static void GetOpts(int argc, char *argv[]) {+ int opt;+ while ((opt = getopt(argc, argv, "hjNa:")) != -1) {+ switch (opt) {+ case 'N':+ nondeterministic = true;+ break;+ case 'j':+ strip_directory_components = true;+ break;+ case 'a':+ alignment = atoi(optarg);+ if (alignment < 1) {+ Die(prog, "alignment must be at least 1");+ }+ if (alignment & (alignment - 1)) {+ Die(prog, "alignment must be two power");+ }+ break;+ case 'h':+ PrintUsage(1, 0);+ default:+ PrintUsage(2, 1);+ }+ }+}++static void GetDosLocalTime(int64_t utcunixts,+ uint16_t *out_time,+ uint16_t *out_date) {+ struct tm tm;+ gmtime_r(&utcunixts, &tm);+ *out_time = DOS_TIME(tm.tm_hour, tm.tm_min, tm.tm_sec);+ *out_date = DOS_DATE(tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday + 1);+}++int main(int argc, char *argv[]) {++ // get name of program+ prog = argv[0];+ if (!prog) prog = "zipalign";++ // parse flags+ GetOpts(argc, argv);+ if (optind == argc) {+ Die(prog, "missing output zip");+ }++ // open output file+ int zfd;+ ssize_t zsize;+ const char *zpath = argv[optind++];+ if ((zfd = open(zpath, O_CREAT | O_RDWR, 0644)) == -1) {+ DieSys(zpath);+ }+ if ((zsize = lseek(zfd, 0, SEEK_END)) == -1) {+ DieSys(zpath);+ }++ // read last 64kb of file+ int amt;+ off_t off;+ if (zsize <= 65536) {+ off = 0;+ amt = zsize;+ } else {+ off = zsize - 65536;+ amt = zsize - off;+ }+ char *last64 = Malloc(amt);+ if (pread(zfd, last64, amt, off) != amt) {+ DieSys(zpath);+ }++ // search backwards for the end-of-central-directory record+ // the eocd (cdir) says where the central directory (cfile) array is located+ // we consistency check some legacy fields, to be extra sure that it is eocd+ unsigned cnt = 0;+ for (int i = amt - Min(kZipCdirHdrMinSize, kZipCdir64LocatorSize); i >= 0; --i) {+ uint32_t magic = ZIP_READ32(last64 + i);+ if (magic == kZipCdir64LocatorMagic && i + kZipCdir64LocatorSize <= amt &&+ pread(zfd, last64, kZipCdir64HdrMinSize,+ ZIP_LOCATE64_OFFSET(last64 + i)) == (long)kZipCdir64HdrMinSize &&+ ZIP_READ32(last64) == kZipCdir64HdrMagic &&+ ZIP_CDIR64_RECORDS(last64) == ZIP_CDIR64_RECORDSONDISK(last64) &&+ ZIP_CDIR64_RECORDS(last64) && ZIP_CDIR64_SIZE(last64) <= INT_MAX) {+ cnt = ZIP_CDIR64_RECORDS(last64);+ off = ZIP_CDIR64_OFFSET(last64);+ amt = ZIP_CDIR64_SIZE(last64);+ break;+ }+ if (magic == kZipCdirHdrMagic && i + kZipCdirHdrMinSize <= amt &&+ ZIP_CDIR_RECORDS(last64 + i) == ZIP_CDIR_RECORDSONDISK(last64 + i) &&+ ZIP_CDIR_RECORDS(last64 + i) && ZIP_CDIR_SIZE(last64 + i) <= INT_MAX &&+ ZIP_CDIR_OFFSET(last64 + i) != 0xffffffffu) {+ cnt = ZIP_CDIR_RECORDS(last64 + i);+ off = ZIP_CDIR_OFFSET(last64 + i);+ amt = ZIP_CDIR_SIZE(last64 + i);+ break;+ }+ }+ if (!cnt) {+ amt = 0;+ }+ free(last64);++ // read central directory+ uint8_t *cdir = Malloc(amt);+ if (pread(zfd, cdir, amt, off) != amt) {+ DieSys(zpath);+ }++ // get time+ struct timespec now;+ if (nondeterministic) {+ now = timespec_real();+ } else {+ now = timespec_fromseconds(1700000000);+ }+ int64_t ft = (now.tv_sec + 11644473600) * 10000000;+ uint16_t mtime, mdate;+ GetDosLocalTime(now.tv_sec, &mtime, &mdate);++ // add inputs+ for (int i = optind; i < argc; ++i) {++ // open input file+ int fd;+ ssize_t size;+ const char *path = argv[i];+ if ((fd = open(path, O_RDONLY)) == -1) {+ DieSys(path);+ }+ if ((size = lseek(fd, 0, SEEK_END)) == -1) {+ DieSys(path);+ }++ // construct zip entry name+ char *name = StrDup(path);+ if (strip_directory_components) {+ name = basename(name);+ }++ // determine size and alignment of local file header+ size_t namlen = strlen(name);+ size_t extlen = (2 + 2 + 8 + 8) + (2 + 2 + 4 + 2 + 2 + 8 + 8 + 8);+ size_t hdrlen = kZipLfileHdrMinSize + namlen + extlen;+ while ((zsize + hdrlen) & (alignment - 1)) ++zsize;++ // copy file+ ssize_t rc;+ uint32_t crc = 0;+ _Alignas(4096) static uint8_t iobuf[2097152];+ for (off_t i = 0; i < size; i += rc) {+ if ((rc = pread(fd, iobuf, Min(size, sizeof(iobuf)), i)) <= 0) {+ DieSys(path);+ }+ crc = crc32(crc, iobuf, rc);+ if (pwrite(zfd, iobuf, rc, zsize + hdrlen + i) != rc) {+ DieSys(zpath);+ }+ }++ // write local file header+ uint8_t *lochdr = Malloc(hdrlen);+ uint8_t *p = lochdr;++ p = ZIP_WRITE32(p, kZipLfileHdrMagic);+ p = ZIP_WRITE16(p, kZipEra2001);+ p = ZIP_WRITE16(p, kZipGflagUtf8);+ p = ZIP_WRITE16(p, kZipCompressionNone);+ p = ZIP_WRITE16(p, mtime);+ p = ZIP_WRITE16(p, mdate);+ p = ZIP_WRITE32(p, crc);+ p = ZIP_WRITE32(p, Min(size, 0xffffffffu)); // compressed size+ p = ZIP_WRITE32(p, Min(size, 0xffffffffu)); // uncompressed size+ p = ZIP_WRITE16(p, namlen);+ p = ZIP_WRITE16(p, extlen);+ p = mempcpy(p, name, namlen);++ p = ZIP_WRITE16(p, kZipExtraZip64);+ p = ZIP_WRITE16(p, 8 + 8);+ p = ZIP_WRITE64(p, size); // uncompressed size+ p = ZIP_WRITE64(p, size); // compressed size++ p = ZIP_WRITE16(p, kZipExtraNtfs);+ p = ZIP_WRITE16(p, 4 + 2 + 2 + 8 + 8 + 8);+ p = ZIP_WRITE32(p, 0); // reserved+ p = ZIP_WRITE16(p, 1); // attribute tag+ p = ZIP_WRITE16(p, 8 + 8 + 8); // size of attribute+ p = ZIP_WRITE64(p, ft); // modified time+ p = ZIP_WRITE64(p, ft); // access time+ p = ZIP_WRITE64(p, ft); // creation time++ unassert(p == lochdr + hdrlen);+ if (pwrite(zfd, lochdr, hdrlen, zsize) != hdrlen) {+ DieSys(zpath);+ }+ free(lochdr);++ // create central directory entry+ extlen = (2 + 2 + 8 + 8 + 8) + (2 + 2 + 4 + 2 + 2 + 8 + 8 + 8);+ hdrlen = kZipCfileHdrMinSize + namlen + extlen;+ cdir = Realloc(cdir, amt + hdrlen);+ uint8_t *cdirhdr = cdir + amt;+ amt += hdrlen;+ p = cdirhdr;++ p = ZIP_WRITE32(p, kZipCfileHdrMagic);+ p = ZIP_WRITE16(p, kZipOsUnix << 8 | kZipEra2001); // version made by+ p = ZIP_WRITE16(p, kZipEra2001); // version needed to extract+ p = ZIP_WRITE16(p, kZipGflagUtf8);+ p = ZIP_WRITE16(p, kZipCompressionNone);+ p = ZIP_WRITE16(p, mtime);+ p = ZIP_WRITE16(p, mdate);+ p = ZIP_WRITE32(p, crc);+ p = ZIP_WRITE32(p, Min(size, 0xffffffffu)); // compressed size+ p = ZIP_WRITE32(p, Min(size, 0xffffffffu)); // uncompressed size+ p = ZIP_WRITE16(p, namlen);+ p = ZIP_WRITE16(p, extlen);+ p = ZIP_WRITE16(p, 0); // comment length+ p = ZIP_WRITE16(p, 0); // disk number start+ p = ZIP_WRITE16(p, kZipIattrBinary);+ p = ZIP_WRITE32(p, 0100644u << 16); // external file attributes+ p = ZIP_WRITE32(p, Min(zsize, 0xffffffffu));+ p = mempcpy(p, name, namlen);++ p = ZIP_WRITE16(p, kZipExtraZip64);+ p = ZIP_WRITE16(p, 8 + 8 + 8);+ p = ZIP_WRITE64(p, size); // uncompressed size+ p = ZIP_WRITE64(p, size); // compressed size+ p = ZIP_WRITE64(p, zsize); // lfile offset++ p = ZIP_WRITE16(p, kZipExtraNtfs);+ p = ZIP_WRITE16(p, 4 + 2 + 2 + 8 + 8 + 8);+ p = ZIP_WRITE32(p, 0); // reserved+ p = ZIP_WRITE16(p, 1); // attribute tag+ p = ZIP_WRITE16(p, 8 + 8 + 8); // size of attribute+ p = ZIP_WRITE64(p, ft); // modified time+ p = ZIP_WRITE64(p, ft); // access time+ p = ZIP_WRITE64(p, ft); // creation time++ unassert(p == cdirhdr + hdrlen);++ // finish up+ ++cnt;+ zsize += hdrlen + size;+ if (close(fd)) {+ DieSys(path);+ }+ }++ // write out the central directory+ if (pwrite(zfd, cdir, amt, zsize) != amt) {+ DieSys(zpath);+ }+ free(cdir);++ char eocd[kZipCdirHdrMinSize];+ char *p = eocd;+ p = ZIP_WRITE32(p, kZipCdirHdrMagic);+ p = ZIP_WRITE16(p, 0); // number of this disk+ p = ZIP_WRITE16(p, 0); // number of disks+ p = ZIP_WRITE16(p, cnt); // number of records on disk+ p = ZIP_WRITE16(p, cnt); // number of records+ p = ZIP_WRITE32(p, amt); // size of central directory+ p = ZIP_WRITE32(p, Min(zsize, 0xffffffffu)); // offset of central directory+ p = ZIP_WRITE16(p, 0); // comment length+ if (pwrite(zfd, eocd, kZipCdirHdrMinSize, zsize + amt) != kZipCdirHdrMinSize) {+ DieSys(zpath);+ }++ char eocd64[kZipCdir64HdrMinSize];+ p = eocd64;+ p = ZIP_WRITE32(p, kZipCdir64HdrMagic);+ p = ZIP_WRITE64(p, sizeof(eocd64) - 12); // size of eocd64+ p = ZIP_WRITE16(p, kZipOsUnix << 8 | kZipEra2001); // version made by+ p = ZIP_WRITE16(p, kZipEra2001); // version needed to extract+ p = ZIP_WRITE32(p, 0); // number of this disk+ p = ZIP_WRITE32(p, 0); // number of disk with start of central directory+ p = ZIP_WRITE64(p, cnt); // number of records on disk+ p = ZIP_WRITE64(p, cnt); // number of records+ p = ZIP_WRITE64(p, amt); // size of central directory+ p = ZIP_WRITE64(p, zsize); // offset of start of central directory+ if (pwrite(zfd, eocd64, kZipCdir64HdrMinSize, zsize + amt + kZipCdirHdrMinSize) != kZipCdir64HdrMinSize) {+ DieSys(zpath);+ }++ char locate64[kZipCdir64LocatorSize];+ p = locate64;+ p = ZIP_WRITE32(p, kZipCdir64LocatorMagic);+ p = ZIP_WRITE32(p, 0); // number of disk with eocd64+ p = ZIP_WRITE64(p, zsize + amt); // offset of eocd64+ p = ZIP_WRITE32(p, 1); // total number of disks+ if (pwrite(zfd, locate64, kZipCdir64LocatorSize, zsize + amt + kZipCdirHdrMinSize + kZipCdir64HdrMinSize) != kZipCdir64LocatorSize) {+ DieSys(zpath);+ }++ // close output+ if (close(zfd)) {+ DieSys(zpath);+ }+}