Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
APFS: libdeflate's decompressor, as released, for the kernel to build in
vcvkk53940ea05e3c
diff --git a/tools/mk-libdeflate.sh b/tools/mk-libdeflate.shnew file mode 100755index 0000000..d057488--- /dev/null+++ b/tools/mk-libdeflate.sh@@ -0,0 +1,35 @@+#!/bin/bash+# Put the decompressing half of libdeflate into the kernel tree, as bsd/miscfs/apfs/libdeflate/:+# the APFS driver inflates the zlib pieces of compressed files with it instead of libkern's zlib.+#+# Why: on the restore-ramdisk volume -- 19149 pieces, 180 MB in, 1229 MB out -- libkern's zlib+# inflates at 344 MB/s on the build machine and libdeflate at 740 MB/s, to the same bytes; and+# on the phone zlib was 3819 of the 4227 ms one reader spent in vnop_read over that volume.+#+# Taken as it is from the release, files unchanged; what the kernel lacks (an allocator, no C+# library) is in bsd/miscfs/apfs/apfs_deflate.c, which compiles these with -DFREESTANDING.+# libdeflate is Copyright 2016 Eric Biggers and 2024 Google LLC, under the MIT licence (COPYING,+# copied alongside).+set -e+VERSION=1.24+SHA256=a0dda1c4b804742066db07b9510876edd09cc0ca06cdc32c5dfe1b2016a26463+ROOT=$(cd "$(dirname "$(readlink -f "$0")")/.." && pwd)+OUT=${XNU_SRC:-$ROOT/xnu-src/xnu-8796.121.2}/bsd/miscfs/apfs/libdeflate+TMP=$(mktemp -d)+trap 'rm -rf "$TMP"' EXIT+curl -sL --fail -o "$TMP/l.tar.gz" \+ https://github.com/ebiggers/libdeflate/releases/download/v$VERSION/libdeflate-$VERSION.tar.gz+echo "$SHA256 $TMP/l.tar.gz" | sha256sum -c --quiet+tar xzf "$TMP/l.tar.gz" -C "$TMP"+S=$TMP/libdeflate-$VERSION+rm -rf "$OUT"+mkdir -p "$OUT/lib/arm"+cp "$S/COPYING" "$S/libdeflate.h" "$S/common_defs.h" "$OUT/"+for f in lib_common.h cpu_features_common.h deflate_constants.h zlib_constants.h \+ decompress_template.h deflate_decompress.c zlib_decompress.c adler32.c; do+ cp "$S/lib/$f" "$OUT/lib/"+done+# the architecture dispatch is included unconditionally; in the kernel it settles on NEON for+# the Adler-32 and the portable code for the rest (see apfs_deflate.c)+cp "$S/lib/arm/"*.h "$OUT/lib/arm/"+echo "mk-libdeflate.sh: $OUT ($(du -sk "$OUT" | cut -f1) KB, libdeflate $VERSION)"diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/COPYING b/xnu/files/bsd/miscfs/apfs/libdeflate/COPYINGnew file mode 100644index 0000000..9b620b3--- /dev/null+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/COPYING@@ -0,0 +1,22 @@+Copyright 2016 Eric Biggers+Copyright 2024 Google LLC++Permission is hereby granted, free of charge, to any person+obtaining a copy of this software and associated documentation files+(the "Software"), to deal in the Software without restriction,+including without limitation the rights to use, copy, modify, merge,+publish, distribute, sublicense, and/or sell copies of the Software,+and to permit persons to whom the Software is furnished to do so,+subject to the following conditions:++The above copyright notice and this permission notice shall be+included in all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS+BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/common_defs.h b/xnu/files/bsd/miscfs/apfs/libdeflate/common_defs.hnew file mode 100644index 0000000..a0868d4--- /dev/null+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/common_defs.h@@ -0,0 +1,748 @@+/*+ * common_defs.h+ *+ * Copyright 2016 Eric Biggers+ *+ * Permission is hereby granted, free of charge, to any person+ * obtaining a copy of this software and associated documentation+ * files (the "Software"), to deal in the Software without+ * restriction, including without limitation the rights to use,+ * copy, modify, merge, publish, distribute, sublicense, and/or sell+ * copies of the Software, and to permit persons to whom the+ * Software is furnished to do so, subject to the following+ * conditions:+ *+ * The above copyright notice and this permission notice shall be+ * included in all copies or substantial portions of the Software.+ *+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR+ * OTHER DEALINGS IN THE SOFTWARE.+ */++#ifndef COMMON_DEFS_H+#define COMMON_DEFS_H++#include "libdeflate.h"++#include <stdbool.h>+#include <stddef.h> /* for size_t */+#include <stdint.h>+#ifdef _MSC_VER+# include <intrin.h> /* for _BitScan*() and other intrinsics */+# include <stdlib.h> /* for _byteswap_*() */+ /* Disable MSVC warnings that are expected. */+ /* /W2 */+# pragma warning(disable : 4146) /* unary minus on unsigned type */+ /* /W3 */+# pragma warning(disable : 4018) /* signed/unsigned mismatch */+# pragma warning(disable : 4244) /* possible loss of data */+# pragma warning(disable : 4267) /* possible loss of precision */+# pragma warning(disable : 4310) /* cast truncates constant value */+ /* /W4 */+# pragma warning(disable : 4100) /* unreferenced formal parameter */+# pragma warning(disable : 4127) /* conditional expression is constant */+# pragma warning(disable : 4189) /* local variable initialized but not referenced */+# pragma warning(disable : 4232) /* nonstandard extension used */+# pragma warning(disable : 4245) /* conversion from 'int' to 'unsigned int' */+# pragma warning(disable : 4295) /* array too small to include terminating null */+#endif+#ifndef FREESTANDING+# include <string.h> /* for memcpy() */+#endif++/* ========================================================================== */+/* Target architecture */+/* ========================================================================== */++/* If possible, define a compiler-independent ARCH_* macro. */+#undef ARCH_X86_64+#undef ARCH_X86_32+#undef ARCH_ARM64+#undef ARCH_ARM32+#undef ARCH_RISCV+#ifdef _MSC_VER+ /* Way too many things are broken in ARM64EC to pretend that it is x86_64. */+# if defined(_M_X64) && !defined(_M_ARM64EC)+# define ARCH_X86_64+# elif defined(_M_IX86)+# define ARCH_X86_32+# elif defined(_M_ARM64)+# define ARCH_ARM64+# elif defined(_M_ARM)+# define ARCH_ARM32+# endif+#else+# if defined(__x86_64__)+# define ARCH_X86_64+# elif defined(__i386__)+# define ARCH_X86_32+# elif defined(__aarch64__)+# define ARCH_ARM64+# elif defined(__arm__)+# define ARCH_ARM32+# elif defined(__riscv)+# define ARCH_RISCV+# endif+#endif++/* ========================================================================== */+/* Type definitions */+/* ========================================================================== */++/* Fixed-width integer types */+typedef uint8_t u8;+typedef uint16_t u16;+typedef uint32_t u32;+typedef uint64_t u64;+typedef int8_t s8;+typedef int16_t s16;+typedef int32_t s32;+typedef int64_t s64;++/* ssize_t, if not available in <sys/types.h> */+#ifdef _MSC_VER+# ifdef _WIN64+ typedef long long ssize_t;+# else+ typedef long ssize_t;+# endif+#endif++/*+ * Word type of the target architecture. Use 'size_t' instead of+ * 'unsigned long' to account for platforms such as Windows that use 32-bit+ * 'unsigned long' on 64-bit architectures.+ */+typedef size_t machine_word_t;++/* Number of bytes in a word */+#define WORDBYTES ((int)sizeof(machine_word_t))++/* Number of bits in a word */+#define WORDBITS (8 * WORDBYTES)++/* ========================================================================== */+/* Optional compiler features */+/* ========================================================================== */++/* Compiler version checks. Only use when absolutely necessary. */+#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)+# define GCC_PREREQ(major, minor) \+ (__GNUC__ > (major) || \+ (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))+# if !GCC_PREREQ(4, 9)+# error "gcc versions older than 4.9 are no longer supported"+# endif+#else+# define GCC_PREREQ(major, minor) 0+#endif+#ifdef __clang__+# ifdef __apple_build_version__+# define CLANG_PREREQ(major, minor, apple_version) \+ (__apple_build_version__ >= (apple_version))+# else+# define CLANG_PREREQ(major, minor, apple_version) \+ (__clang_major__ > (major) || \+ (__clang_major__ == (major) && __clang_minor__ >= (minor)))+# endif+# if !CLANG_PREREQ(3, 9, 8000000)+# error "clang versions older than 3.9 are no longer supported"+# endif+#else+# define CLANG_PREREQ(major, minor, apple_version) 0+#endif+#ifdef _MSC_VER+# define MSVC_PREREQ(version) (_MSC_VER >= (version))+# if !MSVC_PREREQ(1900)+# error "MSVC versions older than Visual Studio 2015 are no longer supported"+# endif+#else+# define MSVC_PREREQ(version) 0+#endif++/*+ * __has_attribute(attribute) - check whether the compiler supports the given+ * attribute (and also supports doing the check in the first place). Mostly+ * useful just for clang, since gcc didn't add this macro until gcc 5.+ */+#ifndef __has_attribute+# define __has_attribute(attribute) 0+#endif++/*+ * __has_builtin(builtin) - check whether the compiler supports the given+ * builtin (and also supports doing the check in the first place). Mostly+ * useful just for clang, since gcc didn't add this macro until gcc 10.+ */+#ifndef __has_builtin+# define __has_builtin(builtin) 0+#endif++/* inline - suggest that a function be inlined */+#ifdef _MSC_VER+# define inline __inline+#endif /* else assume 'inline' is usable as-is */++/* forceinline - force a function to be inlined, if possible */+#if defined(__GNUC__) || __has_attribute(always_inline)+# define forceinline inline __attribute__((always_inline))+#elif defined(_MSC_VER)+# define forceinline __forceinline+#else+# define forceinline inline+#endif++/* MAYBE_UNUSED - mark a function or variable as maybe unused */+#if defined(__GNUC__) || __has_attribute(unused)+# define MAYBE_UNUSED __attribute__((unused))+#else+# define MAYBE_UNUSED+#endif++/* NORETURN - mark a function as never returning, e.g. due to calling abort() */+#if defined(__GNUC__) || __has_attribute(noreturn)+# define NORETURN __attribute__((noreturn))+#else+# define NORETURN+#endif++/*+ * restrict - hint that writes only occur through the given pointer.+ *+ * Don't use MSVC's __restrict, since it has nonstandard behavior.+ * Standard restrict is okay, if it is supported.+ */+#if !defined(__STDC_VERSION__) || (__STDC_VERSION__ < 201112L)+# if defined(__GNUC__) || defined(__clang__)+# define restrict __restrict__+# else+# define restrict+# endif+#endif /* else assume 'restrict' is usable as-is */++/* likely(expr) - hint that an expression is usually true */+#if defined(__GNUC__) || __has_builtin(__builtin_expect)+# define likely(expr) __builtin_expect(!!(expr), 1)+#else+# define likely(expr) (expr)+#endif++/* unlikely(expr) - hint that an expression is usually false */+#if defined(__GNUC__) || __has_builtin(__builtin_expect)+# define unlikely(expr) __builtin_expect(!!(expr), 0)+#else+# define unlikely(expr) (expr)+#endif++/* prefetchr(addr) - prefetch into L1 cache for read */+#undef prefetchr+#if defined(__GNUC__) || __has_builtin(__builtin_prefetch)+# define prefetchr(addr) __builtin_prefetch((addr), 0)+#elif defined(_MSC_VER)+# if defined(ARCH_X86_32) || defined(ARCH_X86_64)+# define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)+# elif defined(ARCH_ARM64)+# define prefetchr(addr) __prefetch2((addr), 0x00 /* prfop=PLDL1KEEP */)+# elif defined(ARCH_ARM32)+# define prefetchr(addr) __prefetch(addr)+# endif+#endif+#ifndef prefetchr+# define prefetchr(addr)+#endif++/* prefetchw(addr) - prefetch into L1 cache for write */+#undef prefetchw+#if defined(__GNUC__) || __has_builtin(__builtin_prefetch)+# define prefetchw(addr) __builtin_prefetch((addr), 1)+#elif defined(_MSC_VER)+# if defined(ARCH_X86_32) || defined(ARCH_X86_64)+# define prefetchw(addr) _m_prefetchw(addr)+# elif defined(ARCH_ARM64)+# define prefetchw(addr) __prefetch2((addr), 0x10 /* prfop=PSTL1KEEP */)+# elif defined(ARCH_ARM32)+# define prefetchw(addr) __prefetchw(addr)+# endif+#endif+#ifndef prefetchw+# define prefetchw(addr)+#endif++/*+ * _aligned_attribute(n) - declare that the annotated variable, or variables of+ * the annotated type, must be aligned on n-byte boundaries.+ */+#undef _aligned_attribute+#if defined(__GNUC__) || __has_attribute(aligned)+# define _aligned_attribute(n) __attribute__((aligned(n)))+#elif defined(_MSC_VER)+# define _aligned_attribute(n) __declspec(align(n))+#endif++/*+ * _target_attribute(attrs) - override the compilation target for a function.+ *+ * This accepts one or more comma-separated suffixes to the -m prefix jointly+ * forming the name of a machine-dependent option. On gcc-like compilers, this+ * enables codegen for the given targets, including arbitrary compiler-generated+ * code as well as the corresponding intrinsics. On other compilers this macro+ * expands to nothing, though MSVC allows intrinsics to be used anywhere anyway.+ */+#if defined(__GNUC__) || __has_attribute(target)+# define _target_attribute(attrs) __attribute__((target(attrs)))+#else+# define _target_attribute(attrs)+#endif++/* ========================================================================== */+/* Miscellaneous macros */+/* ========================================================================== */++#define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))+#define MIN(a, b) ((a) <= (b) ? (a) : (b))+#define MAX(a, b) ((a) >= (b) ? (a) : (b))+#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))+#define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))+#define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))+#define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))++/* ========================================================================== */+/* Endianness handling */+/* ========================================================================== */++/*+ * CPU_IS_LITTLE_ENDIAN() - 1 if the CPU is little endian, or 0 if it is big+ * endian. When possible this is a compile-time macro that can be used in+ * preprocessor conditionals. As a fallback, a generic method is used that+ * can't be used in preprocessor conditionals but should still be optimized out.+ */+#if defined(__BYTE_ORDER__) /* gcc v4.6+ and clang */+# define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)+#elif defined(_MSC_VER)+# define CPU_IS_LITTLE_ENDIAN() true+#else+static forceinline bool CPU_IS_LITTLE_ENDIAN(void)+{+ union {+ u32 w;+ u8 b;+ } u;++ u.w = 1;+ return u.b;+}+#endif++/* bswap16(v) - swap the bytes of a 16-bit integer */+static forceinline u16 bswap16(u16 v)+{+#if defined(__GNUC__) || __has_builtin(__builtin_bswap16)+ return __builtin_bswap16(v);+#elif defined(_MSC_VER)+ return _byteswap_ushort(v);+#else+ return (v << 8) | (v >> 8);+#endif+}++/* bswap32(v) - swap the bytes of a 32-bit integer */+static forceinline u32 bswap32(u32 v)+{+#if defined(__GNUC__) || __has_builtin(__builtin_bswap32)+ return __builtin_bswap32(v);+#elif defined(_MSC_VER)+ return _byteswap_ulong(v);+#else+ return ((v & 0x000000FF) << 24) |+ ((v & 0x0000FF00) << 8) |+ ((v & 0x00FF0000) >> 8) |+ ((v & 0xFF000000) >> 24);+#endif+}++/* bswap64(v) - swap the bytes of a 64-bit integer */+static forceinline u64 bswap64(u64 v)+{+#if defined(__GNUC__) || __has_builtin(__builtin_bswap64)+ return __builtin_bswap64(v);+#elif defined(_MSC_VER)+ return _byteswap_uint64(v);+#else+ return ((v & 0x00000000000000FF) << 56) |+ ((v & 0x000000000000FF00) << 40) |+ ((v & 0x0000000000FF0000) << 24) |+ ((v & 0x00000000FF000000) << 8) |+ ((v & 0x000000FF00000000) >> 8) |+ ((v & 0x0000FF0000000000) >> 24) |+ ((v & 0x00FF000000000000) >> 40) |+ ((v & 0xFF00000000000000) >> 56);+#endif+}++#define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))+#define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))+#define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))+#define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))+#define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))+#define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))++/* ========================================================================== */+/* Unaligned memory accesses */+/* ========================================================================== */++/*+ * UNALIGNED_ACCESS_IS_FAST() - 1 if unaligned memory accesses can be performed+ * efficiently on the target platform, otherwise 0.+ */+#if (defined(__GNUC__) || defined(__clang__)) && \+ (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \+ defined(__ARM_FEATURE_UNALIGNED) || defined(__powerpc64__) || \+ defined(__riscv_misaligned_fast) || \+ /*+ * For all compilation purposes, WebAssembly behaves like any other CPU+ * instruction set. Even though WebAssembly engine might be running on+ * top of different actual CPU architectures, the WebAssembly spec+ * itself permits unaligned access and it will be fast on most of those+ * platforms, and simulated at the engine level on others, so it's+ * worth treating it as a CPU architecture with fast unaligned access.+ */ defined(__wasm__))+# define UNALIGNED_ACCESS_IS_FAST 1+#elif defined(_MSC_VER)+# define UNALIGNED_ACCESS_IS_FAST 1+#else+# define UNALIGNED_ACCESS_IS_FAST 0+#endif++/*+ * Implementing unaligned memory accesses using memcpy() is portable, and it+ * usually gets optimized appropriately by modern compilers. I.e., each+ * memcpy() of 1, 2, 4, or WORDBYTES bytes gets compiled to a load or store+ * instruction, not to an actual function call.+ *+ * We no longer use the "packed struct" approach to unaligned accesses, as that+ * is nonstandard, has unclear semantics, and doesn't receive enough testing+ * (see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94994).+ *+ * arm32 with __ARM_FEATURE_UNALIGNED in gcc 5 and earlier is a known exception+ * where memcpy() generates inefficient code+ * (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67366). However, we no longer+ * consider that one case important enough to maintain different code for.+ * If you run into it, please just use a newer version of gcc (or use clang).+ */++#ifdef FREESTANDING+# define MEMCOPY __builtin_memcpy+#else+# define MEMCOPY memcpy+#endif++/* Unaligned loads and stores without endianness conversion */++#define DEFINE_UNALIGNED_TYPE(type) \+static forceinline type \+load_##type##_unaligned(const void *p) \+{ \+ type v; \+ \+ MEMCOPY(&v, p, sizeof(v)); \+ return v; \+} \+ \+static forceinline void \+store_##type##_unaligned(type v, void *p) \+{ \+ MEMCOPY(p, &v, sizeof(v)); \+}++DEFINE_UNALIGNED_TYPE(u16)+DEFINE_UNALIGNED_TYPE(u32)+DEFINE_UNALIGNED_TYPE(u64)+DEFINE_UNALIGNED_TYPE(machine_word_t)++#undef MEMCOPY++#define load_word_unaligned load_machine_word_t_unaligned+#define store_word_unaligned store_machine_word_t_unaligned++/* Unaligned loads with endianness conversion */++static forceinline u16+get_unaligned_le16(const u8 *p)+{+ if (UNALIGNED_ACCESS_IS_FAST)+ return le16_bswap(load_u16_unaligned(p));+ else+ return ((u16)p[1] << 8) | p[0];+}++static forceinline u16+get_unaligned_be16(const u8 *p)+{+ if (UNALIGNED_ACCESS_IS_FAST)+ return be16_bswap(load_u16_unaligned(p));+ else+ return ((u16)p[0] << 8) | p[1];+}++static forceinline u32+get_unaligned_le32(const u8 *p)+{+ if (UNALIGNED_ACCESS_IS_FAST)+ return le32_bswap(load_u32_unaligned(p));+ else+ return ((u32)p[3] << 24) | ((u32)p[2] << 16) |+ ((u32)p[1] << 8) | p[0];+}++static forceinline u32+get_unaligned_be32(const u8 *p)+{+ if (UNALIGNED_ACCESS_IS_FAST)+ return be32_bswap(load_u32_unaligned(p));+ else+ return ((u32)p[0] << 24) | ((u32)p[1] << 16) |+ ((u32)p[2] << 8) | p[3];+}++static forceinline u64+get_unaligned_le64(const u8 *p)+{+ if (UNALIGNED_ACCESS_IS_FAST)+ return le64_bswap(load_u64_unaligned(p));+ else+ return ((u64)p[7] << 56) | ((u64)p[6] << 48) |+ ((u64)p[5] << 40) | ((u64)p[4] << 32) |+ ((u64)p[3] << 24) | ((u64)p[2] << 16) |+ ((u64)p[1] << 8) | p[0];+}++static forceinline machine_word_t+get_unaligned_leword(const u8 *p)+{+ STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);+ if (WORDBITS == 32)+ return get_unaligned_le32(p);+ else+ return get_unaligned_le64(p);+}++/* Unaligned stores with endianness conversion */++static forceinline void+put_unaligned_le16(u16 v, u8 *p)+{+ if (UNALIGNED_ACCESS_IS_FAST) {+ store_u16_unaligned(le16_bswap(v), p);+ } else {+ p[0] = (u8)(v >> 0);+ p[1] = (u8)(v >> 8);+ }+}++static forceinline void+put_unaligned_be16(u16 v, u8 *p)+{+ if (UNALIGNED_ACCESS_IS_FAST) {+ store_u16_unaligned(be16_bswap(v), p);+ } else {+ p[0] = (u8)(v >> 8);+ p[1] = (u8)(v >> 0);+ }+}++static forceinline void+put_unaligned_le32(u32 v, u8 *p)+{+ if (UNALIGNED_ACCESS_IS_FAST) {+ store_u32_unaligned(le32_bswap(v), p);+ } else {+ p[0] = (u8)(v >> 0);+ p[1] = (u8)(v >> 8);+ p[2] = (u8)(v >> 16);+ p[3] = (u8)(v >> 24);+ }+}++static forceinline void+put_unaligned_be32(u32 v, u8 *p)+{+ if (UNALIGNED_ACCESS_IS_FAST) {+ store_u32_unaligned(be32_bswap(v), p);+ } else {+ p[0] = (u8)(v >> 24);+ p[1] = (u8)(v >> 16);+ p[2] = (u8)(v >> 8);+ p[3] = (u8)(v >> 0);+ }+}++static forceinline void+put_unaligned_le64(u64 v, u8 *p)+{+ if (UNALIGNED_ACCESS_IS_FAST) {+ store_u64_unaligned(le64_bswap(v), p);+ } else {+ p[0] = (u8)(v >> 0);+ p[1] = (u8)(v >> 8);+ p[2] = (u8)(v >> 16);+ p[3] = (u8)(v >> 24);+ p[4] = (u8)(v >> 32);+ p[5] = (u8)(v >> 40);+ p[6] = (u8)(v >> 48);+ p[7] = (u8)(v >> 56);+ }+}++static forceinline void+put_unaligned_leword(machine_word_t v, u8 *p)+{+ STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);+ if (WORDBITS == 32)+ put_unaligned_le32(v, p);+ else+ put_unaligned_le64(v, p);+}++/* ========================================================================== */+/* Bit manipulation functions */+/* ========================================================================== */++/*+ * Bit Scan Reverse (BSR) - find the 0-based index (relative to the least+ * significant end) of the *most* significant 1 bit in the input value. The+ * input value must be nonzero!+ */++static forceinline unsigned+bsr32(u32 v)+{+#if defined(__GNUC__) || __has_builtin(__builtin_clz)+ return 31 - __builtin_clz(v);+#elif defined(_MSC_VER)+ unsigned long i;++ _BitScanReverse(&i, v);+ return i;+#else+ unsigned i = 0;++ while ((v >>= 1) != 0)+ i++;+ return i;+#endif+}++static forceinline unsigned+bsr64(u64 v)+{+#if defined(__GNUC__) || __has_builtin(__builtin_clzll)+ return 63 - __builtin_clzll(v);+#elif defined(_MSC_VER) && defined(_WIN64)+ unsigned long i;++ _BitScanReverse64(&i, v);+ return i;+#else+ unsigned i = 0;++ while ((v >>= 1) != 0)+ i++;+ return i;+#endif+}++static forceinline unsigned+bsrw(machine_word_t v)+{+ STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);+ if (WORDBITS == 32)+ return bsr32(v);+ else+ return bsr64(v);+}++/*+ * Bit Scan Forward (BSF) - find the 0-based index (relative to the least+ * significant end) of the *least* significant 1 bit in the input value. The+ * input value must be nonzero!+ */++static forceinline unsigned+bsf32(u32 v)+{+#if defined(__GNUC__) || __has_builtin(__builtin_ctz)+ return __builtin_ctz(v);+#elif defined(_MSC_VER)+ unsigned long i;++ _BitScanForward(&i, v);+ return i;+#else+ unsigned i = 0;++ for (; (v & 1) == 0; v >>= 1)+ i++;+ return i;+#endif+}++static forceinline unsigned+bsf64(u64 v)+{+#if defined(__GNUC__) || __has_builtin(__builtin_ctzll)+ return __builtin_ctzll(v);+#elif defined(_MSC_VER) && defined(_WIN64)+ unsigned long i;++ _BitScanForward64(&i, v);+ return i;+#else+ unsigned i = 0;++ for (; (v & 1) == 0; v >>= 1)+ i++;+ return i;+#endif+}++static forceinline unsigned+bsfw(machine_word_t v)+{+ STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);+ if (WORDBITS == 32)+ return bsf32(v);+ else+ return bsf64(v);+}++/*+ * rbit32(v): reverse the bits in a 32-bit integer. This doesn't have a+ * fallback implementation; use '#ifdef rbit32' to check if this is available.+ */+#undef rbit32+#if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \+ (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))+static forceinline u32+rbit32(u32 v)+{+ __asm__("rbit %0, %1" : "=r" (v) : "r" (v));+ return v;+}+#define rbit32 rbit32+#elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)+static forceinline u32+rbit32(u32 v)+{+ __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));+ return v;+}+#define rbit32 rbit32+#endif++#endif /* COMMON_DEFS_H */diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/adler32.c b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/adler32.cnew file mode 100644index 0000000..d5f39d8--- /dev/null+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/adler32.c@@ -0,0 +1,162 @@+/*+ * adler32.c - Adler-32 checksum algorithm+ *+ * Copyright 2016 Eric Biggers+ *+ * Permission is hereby granted, free of charge, to any person+ * obtaining a copy of this software and associated documentation+ * files (the "Software"), to deal in the Software without+ * restriction, including without limitation the rights to use,+ * copy, modify, merge, publish, distribute, sublicense, and/or sell+ * copies of the Software, and to permit persons to whom the+ * Software is furnished to do so, subject to the following+ * conditions:+ *+ * The above copyright notice and this permission notice shall be+ * included in all copies or substantial portions of the Software.+ *+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR+ * OTHER DEALINGS IN THE SOFTWARE.+ */++#include "lib_common.h"++/* The Adler-32 divisor, or "base", value */+#define DIVISOR 65521++/*+ * MAX_CHUNK_LEN is the most bytes that can be processed without the possibility+ * of s2 overflowing when it is represented as an unsigned 32-bit integer. This+ * value was computed using the following Python script:+ *+ * divisor = 65521+ * count = 0+ * s1 = divisor - 1+ * s2 = divisor - 1+ * while True:+ * s1 += 0xFF+ * s2 += s1+ * if s2 > 0xFFFFFFFF:+ * break+ * count += 1+ * print(count)+ *+ * Note that to get the correct worst-case value, we must assume that every byte+ * has value 0xFF and that s1 and s2 started with the highest possible values+ * modulo the divisor.+ */+#define MAX_CHUNK_LEN 5552++/*+ * Update the Adler-32 values s1 and s2 using n bytes from p, update p to p + n,+ * update n to 0, and reduce s1 and s2 mod DIVISOR. It is assumed that neither+ * s1 nor s2 can overflow before the reduction at the end, i.e. n plus any bytes+ * already processed after the last reduction must not exceed MAX_CHUNK_LEN.+ *+ * This uses only portable C code. This is used as a fallback when a vectorized+ * implementation of Adler-32 (e.g. AVX2) is unavailable on the platform.+ *+ * Some of the vectorized implementations also use this to handle the end of the+ * data when the data isn't evenly divisible by the length the vectorized code+ * works on. To avoid compiler errors about target-specific option mismatches+ * when this is used in that way, this is a macro rather than a function.+ *+ * Although this is unvectorized, this does include an optimization where the+ * main loop processes four bytes at a time using a strategy similar to that+ * used by vectorized implementations. This provides increased instruction-+ * level parallelism compared to the traditional 's1 += *p++; s2 += s1;'.+ */+#define ADLER32_CHUNK(s1, s2, p, n) \+do { \+ if (n >= 4) { \+ u32 s1_sum = 0; \+ u32 byte_0_sum = 0; \+ u32 byte_1_sum = 0; \+ u32 byte_2_sum = 0; \+ u32 byte_3_sum = 0; \+ \+ do { \+ s1_sum += s1; \+ s1 += p[0] + p[1] + p[2] + p[3]; \+ byte_0_sum += p[0]; \+ byte_1_sum += p[1]; \+ byte_2_sum += p[2]; \+ byte_3_sum += p[3]; \+ p += 4; \+ n -= 4; \+ } while (n >= 4); \+ s2 += (4 * (s1_sum + byte_0_sum)) + (3 * byte_1_sum) + \+ (2 * byte_2_sum) + byte_3_sum; \+ } \+ for (; n; n--, p++) { \+ s1 += *p; \+ s2 += s1; \+ } \+ s1 %= DIVISOR; \+ s2 %= DIVISOR; \+} while (0)++static u32 MAYBE_UNUSED+adler32_generic(u32 adler, const u8 *p, size_t len)+{+ u32 s1 = adler & 0xFFFF;+ u32 s2 = adler >> 16;++ while (len) {+ size_t n = MIN(len, MAX_CHUNK_LEN & ~3);++ len -= n;+ ADLER32_CHUNK(s1, s2, p, n);+ }++ return (s2 << 16) | s1;+}++/* Include architecture-specific implementation(s) if available. */+#undef DEFAULT_IMPL+#undef arch_select_adler32_func+typedef u32 (*adler32_func_t)(u32 adler, const u8 *p, size_t len);+#if defined(ARCH_ARM32) || defined(ARCH_ARM64)+# include "arm/adler32_impl.h"+#elif defined(ARCH_X86_32) || defined(ARCH_X86_64)+# include "x86/adler32_impl.h"+#endif++#ifndef DEFAULT_IMPL+# define DEFAULT_IMPL adler32_generic+#endif++#ifdef arch_select_adler32_func+static u32 dispatch_adler32(u32 adler, const u8 *p, size_t len);++static volatile adler32_func_t adler32_impl = dispatch_adler32;++/* Choose the best implementation at runtime. */+static u32 dispatch_adler32(u32 adler, const u8 *p, size_t len)+{+ adler32_func_t f = arch_select_adler32_func();++ if (f == NULL)+ f = DEFAULT_IMPL;++ adler32_impl = f;+ return f(adler, p, len);+}+#else+/* The best implementation is statically known, so call it directly. */+#define adler32_impl DEFAULT_IMPL+#endif++LIBDEFLATEAPI u32+libdeflate_adler32(u32 adler, const void *buffer, size_t len)+{+ if (buffer == NULL) /* Return initial value. */+ return 1;+ return adler32_impl(adler, buffer, len);+}diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/adler32_impl.h b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/adler32_impl.hnew file mode 100644index 0000000..66d2f2b--- /dev/null+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/adler32_impl.h@@ -0,0 +1,365 @@+/*+ * arm/adler32_impl.h - ARM implementations of Adler-32 checksum algorithm+ *+ * Copyright 2016 Eric Biggers+ *+ * Permission is hereby granted, free of charge, to any person+ * obtaining a copy of this software and associated documentation+ * files (the "Software"), to deal in the Software without+ * restriction, including without limitation the rights to use,+ * copy, modify, merge, publish, distribute, sublicense, and/or sell+ * copies of the Software, and to permit persons to whom the+ * Software is furnished to do so, subject to the following+ * conditions:+ *+ * The above copyright notice and this permission notice shall be+ * included in all copies or substantial portions of the Software.+ *+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR+ * OTHER DEALINGS IN THE SOFTWARE.+ */++#ifndef LIB_ARM_ADLER32_IMPL_H+#define LIB_ARM_ADLER32_IMPL_H++#include "cpu_features.h"++/* Regular NEON implementation */+#if HAVE_NEON_INTRIN && CPU_IS_LITTLE_ENDIAN()+# define adler32_arm_neon adler32_arm_neon+# if HAVE_NEON_NATIVE+ /*+ * Use no attributes if none are needed, to support old versions of clang+ * that don't accept the simd target attribute.+ */+# define ATTRIBUTES+# elif defined(ARCH_ARM32)+# define ATTRIBUTES _target_attribute("fpu=neon")+# elif defined(__clang__)+# define ATTRIBUTES _target_attribute("simd")+# else+# define ATTRIBUTES _target_attribute("+simd")+# endif+static ATTRIBUTES MAYBE_UNUSED u32+adler32_arm_neon(u32 adler, const u8 *p, size_t len)+{+ static const u16 _aligned_attribute(16) mults[64] = {+ 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49,+ 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33,+ 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,+ 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1,+ };+ const uint16x8_t mults_a = vld1q_u16(&mults[0]);+ const uint16x8_t mults_b = vld1q_u16(&mults[8]);+ const uint16x8_t mults_c = vld1q_u16(&mults[16]);+ const uint16x8_t mults_d = vld1q_u16(&mults[24]);+ const uint16x8_t mults_e = vld1q_u16(&mults[32]);+ const uint16x8_t mults_f = vld1q_u16(&mults[40]);+ const uint16x8_t mults_g = vld1q_u16(&mults[48]);+ const uint16x8_t mults_h = vld1q_u16(&mults[56]);+ u32 s1 = adler & 0xFFFF;+ u32 s2 = adler >> 16;++ /*+ * If the length is large and the pointer is misaligned, align it.+ * For smaller lengths, just take the misaligned load penalty.+ */+ if (unlikely(len > 32768 && ((uintptr_t)p & 15))) {+ do {+ s1 += *p++;+ s2 += s1;+ len--;+ } while ((uintptr_t)p & 15);+ s1 %= DIVISOR;+ s2 %= DIVISOR;+ }++ while (len) {+ /*+ * Calculate the length of the next data chunk such that s1 and+ * s2 are guaranteed to not exceed UINT32_MAX.+ */+ size_t n = MIN(len, MAX_CHUNK_LEN & ~63);++ len -= n;++ if (n >= 64) {+ uint32x4_t v_s1 = vdupq_n_u32(0);+ uint32x4_t v_s2 = vdupq_n_u32(0);+ /*+ * v_byte_sums_* contain the sum of the bytes at index i+ * across all 64-byte segments, for each index 0..63.+ */+ uint16x8_t v_byte_sums_a = vdupq_n_u16(0);+ uint16x8_t v_byte_sums_b = vdupq_n_u16(0);+ uint16x8_t v_byte_sums_c = vdupq_n_u16(0);+ uint16x8_t v_byte_sums_d = vdupq_n_u16(0);+ uint16x8_t v_byte_sums_e = vdupq_n_u16(0);+ uint16x8_t v_byte_sums_f = vdupq_n_u16(0);+ uint16x8_t v_byte_sums_g = vdupq_n_u16(0);+ uint16x8_t v_byte_sums_h = vdupq_n_u16(0);++ s2 += s1 * (n & ~63);++ do {+ /* Load the next 64 data bytes. */+ const uint8x16_t data_a = vld1q_u8(p + 0);+ const uint8x16_t data_b = vld1q_u8(p + 16);+ const uint8x16_t data_c = vld1q_u8(p + 32);+ const uint8x16_t data_d = vld1q_u8(p + 48);+ uint16x8_t tmp;++ /*+ * Accumulate the previous s1 counters into the+ * s2 counters. The needed multiplication by 64+ * is delayed to later.+ */+ v_s2 = vaddq_u32(v_s2, v_s1);++ /*+ * Add the 64 data bytes to their v_byte_sums+ * counters, while also accumulating the sums of+ * each adjacent set of 4 bytes into v_s1.+ */+ tmp = vpaddlq_u8(data_a);+ v_byte_sums_a = vaddw_u8(v_byte_sums_a,+ vget_low_u8(data_a));+ v_byte_sums_b = vaddw_u8(v_byte_sums_b,+ vget_high_u8(data_a));+ tmp = vpadalq_u8(tmp, data_b);+ v_byte_sums_c = vaddw_u8(v_byte_sums_c,+ vget_low_u8(data_b));+ v_byte_sums_d = vaddw_u8(v_byte_sums_d,+ vget_high_u8(data_b));+ tmp = vpadalq_u8(tmp, data_c);+ v_byte_sums_e = vaddw_u8(v_byte_sums_e,+ vget_low_u8(data_c));+ v_byte_sums_f = vaddw_u8(v_byte_sums_f,+ vget_high_u8(data_c));+ tmp = vpadalq_u8(tmp, data_d);+ v_byte_sums_g = vaddw_u8(v_byte_sums_g,+ vget_low_u8(data_d));+ v_byte_sums_h = vaddw_u8(v_byte_sums_h,+ vget_high_u8(data_d));+ v_s1 = vpadalq_u16(v_s1, tmp);++ p += 64;+ n -= 64;+ } while (n >= 64);++ /* s2 = 64*s2 + (64*bytesum0 + 63*bytesum1 + ... + 1*bytesum63) */+ #ifdef ARCH_ARM32+ # define umlal2(a, b, c) vmlal_u16((a), vget_high_u16(b), vget_high_u16(c))+ #else+ # define umlal2 vmlal_high_u16+ #endif+ v_s2 = vqshlq_n_u32(v_s2, 6);+ v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_a),+ vget_low_u16(mults_a));+ v_s2 = umlal2(v_s2, v_byte_sums_a, mults_a);+ v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_b),+ vget_low_u16(mults_b));+ v_s2 = umlal2(v_s2, v_byte_sums_b, mults_b);+ v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_c),+ vget_low_u16(mults_c));+ v_s2 = umlal2(v_s2, v_byte_sums_c, mults_c);+ v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_d),+ vget_low_u16(mults_d));+ v_s2 = umlal2(v_s2, v_byte_sums_d, mults_d);+ v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_e),+ vget_low_u16(mults_e));+ v_s2 = umlal2(v_s2, v_byte_sums_e, mults_e);+ v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_f),+ vget_low_u16(mults_f));+ v_s2 = umlal2(v_s2, v_byte_sums_f, mults_f);+ v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_g),+ vget_low_u16(mults_g));+ v_s2 = umlal2(v_s2, v_byte_sums_g, mults_g);+ v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_h),+ vget_low_u16(mults_h));+ v_s2 = umlal2(v_s2, v_byte_sums_h, mults_h);+ #undef umlal2++ /* Horizontal sum to finish up */+ #ifdef ARCH_ARM32+ s1 += vgetq_lane_u32(v_s1, 0) + vgetq_lane_u32(v_s1, 1) ++ vgetq_lane_u32(v_s1, 2) + vgetq_lane_u32(v_s1, 3);+ s2 += vgetq_lane_u32(v_s2, 0) + vgetq_lane_u32(v_s2, 1) ++ vgetq_lane_u32(v_s2, 2) + vgetq_lane_u32(v_s2, 3);+ #else+ s1 += vaddvq_u32(v_s1);+ s2 += vaddvq_u32(v_s2);+ #endif+ }+ /*+ * Process the last 0 <= n < 64 bytes of the chunk using+ * scalar instructions and reduce s1 and s2 mod DIVISOR.+ */+ ADLER32_CHUNK(s1, s2, p, n);+ }+ return (s2 << 16) | s1;+}+#undef ATTRIBUTES+#endif /* Regular NEON implementation */++/* NEON+dotprod implementation */+#if HAVE_DOTPROD_INTRIN && CPU_IS_LITTLE_ENDIAN() && \+ !defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_DOTPROD)+# define adler32_arm_neon_dotprod adler32_arm_neon_dotprod+# ifdef __clang__+# define ATTRIBUTES _target_attribute("dotprod")+ /*+ * Both gcc and binutils originally considered dotprod to depend on+ * arch=armv8.2-a or later. This was fixed in gcc 13.2 by commit+ * 9aac37ab8a7b ("aarch64: Remove architecture dependencies from intrinsics")+ * and in binutils 2.41 by commit 205e4380c800 ("aarch64: Remove version+ * dependencies from features"). Unfortunately, always using arch=armv8.2-a+ * causes build errors with some compiler options because it may reduce the+ * arch rather than increase it. Therefore we try to omit the arch whenever+ * possible. If gcc is 14 or later, then both gcc and binutils are probably+ * fixed, so we omit the arch. We also omit the arch if a feature that+ * depends on armv8.2-a or later (in gcc 13.1 and earlier) is present.+ */+# elif GCC_PREREQ(14, 0) || defined(__ARM_FEATURE_JCVT) \+ || defined(__ARM_FEATURE_DOTPROD)+# define ATTRIBUTES _target_attribute("+dotprod")+# else+# define ATTRIBUTES _target_attribute("arch=armv8.2-a+dotprod")+# endif+static ATTRIBUTES u32+adler32_arm_neon_dotprod(u32 adler, const u8 *p, size_t len)+{+ static const u8 _aligned_attribute(16) mults[64] = {+ 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49,+ 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33,+ 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,+ 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1,+ };+ const uint8x16_t mults_a = vld1q_u8(&mults[0]);+ const uint8x16_t mults_b = vld1q_u8(&mults[16]);+ const uint8x16_t mults_c = vld1q_u8(&mults[32]);+ const uint8x16_t mults_d = vld1q_u8(&mults[48]);+ const uint8x16_t ones = vdupq_n_u8(1);+ u32 s1 = adler & 0xFFFF;+ u32 s2 = adler >> 16;++ /*+ * If the length is large and the pointer is misaligned, align it.+ * For smaller lengths, just take the misaligned load penalty.+ */+ if (unlikely(len > 32768 && ((uintptr_t)p & 15))) {+ do {+ s1 += *p++;+ s2 += s1;+ len--;+ } while ((uintptr_t)p & 15);+ s1 %= DIVISOR;+ s2 %= DIVISOR;+ }++ while (len) {+ /*+ * Calculate the length of the next data chunk such that s1 and+ * s2 are guaranteed to not exceed UINT32_MAX.+ */+ size_t n = MIN(len, MAX_CHUNK_LEN & ~63);++ len -= n;++ if (n >= 64) {+ uint32x4_t v_s1_a = vdupq_n_u32(0);+ uint32x4_t v_s1_b = vdupq_n_u32(0);+ uint32x4_t v_s1_c = vdupq_n_u32(0);+ uint32x4_t v_s1_d = vdupq_n_u32(0);+ uint32x4_t v_s2_a = vdupq_n_u32(0);+ uint32x4_t v_s2_b = vdupq_n_u32(0);+ uint32x4_t v_s2_c = vdupq_n_u32(0);+ uint32x4_t v_s2_d = vdupq_n_u32(0);+ uint32x4_t v_s1_sums_a = vdupq_n_u32(0);+ uint32x4_t v_s1_sums_b = vdupq_n_u32(0);+ uint32x4_t v_s1_sums_c = vdupq_n_u32(0);+ uint32x4_t v_s1_sums_d = vdupq_n_u32(0);+ uint32x4_t v_s1;+ uint32x4_t v_s2;+ uint32x4_t v_s1_sums;++ s2 += s1 * (n & ~63);++ do {+ uint8x16_t data_a = vld1q_u8(p + 0);+ uint8x16_t data_b = vld1q_u8(p + 16);+ uint8x16_t data_c = vld1q_u8(p + 32);+ uint8x16_t data_d = vld1q_u8(p + 48);++ v_s1_sums_a = vaddq_u32(v_s1_sums_a, v_s1_a);+ v_s1_a = vdotq_u32(v_s1_a, data_a, ones);+ v_s2_a = vdotq_u32(v_s2_a, data_a, mults_a);++ v_s1_sums_b = vaddq_u32(v_s1_sums_b, v_s1_b);+ v_s1_b = vdotq_u32(v_s1_b, data_b, ones);+ v_s2_b = vdotq_u32(v_s2_b, data_b, mults_b);++ v_s1_sums_c = vaddq_u32(v_s1_sums_c, v_s1_c);+ v_s1_c = vdotq_u32(v_s1_c, data_c, ones);+ v_s2_c = vdotq_u32(v_s2_c, data_c, mults_c);++ v_s1_sums_d = vaddq_u32(v_s1_sums_d, v_s1_d);+ v_s1_d = vdotq_u32(v_s1_d, data_d, ones);+ v_s2_d = vdotq_u32(v_s2_d, data_d, mults_d);++ p += 64;+ n -= 64;+ } while (n >= 64);++ v_s1 = vaddq_u32(vaddq_u32(v_s1_a, v_s1_b),+ vaddq_u32(v_s1_c, v_s1_d));+ v_s2 = vaddq_u32(vaddq_u32(v_s2_a, v_s2_b),+ vaddq_u32(v_s2_c, v_s2_d));+ v_s1_sums = vaddq_u32(vaddq_u32(v_s1_sums_a,+ v_s1_sums_b),+ vaddq_u32(v_s1_sums_c,+ v_s1_sums_d));+ v_s2 = vaddq_u32(v_s2, vqshlq_n_u32(v_s1_sums, 6));++ s1 += vaddvq_u32(v_s1);+ s2 += vaddvq_u32(v_s2);+ }+ /*+ * Process the last 0 <= n < 64 bytes of the chunk using+ * scalar instructions and reduce s1 and s2 mod DIVISOR.+ */+ ADLER32_CHUNK(s1, s2, p, n);+ }+ return (s2 << 16) | s1;+}+#undef ATTRIBUTES+#endif /* NEON+dotprod implementation */++#if defined(adler32_arm_neon_dotprod) && defined(__ARM_FEATURE_DOTPROD)+#define DEFAULT_IMPL adler32_arm_neon_dotprod+#else+static inline adler32_func_t+arch_select_adler32_func(void)+{+ const u32 features MAYBE_UNUSED = get_arm_cpu_features();++#ifdef adler32_arm_neon_dotprod+ if (HAVE_NEON(features) && HAVE_DOTPROD(features))+ return adler32_arm_neon_dotprod;+#endif+#ifdef adler32_arm_neon+ if (HAVE_NEON(features))+ return adler32_arm_neon;+#endif+ return NULL;+}+#define arch_select_adler32_func arch_select_adler32_func+#endif++#endif /* LIB_ARM_ADLER32_IMPL_H */diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/cpu_features.h b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/cpu_features.hnew file mode 100644index 0000000..dc9ab8a--- /dev/null+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/cpu_features.h@@ -0,0 +1,214 @@+/*+ * arm/cpu_features.h - feature detection for ARM CPUs+ *+ * Copyright 2018 Eric Biggers+ *+ * Permission is hereby granted, free of charge, to any person+ * obtaining a copy of this software and associated documentation+ * files (the "Software"), to deal in the Software without+ * restriction, including without limitation the rights to use,+ * copy, modify, merge, publish, distribute, sublicense, and/or sell+ * copies of the Software, and to permit persons to whom the+ * Software is furnished to do so, subject to the following+ * conditions:+ *+ * The above copyright notice and this permission notice shall be+ * included in all copies or substantial portions of the Software.+ *+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR+ * OTHER DEALINGS IN THE SOFTWARE.+ */++#ifndef LIB_ARM_CPU_FEATURES_H+#define LIB_ARM_CPU_FEATURES_H++#include "../lib_common.h"++#if defined(ARCH_ARM32) || defined(ARCH_ARM64)++#define ARM_CPU_FEATURE_NEON (1 << 0)+#define ARM_CPU_FEATURE_PMULL (1 << 1)+/*+ * PREFER_PMULL indicates that the CPU has very high pmull throughput, and so+ * the 12x wide pmull-based CRC-32 implementation is likely to be faster than an+ * implementation based on the crc32 instructions.+ */+#define ARM_CPU_FEATURE_PREFER_PMULL (1 << 2)+#define ARM_CPU_FEATURE_CRC32 (1 << 3)+#define ARM_CPU_FEATURE_SHA3 (1 << 4)+#define ARM_CPU_FEATURE_DOTPROD (1 << 5)++#if !defined(FREESTANDING) && \+ (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)) && \+ (defined(__linux__) || \+ (defined(__APPLE__) && defined(ARCH_ARM64)) || \+ (defined(_WIN32) && defined(ARCH_ARM64)))+/* Runtime ARM CPU feature detection is supported. */+# define ARM_CPU_FEATURES_KNOWN (1U << 31)+extern volatile u32 libdeflate_arm_cpu_features;++void libdeflate_init_arm_cpu_features(void);++static inline u32 get_arm_cpu_features(void)+{+ if (libdeflate_arm_cpu_features == 0)+ libdeflate_init_arm_cpu_features();+ return libdeflate_arm_cpu_features;+}+#else+static inline u32 get_arm_cpu_features(void) { return 0; }+#endif++/* NEON */+#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(ARCH_ARM64))+# define HAVE_NEON(features) 1+# define HAVE_NEON_NATIVE 1+#else+# define HAVE_NEON(features) ((features) & ARM_CPU_FEATURE_NEON)+# define HAVE_NEON_NATIVE 0+#endif+/*+ * With both gcc and clang, NEON intrinsics require that the main target has+ * NEON enabled already. Exception: with gcc 6.1 and later (r230411 for arm32,+ * r226563 for arm64), hardware floating point support is sufficient.+ */+#if (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)) && \+ (HAVE_NEON_NATIVE || (GCC_PREREQ(6, 1) && defined(__ARM_FP)))+# define HAVE_NEON_INTRIN 1+# include <arm_neon.h>+#else+# define HAVE_NEON_INTRIN 0+#endif++/* PMULL */+#ifdef __ARM_FEATURE_CRYPTO+# define HAVE_PMULL(features) 1+#else+# define HAVE_PMULL(features) ((features) & ARM_CPU_FEATURE_PMULL)+#endif+#if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \+ (GCC_PREREQ(7, 1) || defined(__clang__) || defined(_MSC_VER)) && \+ CPU_IS_LITTLE_ENDIAN() /* untested on big endian */+# define HAVE_PMULL_INTRIN 1+ /* Work around MSVC's vmull_p64() taking poly64x1_t instead of poly64_t */+# ifdef _MSC_VER+# define compat_vmull_p64(a, b) vmull_p64(vcreate_p64(a), vcreate_p64(b))+# else+# define compat_vmull_p64(a, b) vmull_p64((a), (b))+# endif+#else+# define HAVE_PMULL_INTRIN 0+#endif++/* CRC32 */+#ifdef __ARM_FEATURE_CRC32+# define HAVE_CRC32(features) 1+#else+# define HAVE_CRC32(features) ((features) & ARM_CPU_FEATURE_CRC32)+#endif+#if defined(ARCH_ARM64) && \+ (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER))+# define HAVE_CRC32_INTRIN 1+# if defined(__GNUC__) || defined(__clang__)+# include <arm_acle.h>+# endif+ /*+ * Use an inline assembly fallback for clang 15 and earlier, which only+ * defined the crc32 intrinsics when crc32 is enabled in the main target.+ */+# if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \+ !defined(__ARM_FEATURE_CRC32)+# undef __crc32b+# define __crc32b(a, b) \+ ({ uint32_t res; \+ __asm__("crc32b %w0, %w1, %w2" \+ : "=r" (res) : "r" (a), "r" (b)); \+ res; })+# undef __crc32h+# define __crc32h(a, b) \+ ({ uint32_t res; \+ __asm__("crc32h %w0, %w1, %w2" \+ : "=r" (res) : "r" (a), "r" (b)); \+ res; })+# undef __crc32w+# define __crc32w(a, b) \+ ({ uint32_t res; \+ __asm__("crc32w %w0, %w1, %w2" \+ : "=r" (res) : "r" (a), "r" (b)); \+ res; })+# undef __crc32d+# define __crc32d(a, b) \+ ({ uint32_t res; \+ __asm__("crc32x %w0, %w1, %2" \+ : "=r" (res) : "r" (a), "r" (b)); \+ res; })+# pragma clang diagnostic ignored "-Wgnu-statement-expression"+# endif+#else+# define HAVE_CRC32_INTRIN 0+#endif++/* SHA3 (needed for the eor3 instruction) */+#ifdef __ARM_FEATURE_SHA3+# define HAVE_SHA3(features) 1+#else+# define HAVE_SHA3(features) ((features) & ARM_CPU_FEATURE_SHA3)+#endif+#if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \+ (GCC_PREREQ(9, 1) /* r268049 */ || \+ CLANG_PREREQ(7, 0, 10010463) /* r338010 */)+# define HAVE_SHA3_INTRIN 1+ /*+ * Use an inline assembly fallback for clang 15 and earlier, which only+ * defined the sha3 intrinsics when sha3 is enabled in the main target.+ */+# if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \+ !defined(__ARM_FEATURE_SHA3)+# undef veor3q_u8+# define veor3q_u8(a, b, c) \+ ({ uint8x16_t res; \+ __asm__("eor3 %0.16b, %1.16b, %2.16b, %3.16b" \+ : "=w" (res) : "w" (a), "w" (b), "w" (c)); \+ res; })+# pragma clang diagnostic ignored "-Wgnu-statement-expression"+# endif+#else+# define HAVE_SHA3_INTRIN 0+#endif++/* dotprod */+#ifdef __ARM_FEATURE_DOTPROD+# define HAVE_DOTPROD(features) 1+#else+# define HAVE_DOTPROD(features) ((features) & ARM_CPU_FEATURE_DOTPROD)+#endif+#if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \+ (GCC_PREREQ(8, 1) || CLANG_PREREQ(7, 0, 10010000) || defined(_MSC_VER))+# define HAVE_DOTPROD_INTRIN 1+ /*+ * Use an inline assembly fallback for clang 15 and earlier, which only+ * defined the dotprod intrinsics when dotprod is enabled in the main target.+ */+# if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \+ !defined(__ARM_FEATURE_DOTPROD)+# undef vdotq_u32+# define vdotq_u32(a, b, c) \+ ({ uint32x4_t res = (a); \+ __asm__("udot %0.4s, %1.16b, %2.16b" \+ : "+w" (res) : "w" (b), "w" (c)); \+ res; })+# pragma clang diagnostic ignored "-Wgnu-statement-expression"+# endif+#else+# define HAVE_DOTPROD_INTRIN 0+#endif++#endif /* ARCH_ARM32 || ARCH_ARM64 */++#endif /* LIB_ARM_CPU_FEATURES_H */diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/crc32_impl.h b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/crc32_impl.hnew file mode 100644index 0000000..696af5a--- /dev/null+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/crc32_impl.h@@ -0,0 +1,594 @@+/*+ * arm/crc32_impl.h - ARM implementations of the gzip CRC-32 algorithm+ *+ * Copyright 2022 Eric Biggers+ *+ * Permission is hereby granted, free of charge, to any person+ * obtaining a copy of this software and associated documentation+ * files (the "Software"), to deal in the Software without+ * restriction, including without limitation the rights to use,+ * copy, modify, merge, publish, distribute, sublicense, and/or sell+ * copies of the Software, and to permit persons to whom the+ * Software is furnished to do so, subject to the following+ * conditions:+ *+ * The above copyright notice and this permission notice shall be+ * included in all copies or substantial portions of the Software.+ *+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR+ * OTHER DEALINGS IN THE SOFTWARE.+ */++#ifndef LIB_ARM_CRC32_IMPL_H+#define LIB_ARM_CRC32_IMPL_H++#include "cpu_features.h"++/*+ * crc32_arm_crc() - implementation using crc32 instructions (only)+ *+ * In general this implementation is straightforward. However, naive use of the+ * crc32 instructions is serial: one of the two inputs to each crc32 instruction+ * is the output of the previous one. To take advantage of CPUs that can+ * execute multiple crc32 instructions in parallel, when possible we interleave+ * the checksumming of several adjacent chunks, then combine their CRCs.+ *+ * However, without pmull, combining CRCs is fairly slow. So in this pmull-less+ * version, we only use a large chunk length, and thus we only do chunked+ * processing if there is a lot of data to checksum. This also means that a+ * variable chunk length wouldn't help much, so we just support a fixed length.+ */+#if HAVE_CRC32_INTRIN+# ifdef __clang__+# define ATTRIBUTES _target_attribute("crc")+# else+# define ATTRIBUTES _target_attribute("+crc")+# endif++/*+ * Combine the CRCs for 4 adjacent chunks of length L = CRC32_FIXED_CHUNK_LEN+ * bytes each by computing:+ *+ * [ crc0*x^(3*8*L) + crc1*x^(2*8*L) + crc2*x^(1*8*L) + crc3 ] mod G(x)+ *+ * This has been optimized in several ways:+ *+ * - The needed multipliers (x to some power, reduced mod G(x)) were+ * precomputed.+ *+ * - The 3 multiplications are interleaved.+ *+ * - The reduction mod G(x) is delayed to the end and done using __crc32d.+ * Note that the use of __crc32d introduces an extra factor of x^32. To+ * cancel that out along with the extra factor of x^1 that gets introduced+ * because of how the 63-bit products are aligned in their 64-bit integers,+ * the multipliers are actually x^(j*8*L - 33) instead of x^(j*8*L).+ */+static forceinline ATTRIBUTES u32+combine_crcs_slow(u32 crc0, u32 crc1, u32 crc2, u32 crc3)+{+ u64 res0 = 0, res1 = 0, res2 = 0;+ int i;++ /* Multiply crc{0,1,2} by CRC32_FIXED_CHUNK_MULT_{3,2,1}. */+ for (i = 0; i < 32; i++) {+ if (CRC32_FIXED_CHUNK_MULT_3 & (1U << i))+ res0 ^= (u64)crc0 << i;+ if (CRC32_FIXED_CHUNK_MULT_2 & (1U << i))+ res1 ^= (u64)crc1 << i;+ if (CRC32_FIXED_CHUNK_MULT_1 & (1U << i))+ res2 ^= (u64)crc2 << i;+ }+ /* Add the different parts and reduce mod G(x). */+ return __crc32d(0, res0 ^ res1 ^ res2) ^ crc3;+}++#define crc32_arm_crc crc32_arm_crc+static ATTRIBUTES u32+crc32_arm_crc(u32 crc, const u8 *p, size_t len)+{+ if (len >= 64) {+ const size_t align = -(uintptr_t)p & 7;++ /* Align p to the next 8-byte boundary. */+ if (align) {+ if (align & 1)+ crc = __crc32b(crc, *p++);+ if (align & 2) {+ crc = __crc32h(crc, le16_bswap(*(u16 *)p));+ p += 2;+ }+ if (align & 4) {+ crc = __crc32w(crc, le32_bswap(*(u32 *)p));+ p += 4;+ }+ len -= align;+ }+ /*+ * Interleave the processing of multiple adjacent data chunks to+ * take advantage of instruction-level parallelism.+ *+ * Some CPUs don't prefetch the data if it's being fetched in+ * multiple interleaved streams, so do explicit prefetching.+ */+ while (len >= CRC32_NUM_CHUNKS * CRC32_FIXED_CHUNK_LEN) {+ const u64 *wp0 = (const u64 *)p;+ const u64 * const wp0_end =+ (const u64 *)(p + CRC32_FIXED_CHUNK_LEN);+ u32 crc1 = 0, crc2 = 0, crc3 = 0;++ STATIC_ASSERT(CRC32_NUM_CHUNKS == 4);+ STATIC_ASSERT(CRC32_FIXED_CHUNK_LEN % (4 * 8) == 0);+ do {+ prefetchr(&wp0[64 + 0*CRC32_FIXED_CHUNK_LEN/8]);+ prefetchr(&wp0[64 + 1*CRC32_FIXED_CHUNK_LEN/8]);+ prefetchr(&wp0[64 + 2*CRC32_FIXED_CHUNK_LEN/8]);+ prefetchr(&wp0[64 + 3*CRC32_FIXED_CHUNK_LEN/8]);+ crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_FIXED_CHUNK_LEN/8]));+ crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_FIXED_CHUNK_LEN/8]));+ crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_FIXED_CHUNK_LEN/8]));+ crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_FIXED_CHUNK_LEN/8]));+ wp0++;+ crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_FIXED_CHUNK_LEN/8]));+ crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_FIXED_CHUNK_LEN/8]));+ crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_FIXED_CHUNK_LEN/8]));+ crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_FIXED_CHUNK_LEN/8]));+ wp0++;+ crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_FIXED_CHUNK_LEN/8]));+ crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_FIXED_CHUNK_LEN/8]));+ crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_FIXED_CHUNK_LEN/8]));+ crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_FIXED_CHUNK_LEN/8]));+ wp0++;+ crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_FIXED_CHUNK_LEN/8]));+ crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_FIXED_CHUNK_LEN/8]));+ crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_FIXED_CHUNK_LEN/8]));+ crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_FIXED_CHUNK_LEN/8]));+ wp0++;+ } while (wp0 != wp0_end);+ crc = combine_crcs_slow(crc, crc1, crc2, crc3);+ p += CRC32_NUM_CHUNKS * CRC32_FIXED_CHUNK_LEN;+ len -= CRC32_NUM_CHUNKS * CRC32_FIXED_CHUNK_LEN;+ }+ /*+ * Due to the large fixed chunk length used above, there might+ * still be a lot of data left. So use a 64-byte loop here,+ * instead of a loop that is less unrolled.+ */+ while (len >= 64) {+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 0)));+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 8)));+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 16)));+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 24)));+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 32)));+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 40)));+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 48)));+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 56)));+ p += 64;+ len -= 64;+ }+ }+ if (len & 32) {+ crc = __crc32d(crc, get_unaligned_le64(p + 0));+ crc = __crc32d(crc, get_unaligned_le64(p + 8));+ crc = __crc32d(crc, get_unaligned_le64(p + 16));+ crc = __crc32d(crc, get_unaligned_le64(p + 24));+ p += 32;+ }+ if (len & 16) {+ crc = __crc32d(crc, get_unaligned_le64(p + 0));+ crc = __crc32d(crc, get_unaligned_le64(p + 8));+ p += 16;+ }+ if (len & 8) {+ crc = __crc32d(crc, get_unaligned_le64(p));+ p += 8;+ }+ if (len & 4) {+ crc = __crc32w(crc, get_unaligned_le32(p));+ p += 4;+ }+ if (len & 2) {+ crc = __crc32h(crc, get_unaligned_le16(p));+ p += 2;+ }+ if (len & 1)+ crc = __crc32b(crc, *p);+ return crc;+}+#undef ATTRIBUTES+#endif /* crc32_arm_crc() */++/*+ * crc32_arm_crc_pmullcombine() - implementation using crc32 instructions, plus+ * pmull instructions for CRC combining+ *+ * This is similar to crc32_arm_crc(), but it enables the use of pmull+ * (carryless multiplication) instructions for the steps where the CRCs of+ * adjacent data chunks are combined. As this greatly speeds up CRC+ * combination, this implementation also differs from crc32_arm_crc() in that it+ * uses a variable chunk length which can get fairly small. The precomputed+ * multipliers needed for the selected chunk length are loaded from a table.+ *+ * Note that pmull is used here only for combining the CRCs of separately+ * checksummed chunks, not for folding the data itself. See crc32_arm_pmull*()+ * for implementations that use pmull for folding the data itself.+ */+#if HAVE_CRC32_INTRIN && HAVE_PMULL_INTRIN+# ifdef __clang__+# define ATTRIBUTES _target_attribute("crc,aes")+# else+# define ATTRIBUTES _target_attribute("+crc,+crypto")+# endif++/* Do carryless multiplication of two 32-bit values. */+static forceinline ATTRIBUTES u64+clmul_u32(u32 a, u32 b)+{+ uint64x2_t res = vreinterpretq_u64_p128(+ compat_vmull_p64((poly64_t)a, (poly64_t)b));++ return vgetq_lane_u64(res, 0);+}++/*+ * Like combine_crcs_slow(), but uses vmull_p64 to do the multiplications more+ * quickly, and supports a variable chunk length. The chunk length is+ * 'i * CRC32_MIN_VARIABLE_CHUNK_LEN'+ * where 1 <= i < ARRAY_LEN(crc32_mults_for_chunklen).+ */+static forceinline ATTRIBUTES u32+combine_crcs_fast(u32 crc0, u32 crc1, u32 crc2, u32 crc3, size_t i)+{+ u64 res0 = clmul_u32(crc0, crc32_mults_for_chunklen[i][0]);+ u64 res1 = clmul_u32(crc1, crc32_mults_for_chunklen[i][1]);+ u64 res2 = clmul_u32(crc2, crc32_mults_for_chunklen[i][2]);++ return __crc32d(0, res0 ^ res1 ^ res2) ^ crc3;+}++#define crc32_arm_crc_pmullcombine crc32_arm_crc_pmullcombine+static ATTRIBUTES u32+crc32_arm_crc_pmullcombine(u32 crc, const u8 *p, size_t len)+{+ const size_t align = -(uintptr_t)p & 7;++ if (len >= align + CRC32_NUM_CHUNKS * CRC32_MIN_VARIABLE_CHUNK_LEN) {+ /* Align p to the next 8-byte boundary. */+ if (align) {+ if (align & 1)+ crc = __crc32b(crc, *p++);+ if (align & 2) {+ crc = __crc32h(crc, le16_bswap(*(u16 *)p));+ p += 2;+ }+ if (align & 4) {+ crc = __crc32w(crc, le32_bswap(*(u32 *)p));+ p += 4;+ }+ len -= align;+ }+ /*+ * Handle CRC32_MAX_VARIABLE_CHUNK_LEN specially, so that better+ * code is generated for it.+ */+ while (len >= CRC32_NUM_CHUNKS * CRC32_MAX_VARIABLE_CHUNK_LEN) {+ const u64 *wp0 = (const u64 *)p;+ const u64 * const wp0_end =+ (const u64 *)(p + CRC32_MAX_VARIABLE_CHUNK_LEN);+ u32 crc1 = 0, crc2 = 0, crc3 = 0;++ STATIC_ASSERT(CRC32_NUM_CHUNKS == 4);+ STATIC_ASSERT(CRC32_MAX_VARIABLE_CHUNK_LEN % (4 * 8) == 0);+ do {+ prefetchr(&wp0[64 + 0*CRC32_MAX_VARIABLE_CHUNK_LEN/8]);+ prefetchr(&wp0[64 + 1*CRC32_MAX_VARIABLE_CHUNK_LEN/8]);+ prefetchr(&wp0[64 + 2*CRC32_MAX_VARIABLE_CHUNK_LEN/8]);+ prefetchr(&wp0[64 + 3*CRC32_MAX_VARIABLE_CHUNK_LEN/8]);+ crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));+ crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));+ crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));+ crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));+ wp0++;+ crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));+ crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));+ crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));+ crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));+ wp0++;+ crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));+ crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));+ crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));+ crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));+ wp0++;+ crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));+ crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));+ crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));+ crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));+ wp0++;+ } while (wp0 != wp0_end);+ crc = combine_crcs_fast(crc, crc1, crc2, crc3,+ ARRAY_LEN(crc32_mults_for_chunklen) - 1);+ p += CRC32_NUM_CHUNKS * CRC32_MAX_VARIABLE_CHUNK_LEN;+ len -= CRC32_NUM_CHUNKS * CRC32_MAX_VARIABLE_CHUNK_LEN;+ }+ /* Handle up to one variable-length chunk. */+ if (len >= CRC32_NUM_CHUNKS * CRC32_MIN_VARIABLE_CHUNK_LEN) {+ const size_t i = len / (CRC32_NUM_CHUNKS *+ CRC32_MIN_VARIABLE_CHUNK_LEN);+ const size_t chunk_len =+ i * CRC32_MIN_VARIABLE_CHUNK_LEN;+ const u64 *wp0 = (const u64 *)(p + 0*chunk_len);+ const u64 *wp1 = (const u64 *)(p + 1*chunk_len);+ const u64 *wp2 = (const u64 *)(p + 2*chunk_len);+ const u64 *wp3 = (const u64 *)(p + 3*chunk_len);+ const u64 * const wp0_end = wp1;+ u32 crc1 = 0, crc2 = 0, crc3 = 0;++ STATIC_ASSERT(CRC32_NUM_CHUNKS == 4);+ STATIC_ASSERT(CRC32_MIN_VARIABLE_CHUNK_LEN % (4 * 8) == 0);+ do {+ prefetchr(wp0 + 64);+ prefetchr(wp1 + 64);+ prefetchr(wp2 + 64);+ prefetchr(wp3 + 64);+ crc = __crc32d(crc, le64_bswap(*wp0++));+ crc1 = __crc32d(crc1, le64_bswap(*wp1++));+ crc2 = __crc32d(crc2, le64_bswap(*wp2++));+ crc3 = __crc32d(crc3, le64_bswap(*wp3++));+ crc = __crc32d(crc, le64_bswap(*wp0++));+ crc1 = __crc32d(crc1, le64_bswap(*wp1++));+ crc2 = __crc32d(crc2, le64_bswap(*wp2++));+ crc3 = __crc32d(crc3, le64_bswap(*wp3++));+ crc = __crc32d(crc, le64_bswap(*wp0++));+ crc1 = __crc32d(crc1, le64_bswap(*wp1++));+ crc2 = __crc32d(crc2, le64_bswap(*wp2++));+ crc3 = __crc32d(crc3, le64_bswap(*wp3++));+ crc = __crc32d(crc, le64_bswap(*wp0++));+ crc1 = __crc32d(crc1, le64_bswap(*wp1++));+ crc2 = __crc32d(crc2, le64_bswap(*wp2++));+ crc3 = __crc32d(crc3, le64_bswap(*wp3++));+ } while (wp0 != wp0_end);+ crc = combine_crcs_fast(crc, crc1, crc2, crc3, i);+ p += CRC32_NUM_CHUNKS * chunk_len;+ len -= CRC32_NUM_CHUNKS * chunk_len;+ }++ while (len >= 32) {+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 0)));+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 8)));+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 16)));+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 24)));+ p += 32;+ len -= 32;+ }+ } else {+ while (len >= 32) {+ crc = __crc32d(crc, get_unaligned_le64(p + 0));+ crc = __crc32d(crc, get_unaligned_le64(p + 8));+ crc = __crc32d(crc, get_unaligned_le64(p + 16));+ crc = __crc32d(crc, get_unaligned_le64(p + 24));+ p += 32;+ len -= 32;+ }+ }+ if (len & 16) {+ crc = __crc32d(crc, get_unaligned_le64(p + 0));+ crc = __crc32d(crc, get_unaligned_le64(p + 8));+ p += 16;+ }+ if (len & 8) {+ crc = __crc32d(crc, get_unaligned_le64(p));+ p += 8;+ }+ if (len & 4) {+ crc = __crc32w(crc, get_unaligned_le32(p));+ p += 4;+ }+ if (len & 2) {+ crc = __crc32h(crc, get_unaligned_le16(p));+ p += 2;+ }+ if (len & 1)+ crc = __crc32b(crc, *p);+ return crc;+}+#undef ATTRIBUTES+#endif /* crc32_arm_crc_pmullcombine() */++/*+ * crc32_arm_pmullx4() - implementation using "folding" with pmull instructions+ *+ * This implementation is intended for CPUs that support pmull instructions but+ * not crc32 instructions.+ */+#if HAVE_PMULL_INTRIN+# define crc32_arm_pmullx4 crc32_arm_pmullx4+# define SUFFIX _pmullx4+# ifdef __clang__+ /*+ * This used to use "crypto", but that stopped working with clang 16.+ * Now only "aes" works. "aes" works with older versions too, so use+ * that. No "+" prefix; clang 15 and earlier doesn't accept that.+ */+# define ATTRIBUTES _target_attribute("aes")+# else+ /*+ * With gcc, only "+crypto" works. Both the "+" prefix and the+ * "crypto" (not "aes") are essential...+ */+# define ATTRIBUTES _target_attribute("+crypto")+# endif+# define ENABLE_EOR3 0+# include "crc32_pmull_helpers.h"++static ATTRIBUTES u32+crc32_arm_pmullx4(u32 crc, const u8 *p, size_t len)+{+ static const u64 _aligned_attribute(16) mults[3][2] = {+ { CRC32_X159_MODG, CRC32_X95_MODG }, /* 1 vecs */+ { CRC32_X543_MODG, CRC32_X479_MODG }, /* 4 vecs */+ { CRC32_X287_MODG, CRC32_X223_MODG }, /* 2 vecs */+ };+ static const u64 _aligned_attribute(16) barrett_consts[3][2] = {+ { CRC32_X95_MODG, },+ { CRC32_BARRETT_CONSTANT_1, },+ { CRC32_BARRETT_CONSTANT_2, },+ };+ const poly64x2_t multipliers_1 = load_multipliers(mults[0]);+ uint8x16_t v0, v1, v2, v3;++ if (len < 64 + 15) {+ if (len < 16)+ return crc32_slice1(crc, p, len);+ v0 = veorq_u8(vld1q_u8(p), u32_to_bytevec(crc));+ p += 16;+ len -= 16;+ while (len >= 16) {+ v0 = fold_vec(v0, vld1q_u8(p), multipliers_1);+ p += 16;+ len -= 16;+ }+ } else {+ const poly64x2_t multipliers_4 = load_multipliers(mults[1]);+ const poly64x2_t multipliers_2 = load_multipliers(mults[2]);+ const size_t align = -(uintptr_t)p & 15;+ const uint8x16_t *vp;++ v0 = veorq_u8(vld1q_u8(p), u32_to_bytevec(crc));+ p += 16;+ /* Align p to the next 16-byte boundary. */+ if (align) {+ v0 = fold_partial_vec(v0, p, align, multipliers_1);+ p += align;+ len -= align;+ }+ vp = (const uint8x16_t *)p;+ v1 = *vp++;+ v2 = *vp++;+ v3 = *vp++;+ while (len >= 64 + 64) {+ v0 = fold_vec(v0, *vp++, multipliers_4);+ v1 = fold_vec(v1, *vp++, multipliers_4);+ v2 = fold_vec(v2, *vp++, multipliers_4);+ v3 = fold_vec(v3, *vp++, multipliers_4);+ len -= 64;+ }+ v0 = fold_vec(v0, v2, multipliers_2);+ v1 = fold_vec(v1, v3, multipliers_2);+ if (len & 32) {+ v0 = fold_vec(v0, *vp++, multipliers_2);+ v1 = fold_vec(v1, *vp++, multipliers_2);+ }+ v0 = fold_vec(v0, v1, multipliers_1);+ if (len & 16)+ v0 = fold_vec(v0, *vp++, multipliers_1);+ p = (const u8 *)vp;+ len &= 15;+ }++ /* Handle any remaining partial block now before reducing to 32 bits. */+ if (len)+ v0 = fold_partial_vec(v0, p, len, multipliers_1);++ /* Reduce to 32 bits, following lib/x86/crc32_pclmul_template.h */+ v0 = veorq_u8(clmul_low(v0, load_multipliers(barrett_consts[0])),+ vextq_u8(v0, vdupq_n_u8(0), 8));+ v1 = clmul_low(v0, load_multipliers(barrett_consts[1]));+ v1 = clmul_low(v1, load_multipliers(barrett_consts[2]));+ v0 = veorq_u8(v0, v1);+ return vgetq_lane_u32(vreinterpretq_u32_u8(v0), 2);+}+#undef SUFFIX+#undef ATTRIBUTES+#undef ENABLE_EOR3+#endif /* crc32_arm_pmullx4() */++/*+ * crc32_arm_pmullx12_crc() - large-stride implementation using "folding" with+ * pmull instructions, where crc32 instructions are also available+ *+ * See crc32_pmull_wide.h for explanation.+ */+#if HAVE_PMULL_INTRIN && HAVE_CRC32_INTRIN+# define crc32_arm_pmullx12_crc crc32_arm_pmullx12_crc+# define SUFFIX _pmullx12_crc+# ifdef __clang__+# define ATTRIBUTES _target_attribute("aes,crc")+# else+# define ATTRIBUTES _target_attribute("+crypto,+crc")+# endif+# define ENABLE_EOR3 0+# include "crc32_pmull_wide.h"+#endif++/*+ * crc32_arm_pmullx12_crc_eor3()+ *+ * This like crc32_arm_pmullx12_crc(), but it adds the eor3 instruction (from+ * the sha3 extension) for even better performance.+ */+#if HAVE_PMULL_INTRIN && HAVE_CRC32_INTRIN && HAVE_SHA3_INTRIN && \+ !defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_SHA3)+# define crc32_arm_pmullx12_crc_eor3 crc32_arm_pmullx12_crc_eor3+# define SUFFIX _pmullx12_crc_eor3+# ifdef __clang__+# define ATTRIBUTES _target_attribute("aes,crc,sha3")+ /*+ * Both gcc and binutils originally considered sha3 to depend on+ * arch=armv8.2-a or later. This was fixed in gcc 13.2 by commit+ * 9aac37ab8a7b ("aarch64: Remove architecture dependencies from intrinsics")+ * and in binutils 2.41 by commit 205e4380c800 ("aarch64: Remove version+ * dependencies from features"). Unfortunately, always using arch=armv8.2-a+ * causes build errors with some compiler options because it may reduce the+ * arch rather than increase it. Therefore we try to omit the arch whenever+ * possible. If gcc is 14 or later, then both gcc and binutils are probably+ * fixed, so we omit the arch. We also omit the arch if a feature that+ * depends on armv8.2-a or later (in gcc 13.1 and earlier) is present.+ */+# elif GCC_PREREQ(14, 0) || defined(__ARM_FEATURE_JCVT) \+ || defined(__ARM_FEATURE_DOTPROD)+# define ATTRIBUTES _target_attribute("+crypto,+crc,+sha3")+# else+# define ATTRIBUTES _target_attribute("arch=armv8.2-a+crypto+crc+sha3")+# endif+# define ENABLE_EOR3 1+# include "crc32_pmull_wide.h"+#endif++static inline crc32_func_t+arch_select_crc32_func(void)+{+ const u32 features MAYBE_UNUSED = get_arm_cpu_features();++#ifdef crc32_arm_pmullx12_crc_eor3+ if ((features & ARM_CPU_FEATURE_PREFER_PMULL) &&+ HAVE_PMULL(features) && HAVE_CRC32(features) && HAVE_SHA3(features))+ return crc32_arm_pmullx12_crc_eor3;+#endif+#ifdef crc32_arm_pmullx12_crc+ if ((features & ARM_CPU_FEATURE_PREFER_PMULL) &&+ HAVE_PMULL(features) && HAVE_CRC32(features))+ return crc32_arm_pmullx12_crc;+#endif+#ifdef crc32_arm_crc_pmullcombine+ if (HAVE_CRC32(features) && HAVE_PMULL(features))+ return crc32_arm_crc_pmullcombine;+#endif+#ifdef crc32_arm_crc+ if (HAVE_CRC32(features))+ return crc32_arm_crc;+#endif+#ifdef crc32_arm_pmullx4+ if (HAVE_PMULL(features))+ return crc32_arm_pmullx4;+#endif+ return NULL;+}+#define arch_select_crc32_func arch_select_crc32_func++#endif /* LIB_ARM_CRC32_IMPL_H */diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/crc32_pmull_helpers.h b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/crc32_pmull_helpers.hnew file mode 100644index 0000000..023b904--- /dev/null+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/crc32_pmull_helpers.h@@ -0,0 +1,156 @@+/*+ * arm/crc32_pmull_helpers.h - helper functions for CRC-32 folding with PMULL+ *+ * Copyright 2022 Eric Biggers+ *+ * Permission is hereby granted, free of charge, to any person+ * obtaining a copy of this software and associated documentation+ * files (the "Software"), to deal in the Software without+ * restriction, including without limitation the rights to use,+ * copy, modify, merge, publish, distribute, sublicense, and/or sell+ * copies of the Software, and to permit persons to whom the+ * Software is furnished to do so, subject to the following+ * conditions:+ *+ * The above copyright notice and this permission notice shall be+ * included in all copies or substantial portions of the Software.+ *+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR+ * OTHER DEALINGS IN THE SOFTWARE.+ */++/*+ * This file is a "template" for instantiating helper functions for CRC folding+ * with pmull instructions. It accepts the following parameters:+ *+ * SUFFIX:+ * Name suffix to append to all instantiated functions.+ * ATTRIBUTES:+ * Target function attributes to use.+ * ENABLE_EOR3:+ * Use the eor3 instruction (from the sha3 extension).+ */++/* Create a vector with 'a' in the first 4 bytes, and the rest zeroed out. */+#undef u32_to_bytevec+static forceinline ATTRIBUTES uint8x16_t+ADD_SUFFIX(u32_to_bytevec)(u32 a)+{+ return vreinterpretq_u8_u32(vsetq_lane_u32(a, vdupq_n_u32(0), 0));+}+#define u32_to_bytevec ADD_SUFFIX(u32_to_bytevec)++/* Load two 64-bit values into a vector. */+#undef load_multipliers+static forceinline ATTRIBUTES poly64x2_t+ADD_SUFFIX(load_multipliers)(const u64 p[2])+{+ return vreinterpretq_p64_u64(vld1q_u64(p));+}+#define load_multipliers ADD_SUFFIX(load_multipliers)++/* Do carryless multiplication of the low halves of two vectors. */+#undef clmul_low+static forceinline ATTRIBUTES uint8x16_t+ADD_SUFFIX(clmul_low)(uint8x16_t a, poly64x2_t b)+{+ return vreinterpretq_u8_p128(+ compat_vmull_p64(vgetq_lane_p64(vreinterpretq_p64_u8(a), 0),+ vgetq_lane_p64(b, 0)));+}+#define clmul_low ADD_SUFFIX(clmul_low)++/* Do carryless multiplication of the high halves of two vectors. */+#undef clmul_high+static forceinline ATTRIBUTES uint8x16_t+ADD_SUFFIX(clmul_high)(uint8x16_t a, poly64x2_t b)+{+#ifdef __clang__+ /*+ * Use inline asm to ensure that pmull2 is really used. This works+ * around clang bug https://github.com/llvm/llvm-project/issues/52868.+ */+ uint8x16_t res;++ __asm__("pmull2 %0.1q, %1.2d, %2.2d" : "=w" (res) : "w" (a), "w" (b));+ return res;+#else+ return vreinterpretq_u8_p128(vmull_high_p64(vreinterpretq_p64_u8(a), b));+#endif+}+#define clmul_high ADD_SUFFIX(clmul_high)++#undef eor3+static forceinline ATTRIBUTES uint8x16_t+ADD_SUFFIX(eor3)(uint8x16_t a, uint8x16_t b, uint8x16_t c)+{+#if ENABLE_EOR3+ return veor3q_u8(a, b, c);+#else+ return veorq_u8(veorq_u8(a, b), c);+#endif+}+#define eor3 ADD_SUFFIX(eor3)++#undef fold_vec+static forceinline ATTRIBUTES uint8x16_t+ADD_SUFFIX(fold_vec)(uint8x16_t src, uint8x16_t dst, poly64x2_t multipliers)+{+ uint8x16_t a = clmul_low(src, multipliers);+ uint8x16_t b = clmul_high(src, multipliers);++ return eor3(a, b, dst);+}+#define fold_vec ADD_SUFFIX(fold_vec)++/*+ * Given v containing a 16-byte polynomial, and a pointer 'p' that points to the+ * next '1 <= len <= 15' data bytes, rearrange the concatenation of v and the+ * data into vectors x0 and x1 that contain 'len' bytes and 16 bytes,+ * respectively. Then fold x0 into x1 and return the result. Assumes that+ * 'p + len - 16' is in-bounds.+ */+#undef fold_partial_vec+static forceinline ATTRIBUTES MAYBE_UNUSED uint8x16_t+ADD_SUFFIX(fold_partial_vec)(uint8x16_t v, const u8 *p, size_t len,+ poly64x2_t multipliers_1)+{+ /*+ * vqtbl1q_u8(v, shift_tab[len..len+15]) left shifts v by 16-len bytes.+ * vqtbl1q_u8(v, shift_tab[len+16..len+31]) right shifts v by len bytes.+ */+ static const u8 shift_tab[48] = {+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,+ };+ const uint8x16_t lshift = vld1q_u8(&shift_tab[len]);+ const uint8x16_t rshift = vld1q_u8(&shift_tab[len + 16]);+ uint8x16_t x0, x1, bsl_mask;++ /* x0 = v left-shifted by '16 - len' bytes */+ x0 = vqtbl1q_u8(v, lshift);++ /* Create a vector of '16 - len' 0x00 bytes, then 'len' 0xff bytes. */+ bsl_mask = vreinterpretq_u8_s8(+ vshrq_n_s8(vreinterpretq_s8_u8(rshift), 7));++ /*+ * x1 = the last '16 - len' bytes from v (i.e. v right-shifted by 'len'+ * bytes) followed by the remaining data.+ */+ x1 = vbslq_u8(bsl_mask /* 0 bits select from arg3, 1 bits from arg2 */,+ vld1q_u8(p + len - 16), vqtbl1q_u8(v, rshift));++ return fold_vec(x0, x1, multipliers_1);+}+#define fold_partial_vec ADD_SUFFIX(fold_partial_vec)diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/crc32_pmull_wide.h b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/crc32_pmull_wide.hnew file mode 100644index 0000000..300e282--- /dev/null+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/crc32_pmull_wide.h@@ -0,0 +1,226 @@+/*+ * arm/crc32_pmull_wide.h - gzip CRC-32 with PMULL (extra-wide version)+ *+ * Copyright 2022 Eric Biggers+ *+ * Permission is hereby granted, free of charge, to any person+ * obtaining a copy of this software and associated documentation+ * files (the "Software"), to deal in the Software without+ * restriction, including without limitation the rights to use,+ * copy, modify, merge, publish, distribute, sublicense, and/or sell+ * copies of the Software, and to permit persons to whom the+ * Software is furnished to do so, subject to the following+ * conditions:+ *+ * The above copyright notice and this permission notice shall be+ * included in all copies or substantial portions of the Software.+ *+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR+ * OTHER DEALINGS IN THE SOFTWARE.+ */++/*+ * This file is a "template" for instantiating PMULL-based crc32_arm functions.+ * The "parameters" are:+ *+ * SUFFIX:+ * Name suffix to append to all instantiated functions.+ * ATTRIBUTES:+ * Target function attributes to use.+ * ENABLE_EOR3:+ * Use the eor3 instruction (from the sha3 extension).+ *+ * This is the extra-wide version; it uses an unusually large stride length of+ * 12, and it assumes that crc32 instructions are available too. It's intended+ * for powerful CPUs that support both pmull and crc32 instructions, but where+ * throughput of pmull and xor (given enough instructions issued in parallel) is+ * significantly higher than that of crc32, thus making the crc32 instructions+ * (counterintuitively) not actually the fastest way to compute the CRC-32. The+ * Apple M1 processor is an example of such a CPU.+ */++#include "crc32_pmull_helpers.h"++static ATTRIBUTES u32+ADD_SUFFIX(crc32_arm)(u32 crc, const u8 *p, size_t len)+{+ uint8x16_t v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11;++ if (len < 3 * 192) {+ static const u64 _aligned_attribute(16) mults[3][2] = {+ { CRC32_X543_MODG, CRC32_X479_MODG }, /* 4 vecs */+ { CRC32_X287_MODG, CRC32_X223_MODG }, /* 2 vecs */+ { CRC32_X159_MODG, CRC32_X95_MODG }, /* 1 vecs */+ };+ poly64x2_t multipliers_4, multipliers_2, multipliers_1;++ if (len < 64)+ goto tail;+ multipliers_4 = load_multipliers(mults[0]);+ multipliers_2 = load_multipliers(mults[1]);+ multipliers_1 = load_multipliers(mults[2]);+ /*+ * Short length; don't bother aligning the pointer, and fold+ * 64 bytes (4 vectors) at a time, at most.+ */+ v0 = veorq_u8(vld1q_u8(p + 0), u32_to_bytevec(crc));+ v1 = vld1q_u8(p + 16);+ v2 = vld1q_u8(p + 32);+ v3 = vld1q_u8(p + 48);+ p += 64;+ len -= 64;+ while (len >= 64) {+ v0 = fold_vec(v0, vld1q_u8(p + 0), multipliers_4);+ v1 = fold_vec(v1, vld1q_u8(p + 16), multipliers_4);+ v2 = fold_vec(v2, vld1q_u8(p + 32), multipliers_4);+ v3 = fold_vec(v3, vld1q_u8(p + 48), multipliers_4);+ p += 64;+ len -= 64;+ }+ v0 = fold_vec(v0, v2, multipliers_2);+ v1 = fold_vec(v1, v3, multipliers_2);+ if (len >= 32) {+ v0 = fold_vec(v0, vld1q_u8(p + 0), multipliers_2);+ v1 = fold_vec(v1, vld1q_u8(p + 16), multipliers_2);+ p += 32;+ len -= 32;+ }+ v0 = fold_vec(v0, v1, multipliers_1);+ } else {+ static const u64 _aligned_attribute(16) mults[4][2] = {+ { CRC32_X1567_MODG, CRC32_X1503_MODG }, /* 12 vecs */+ { CRC32_X799_MODG, CRC32_X735_MODG }, /* 6 vecs */+ { CRC32_X415_MODG, CRC32_X351_MODG }, /* 3 vecs */+ { CRC32_X159_MODG, CRC32_X95_MODG }, /* 1 vecs */+ };+ const poly64x2_t multipliers_12 = load_multipliers(mults[0]);+ const poly64x2_t multipliers_6 = load_multipliers(mults[1]);+ const poly64x2_t multipliers_3 = load_multipliers(mults[2]);+ const poly64x2_t multipliers_1 = load_multipliers(mults[3]);+ const size_t align = -(uintptr_t)p & 15;+ const uint8x16_t *vp;++ /* Align p to the next 16-byte boundary. */+ if (align) {+ if (align & 1)+ crc = __crc32b(crc, *p++);+ if (align & 2) {+ crc = __crc32h(crc, le16_bswap(*(u16 *)p));+ p += 2;+ }+ if (align & 4) {+ crc = __crc32w(crc, le32_bswap(*(u32 *)p));+ p += 4;+ }+ if (align & 8) {+ crc = __crc32d(crc, le64_bswap(*(u64 *)p));+ p += 8;+ }+ len -= align;+ }+ vp = (const uint8x16_t *)p;+ v0 = veorq_u8(*vp++, u32_to_bytevec(crc));+ v1 = *vp++;+ v2 = *vp++;+ v3 = *vp++;+ v4 = *vp++;+ v5 = *vp++;+ v6 = *vp++;+ v7 = *vp++;+ v8 = *vp++;+ v9 = *vp++;+ v10 = *vp++;+ v11 = *vp++;+ len -= 192;+ /* Fold 192 bytes (12 vectors) at a time. */+ do {+ v0 = fold_vec(v0, *vp++, multipliers_12);+ v1 = fold_vec(v1, *vp++, multipliers_12);+ v2 = fold_vec(v2, *vp++, multipliers_12);+ v3 = fold_vec(v3, *vp++, multipliers_12);+ v4 = fold_vec(v4, *vp++, multipliers_12);+ v5 = fold_vec(v5, *vp++, multipliers_12);+ v6 = fold_vec(v6, *vp++, multipliers_12);+ v7 = fold_vec(v7, *vp++, multipliers_12);+ v8 = fold_vec(v8, *vp++, multipliers_12);+ v9 = fold_vec(v9, *vp++, multipliers_12);+ v10 = fold_vec(v10, *vp++, multipliers_12);+ v11 = fold_vec(v11, *vp++, multipliers_12);+ len -= 192;+ } while (len >= 192);++ /*+ * Fewer than 192 bytes left. Fold v0-v11 down to just v0,+ * while processing up to 144 more bytes.+ */+ v0 = fold_vec(v0, v6, multipliers_6);+ v1 = fold_vec(v1, v7, multipliers_6);+ v2 = fold_vec(v2, v8, multipliers_6);+ v3 = fold_vec(v3, v9, multipliers_6);+ v4 = fold_vec(v4, v10, multipliers_6);+ v5 = fold_vec(v5, v11, multipliers_6);+ if (len >= 96) {+ v0 = fold_vec(v0, *vp++, multipliers_6);+ v1 = fold_vec(v1, *vp++, multipliers_6);+ v2 = fold_vec(v2, *vp++, multipliers_6);+ v3 = fold_vec(v3, *vp++, multipliers_6);+ v4 = fold_vec(v4, *vp++, multipliers_6);+ v5 = fold_vec(v5, *vp++, multipliers_6);+ len -= 96;+ }+ v0 = fold_vec(v0, v3, multipliers_3);+ v1 = fold_vec(v1, v4, multipliers_3);+ v2 = fold_vec(v2, v5, multipliers_3);+ if (len >= 48) {+ v0 = fold_vec(v0, *vp++, multipliers_3);+ v1 = fold_vec(v1, *vp++, multipliers_3);+ v2 = fold_vec(v2, *vp++, multipliers_3);+ len -= 48;+ }+ v0 = fold_vec(v0, v1, multipliers_1);+ v0 = fold_vec(v0, v2, multipliers_1);+ p = (const u8 *)vp;+ }+ /* Reduce 128 to 32 bits using crc32 instructions. */+ crc = __crc32d(0, vgetq_lane_u64(vreinterpretq_u64_u8(v0), 0));+ crc = __crc32d(crc, vgetq_lane_u64(vreinterpretq_u64_u8(v0), 1));+tail:+ /* Finish up the remainder using crc32 instructions. */+ if (len & 32) {+ crc = __crc32d(crc, get_unaligned_le64(p + 0));+ crc = __crc32d(crc, get_unaligned_le64(p + 8));+ crc = __crc32d(crc, get_unaligned_le64(p + 16));+ crc = __crc32d(crc, get_unaligned_le64(p + 24));+ p += 32;+ }+ if (len & 16) {+ crc = __crc32d(crc, get_unaligned_le64(p + 0));+ crc = __crc32d(crc, get_unaligned_le64(p + 8));+ p += 16;+ }+ if (len & 8) {+ crc = __crc32d(crc, get_unaligned_le64(p));+ p += 8;+ }+ if (len & 4) {+ crc = __crc32w(crc, get_unaligned_le32(p));+ p += 4;+ }+ if (len & 2) {+ crc = __crc32h(crc, get_unaligned_le16(p));+ p += 2;+ }+ if (len & 1)+ crc = __crc32b(crc, *p);+ return crc;+}++#undef SUFFIX+#undef ATTRIBUTES+#undef ENABLE_EOR3diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/matchfinder_impl.h b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/matchfinder_impl.hnew file mode 100644index 0000000..79c1dbc--- /dev/null+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/matchfinder_impl.h@@ -0,0 +1,78 @@+/*+ * arm/matchfinder_impl.h - ARM implementations of matchfinder functions+ *+ * Copyright 2016 Eric Biggers+ *+ * Permission is hereby granted, free of charge, to any person+ * obtaining a copy of this software and associated documentation+ * files (the "Software"), to deal in the Software without+ * restriction, including without limitation the rights to use,+ * copy, modify, merge, publish, distribute, sublicense, and/or sell+ * copies of the Software, and to permit persons to whom the+ * Software is furnished to do so, subject to the following+ * conditions:+ *+ * The above copyright notice and this permission notice shall be+ * included in all copies or substantial portions of the Software.+ *+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR+ * OTHER DEALINGS IN THE SOFTWARE.+ */++#ifndef LIB_ARM_MATCHFINDER_IMPL_H+#define LIB_ARM_MATCHFINDER_IMPL_H++#include "cpu_features.h"++#if HAVE_NEON_NATIVE+static forceinline void+matchfinder_init_neon(mf_pos_t *data, size_t size)+{+ int16x8_t *p = (int16x8_t *)data;+ int16x8_t v = vdupq_n_s16(MATCHFINDER_INITVAL);++ STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);+ STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);+ STATIC_ASSERT(sizeof(mf_pos_t) == 2);++ do {+ p[0] = v;+ p[1] = v;+ p[2] = v;+ p[3] = v;+ p += 4;+ size -= 4 * sizeof(*p);+ } while (size != 0);+}+#define matchfinder_init matchfinder_init_neon++static forceinline void+matchfinder_rebase_neon(mf_pos_t *data, size_t size)+{+ int16x8_t *p = (int16x8_t *)data;+ int16x8_t v = vdupq_n_s16((u16)-MATCHFINDER_WINDOW_SIZE);++ STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);+ STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);+ STATIC_ASSERT(sizeof(mf_pos_t) == 2);++ do {+ p[0] = vqaddq_s16(p[0], v);+ p[1] = vqaddq_s16(p[1], v);+ p[2] = vqaddq_s16(p[2], v);+ p[3] = vqaddq_s16(p[3], v);+ p += 4;+ size -= 4 * sizeof(*p);+ } while (size != 0);+}+#define matchfinder_rebase matchfinder_rebase_neon++#endif /* HAVE_NEON_NATIVE */++#endif /* LIB_ARM_MATCHFINDER_IMPL_H */diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/cpu_features_common.h b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/cpu_features_common.hnew file mode 100644index 0000000..d0531d5--- /dev/null+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/cpu_features_common.h@@ -0,0 +1,93 @@+/*+ * cpu_features_common.h - code shared by all lib/$arch/cpu_features.c+ *+ * Copyright 2020 Eric Biggers+ *+ * Permission is hereby granted, free of charge, to any person+ * obtaining a copy of this software and associated documentation+ * files (the "Software"), to deal in the Software without+ * restriction, including without limitation the rights to use,+ * copy, modify, merge, publish, distribute, sublicense, and/or sell+ * copies of the Software, and to permit persons to whom the+ * Software is furnished to do so, subject to the following+ * conditions:+ *+ * The above copyright notice and this permission notice shall be+ * included in all copies or substantial portions of the Software.+ *+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR+ * OTHER DEALINGS IN THE SOFTWARE.+ */++#ifndef LIB_CPU_FEATURES_COMMON_H+#define LIB_CPU_FEATURES_COMMON_H++#if defined(TEST_SUPPORT__DO_NOT_USE) && !defined(FREESTANDING)+ /* for strdup() and strtok_r() */+# undef _ANSI_SOURCE+# ifndef __APPLE__+# undef _GNU_SOURCE+# define _GNU_SOURCE+# endif+# include <stdio.h>+# include <stdlib.h>+# include <string.h>+#endif++#include "lib_common.h"++struct cpu_feature {+ u32 bit;+ const char *name;+};++#if defined(TEST_SUPPORT__DO_NOT_USE) && !defined(FREESTANDING)+/* Disable any features that are listed in $LIBDEFLATE_DISABLE_CPU_FEATURES. */+static inline void+disable_cpu_features_for_testing(u32 *features,+ const struct cpu_feature *feature_table,+ size_t feature_table_length)+{+ char *env_value, *strbuf, *p, *saveptr = NULL;+ size_t i;++ env_value = getenv("LIBDEFLATE_DISABLE_CPU_FEATURES");+ if (!env_value)+ return;+ strbuf = strdup(env_value);+ if (!strbuf)+ abort();+ p = strtok_r(strbuf, ",", &saveptr);+ while (p) {+ for (i = 0; i < feature_table_length; i++) {+ if (strcmp(p, feature_table[i].name) == 0) {+ *features &= ~feature_table[i].bit;+ break;+ }+ }+ if (i == feature_table_length) {+ fprintf(stderr,+ "unrecognized feature in LIBDEFLATE_DISABLE_CPU_FEATURES: \"%s\"\n",+ p);+ abort();+ }+ p = strtok_r(NULL, ",", &saveptr);+ }+ free(strbuf);+}+#else /* TEST_SUPPORT__DO_NOT_USE */+static inline void+disable_cpu_features_for_testing(u32 *features,+ const struct cpu_feature *feature_table,+ size_t feature_table_length)+{+}+#endif /* !TEST_SUPPORT__DO_NOT_USE */++#endif /* LIB_CPU_FEATURES_COMMON_H */diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/decompress_template.h b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/decompress_template.hnew file mode 100644index 0000000..8c874c3--- /dev/null+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/decompress_template.h@@ -0,0 +1,777 @@+/*+ * decompress_template.h+ *+ * Copyright 2016 Eric Biggers+ *+ * Permission is hereby granted, free of charge, to any person+ * obtaining a copy of this software and associated documentation+ * files (the "Software"), to deal in the Software without+ * restriction, including without limitation the rights to use,+ * copy, modify, merge, publish, distribute, sublicense, and/or sell+ * copies of the Software, and to permit persons to whom the+ * Software is furnished to do so, subject to the following+ * conditions:+ *+ * The above copyright notice and this permission notice shall be+ * included in all copies or substantial portions of the Software.+ *+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR+ * OTHER DEALINGS IN THE SOFTWARE.+ */++/*+ * This is the actual DEFLATE decompression routine, lifted out of+ * deflate_decompress.c so that it can be compiled multiple times with different+ * target instruction sets.+ */++#ifndef ATTRIBUTES+# define ATTRIBUTES+#endif+#ifndef EXTRACT_VARBITS+# define EXTRACT_VARBITS(word, count) ((word) & BITMASK(count))+#endif+#ifndef EXTRACT_VARBITS8+# define EXTRACT_VARBITS8(word, count) ((word) & BITMASK((u8)(count)))+#endif++static ATTRIBUTES MAYBE_UNUSED enum libdeflate_result+FUNCNAME(struct libdeflate_decompressor * restrict d,+ const void * restrict in, size_t in_nbytes,+ void * restrict out, size_t out_nbytes_avail,+ size_t *actual_in_nbytes_ret, size_t *actual_out_nbytes_ret)+{+ u8 *out_next = out;+ u8 * const out_end = out_next + out_nbytes_avail;+ u8 * const out_fastloop_end =+ out_end - MIN(out_nbytes_avail, FASTLOOP_MAX_BYTES_WRITTEN);++ /* Input bitstream state; see deflate_decompress.c for documentation */+ const u8 *in_next = in;+ const u8 * const in_end = in_next + in_nbytes;+ const u8 * const in_fastloop_end =+ in_end - MIN(in_nbytes, FASTLOOP_MAX_BYTES_READ);+ bitbuf_t bitbuf = 0;+ bitbuf_t saved_bitbuf;+ u32 bitsleft = 0;+ size_t overread_count = 0;++ bool is_final_block;+ unsigned block_type;+ unsigned num_litlen_syms;+ unsigned num_offset_syms;+ bitbuf_t litlen_tablemask;+ u32 entry;++next_block:+ /* Starting to read the next block */+ ;++ STATIC_ASSERT(CAN_CONSUME(1 + 2 + 5 + 5 + 4 + 3));+ REFILL_BITS();++ /* BFINAL: 1 bit */+ is_final_block = bitbuf & BITMASK(1);++ /* BTYPE: 2 bits */+ block_type = (bitbuf >> 1) & BITMASK(2);++ if (block_type == DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN) {++ /* Dynamic Huffman block */++ /* The order in which precode lengths are stored */+ static const u8 deflate_precode_lens_permutation[DEFLATE_NUM_PRECODE_SYMS] = {+ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15+ };++ unsigned num_explicit_precode_lens;+ unsigned i;++ /* Read the codeword length counts. */++ STATIC_ASSERT(DEFLATE_NUM_LITLEN_SYMS == 257 + BITMASK(5));+ num_litlen_syms = 257 + ((bitbuf >> 3) & BITMASK(5));++ STATIC_ASSERT(DEFLATE_NUM_OFFSET_SYMS == 1 + BITMASK(5));+ num_offset_syms = 1 + ((bitbuf >> 8) & BITMASK(5));++ STATIC_ASSERT(DEFLATE_NUM_PRECODE_SYMS == 4 + BITMASK(4));+ num_explicit_precode_lens = 4 + ((bitbuf >> 13) & BITMASK(4));++ d->static_codes_loaded = false;++ /*+ * Read the precode codeword lengths.+ *+ * A 64-bit bitbuffer is just one bit too small to hold the+ * maximum number of precode lens, so to minimize branches we+ * merge one len with the previous fields.+ */+ STATIC_ASSERT(DEFLATE_MAX_PRE_CODEWORD_LEN == (1 << 3) - 1);+ if (CAN_CONSUME(3 * (DEFLATE_NUM_PRECODE_SYMS - 1))) {+ d->u.precode_lens[deflate_precode_lens_permutation[0]] =+ (bitbuf >> 17) & BITMASK(3);+ bitbuf >>= 20;+ bitsleft -= 20;+ REFILL_BITS();+ i = 1;+ do {+ d->u.precode_lens[deflate_precode_lens_permutation[i]] =+ bitbuf & BITMASK(3);+ bitbuf >>= 3;+ bitsleft -= 3;+ } while (++i < num_explicit_precode_lens);+ } else {+ bitbuf >>= 17;+ bitsleft -= 17;+ i = 0;+ do {+ if ((u8)bitsleft < 3)+ REFILL_BITS();+ d->u.precode_lens[deflate_precode_lens_permutation[i]] =+ bitbuf & BITMASK(3);+ bitbuf >>= 3;+ bitsleft -= 3;+ } while (++i < num_explicit_precode_lens);+ }+ for (; i < DEFLATE_NUM_PRECODE_SYMS; i++)+ d->u.precode_lens[deflate_precode_lens_permutation[i]] = 0;++ /* Build the decode table for the precode. */+ SAFETY_CHECK(build_precode_decode_table(d));++ /* Decode the litlen and offset codeword lengths. */+ i = 0;+ do {+ unsigned presym;+ u8 rep_val;+ unsigned rep_count;++ if ((u8)bitsleft < DEFLATE_MAX_PRE_CODEWORD_LEN + 7)+ REFILL_BITS();++ /*+ * The code below assumes that the precode decode table+ * doesn't have any subtables.+ */+ STATIC_ASSERT(PRECODE_TABLEBITS == DEFLATE_MAX_PRE_CODEWORD_LEN);++ /* Decode the next precode symbol. */+ entry = d->u.l.precode_decode_table[+ bitbuf & BITMASK(DEFLATE_MAX_PRE_CODEWORD_LEN)];+ bitbuf >>= (u8)entry;+ bitsleft -= entry; /* optimization: subtract full entry */+ presym = entry >> 16;++ if (presym < 16) {+ /* Explicit codeword length */+ d->u.l.lens[i++] = presym;+ continue;+ }++ /* Run-length encoded codeword lengths */++ /*+ * Note: we don't need to immediately verify that the+ * repeat count doesn't overflow the number of elements,+ * since we've sized the lens array to have enough extra+ * space to allow for the worst-case overrun (138 zeroes+ * when only 1 length was remaining).+ *+ * In the case of the small repeat counts (presyms 16+ * and 17), it is fastest to always write the maximum+ * number of entries. That gets rid of branches that+ * would otherwise be required.+ *+ * It is not just because of the numerical order that+ * our checks go in the order 'presym < 16', 'presym ==+ * 16', and 'presym == 17'. For typical data this is+ * ordered from most frequent to least frequent case.+ */+ STATIC_ASSERT(DEFLATE_MAX_LENS_OVERRUN == 138 - 1);++ if (presym == 16) {+ /* Repeat the previous length 3 - 6 times. */+ SAFETY_CHECK(i != 0);+ rep_val = d->u.l.lens[i - 1];+ STATIC_ASSERT(3 + BITMASK(2) == 6);+ rep_count = 3 + (bitbuf & BITMASK(2));+ bitbuf >>= 2;+ bitsleft -= 2;+ d->u.l.lens[i + 0] = rep_val;+ d->u.l.lens[i + 1] = rep_val;+ d->u.l.lens[i + 2] = rep_val;+ d->u.l.lens[i + 3] = rep_val;+ d->u.l.lens[i + 4] = rep_val;+ d->u.l.lens[i + 5] = rep_val;+ i += rep_count;+ } else if (presym == 17) {+ /* Repeat zero 3 - 10 times. */+ STATIC_ASSERT(3 + BITMASK(3) == 10);+ rep_count = 3 + (bitbuf & BITMASK(3));+ bitbuf >>= 3;+ bitsleft -= 3;+ d->u.l.lens[i + 0] = 0;+ d->u.l.lens[i + 1] = 0;+ d->u.l.lens[i + 2] = 0;+ d->u.l.lens[i + 3] = 0;+ d->u.l.lens[i + 4] = 0;+ d->u.l.lens[i + 5] = 0;+ d->u.l.lens[i + 6] = 0;+ d->u.l.lens[i + 7] = 0;+ d->u.l.lens[i + 8] = 0;+ d->u.l.lens[i + 9] = 0;+ i += rep_count;+ } else {+ /* Repeat zero 11 - 138 times. */+ STATIC_ASSERT(11 + BITMASK(7) == 138);+ rep_count = 11 + (bitbuf & BITMASK(7));+ bitbuf >>= 7;+ bitsleft -= 7;+ memset(&d->u.l.lens[i], 0,+ rep_count * sizeof(d->u.l.lens[i]));+ i += rep_count;+ }+ } while (i < num_litlen_syms + num_offset_syms);++ /* Unnecessary, but check this for consistency with zlib. */+ SAFETY_CHECK(i == num_litlen_syms + num_offset_syms);++ } else if (block_type == DEFLATE_BLOCKTYPE_UNCOMPRESSED) {+ u16 len, nlen;++ /*+ * Uncompressed block: copy 'len' bytes literally from the input+ * buffer to the output buffer.+ */++ bitsleft -= 3; /* for BTYPE and BFINAL */++ /*+ * Align the bitstream to the next byte boundary. This means+ * the next byte boundary as if we were reading a byte at a+ * time. Therefore, we have to rewind 'in_next' by any bytes+ * that have been refilled but not actually consumed yet (not+ * counting overread bytes, which don't increment 'in_next').+ */+ bitsleft = (u8)bitsleft;+ SAFETY_CHECK(overread_count <= (bitsleft >> 3));+ in_next -= (bitsleft >> 3) - overread_count;+ overread_count = 0;+ bitbuf = 0;+ bitsleft = 0;++ SAFETY_CHECK(in_end - in_next >= 4);+ len = get_unaligned_le16(in_next);+ nlen = get_unaligned_le16(in_next + 2);+ in_next += 4;++ SAFETY_CHECK(len == (u16)~nlen);+ if (unlikely(len > out_end - out_next))+ return LIBDEFLATE_INSUFFICIENT_SPACE;+ SAFETY_CHECK(len <= in_end - in_next);++ memcpy(out_next, in_next, len);+ in_next += len;+ out_next += len;++ goto block_done;++ } else {+ unsigned i;++ SAFETY_CHECK(block_type == DEFLATE_BLOCKTYPE_STATIC_HUFFMAN);++ /*+ * Static Huffman block: build the decode tables for the static+ * codes. Skip doing so if the tables are already set up from+ * an earlier static block; this speeds up decompression of+ * degenerate input of many empty or very short static blocks.+ *+ * Afterwards, the remainder is the same as decompressing a+ * dynamic Huffman block.+ */++ bitbuf >>= 3; /* for BTYPE and BFINAL */+ bitsleft -= 3;++ if (d->static_codes_loaded)+ goto have_decode_tables;++ d->static_codes_loaded = true;++ STATIC_ASSERT(DEFLATE_NUM_LITLEN_SYMS == 288);+ STATIC_ASSERT(DEFLATE_NUM_OFFSET_SYMS == 32);++ for (i = 0; i < 144; i++)+ d->u.l.lens[i] = 8;+ for (; i < 256; i++)+ d->u.l.lens[i] = 9;+ for (; i < 280; i++)+ d->u.l.lens[i] = 7;+ for (; i < 288; i++)+ d->u.l.lens[i] = 8;++ for (; i < 288 + 32; i++)+ d->u.l.lens[i] = 5;++ num_litlen_syms = 288;+ num_offset_syms = 32;+ }++ /* Decompressing a Huffman block (either dynamic or static) */++ SAFETY_CHECK(build_offset_decode_table(d, num_litlen_syms, num_offset_syms));+ SAFETY_CHECK(build_litlen_decode_table(d, num_litlen_syms, num_offset_syms));+have_decode_tables:+ litlen_tablemask = BITMASK(d->litlen_tablebits);++ /*+ * This is the "fastloop" for decoding literals and matches. It does+ * bounds checks on in_next and out_next in the loop conditions so that+ * additional bounds checks aren't needed inside the loop body.+ *+ * To reduce latency, the bitbuffer is refilled and the next litlen+ * decode table entry is preloaded before each loop iteration.+ */+ if (in_next >= in_fastloop_end || out_next >= out_fastloop_end)+ goto generic_loop;+ REFILL_BITS_IN_FASTLOOP();+ entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];+ do {+ u32 length, offset, lit;+ const u8 *src;+ u8 *dst;++ /*+ * Consume the bits for the litlen decode table entry. Save the+ * original bitbuf for later, in case the extra match length+ * bits need to be extracted from it.+ */+ saved_bitbuf = bitbuf;+ bitbuf >>= (u8)entry;+ bitsleft -= entry; /* optimization: subtract full entry */++ /*+ * Begin by checking for a "fast" literal, i.e. a literal that+ * doesn't need a subtable.+ */+ if (entry & HUFFDEC_LITERAL) {+ /*+ * On 64-bit platforms, we decode up to 2 extra fast+ * literals in addition to the primary item, as this+ * increases performance and still leaves enough bits+ * remaining for what follows. We could actually do 3,+ * assuming LITLEN_TABLEBITS=11, but that actually+ * decreases performance slightly (perhaps by messing+ * with the branch prediction of the conditional refill+ * that happens later while decoding the match offset).+ *+ * Note: the definitions of FASTLOOP_MAX_BYTES_WRITTEN+ * and FASTLOOP_MAX_BYTES_READ need to be updated if the+ * number of extra literals decoded here is changed.+ */+ if (/* enough bits for 2 fast literals + length + offset preload? */+ CAN_CONSUME_AND_THEN_PRELOAD(2 * LITLEN_TABLEBITS ++ LENGTH_MAXBITS,+ OFFSET_TABLEBITS) &&+ /* enough bits for 2 fast literals + slow literal + litlen preload? */+ CAN_CONSUME_AND_THEN_PRELOAD(2 * LITLEN_TABLEBITS ++ DEFLATE_MAX_LITLEN_CODEWORD_LEN,+ LITLEN_TABLEBITS)) {+ /* 1st extra fast literal */+ lit = entry >> 16;+ entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];+ saved_bitbuf = bitbuf;+ bitbuf >>= (u8)entry;+ bitsleft -= entry;+ *out_next++ = lit;+ if (entry & HUFFDEC_LITERAL) {+ /* 2nd extra fast literal */+ lit = entry >> 16;+ entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];+ saved_bitbuf = bitbuf;+ bitbuf >>= (u8)entry;+ bitsleft -= entry;+ *out_next++ = lit;+ if (entry & HUFFDEC_LITERAL) {+ /*+ * Another fast literal, but+ * this one is in lieu of the+ * primary item, so it doesn't+ * count as one of the extras.+ */+ lit = entry >> 16;+ entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];+ REFILL_BITS_IN_FASTLOOP();+ *out_next++ = lit;+ continue;+ }+ }+ } else {+ /*+ * Decode a literal. While doing so, preload+ * the next litlen decode table entry and refill+ * the bitbuffer. To reduce latency, we've+ * arranged for there to be enough "preloadable"+ * bits remaining to do the table preload+ * independently of the refill.+ */+ STATIC_ASSERT(CAN_CONSUME_AND_THEN_PRELOAD(+ LITLEN_TABLEBITS, LITLEN_TABLEBITS));+ lit = entry >> 16;+ entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];+ REFILL_BITS_IN_FASTLOOP();+ *out_next++ = lit;+ continue;+ }+ }++ /*+ * It's not a literal entry, so it can be a length entry, a+ * subtable pointer entry, or an end-of-block entry. Detect the+ * two unlikely cases by testing the HUFFDEC_EXCEPTIONAL flag.+ */+ if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {+ /* Subtable pointer or end-of-block entry */++ if (unlikely(entry & HUFFDEC_END_OF_BLOCK))+ goto block_done;++ /*+ * A subtable is required. Load and consume the+ * subtable entry. The subtable entry can be of any+ * type: literal, length, or end-of-block.+ */+ entry = d->u.litlen_decode_table[(entry >> 16) ++ EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];+ saved_bitbuf = bitbuf;+ bitbuf >>= (u8)entry;+ bitsleft -= entry;++ /*+ * 32-bit platforms that use the byte-at-a-time refill+ * method have to do a refill here for there to always+ * be enough bits to decode a literal that requires a+ * subtable, then preload the next litlen decode table+ * entry; or to decode a match length that requires a+ * subtable, then preload the offset decode table entry.+ */+ if (!CAN_CONSUME_AND_THEN_PRELOAD(DEFLATE_MAX_LITLEN_CODEWORD_LEN,+ LITLEN_TABLEBITS) ||+ !CAN_CONSUME_AND_THEN_PRELOAD(LENGTH_MAXBITS,+ OFFSET_TABLEBITS))+ REFILL_BITS_IN_FASTLOOP();+ if (entry & HUFFDEC_LITERAL) {+ /* Decode a literal that required a subtable. */+ lit = entry >> 16;+ entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];+ REFILL_BITS_IN_FASTLOOP();+ *out_next++ = lit;+ continue;+ }+ if (unlikely(entry & HUFFDEC_END_OF_BLOCK))+ goto block_done;+ /* Else, it's a length that required a subtable. */+ }++ /*+ * Decode the match length: the length base value associated+ * with the litlen symbol (which we extract from the decode+ * table entry), plus the extra length bits. We don't need to+ * consume the extra length bits here, as they were included in+ * the bits consumed by the entry earlier. We also don't need+ * to check for too-long matches here, as this is inside the+ * fastloop where it's already been verified that the output+ * buffer has enough space remaining to copy a max-length match.+ */+ length = entry >> 16;+ length += EXTRACT_VARBITS8(saved_bitbuf, entry) >> (u8)(entry >> 8);++ /*+ * Decode the match offset. There are enough "preloadable" bits+ * remaining to preload the offset decode table entry, but a+ * refill might be needed before consuming it.+ */+ STATIC_ASSERT(CAN_CONSUME_AND_THEN_PRELOAD(LENGTH_MAXFASTBITS,+ OFFSET_TABLEBITS));+ entry = d->offset_decode_table[bitbuf & BITMASK(OFFSET_TABLEBITS)];+ if (CAN_CONSUME_AND_THEN_PRELOAD(OFFSET_MAXBITS,+ LITLEN_TABLEBITS)) {+ /*+ * Decoding a match offset on a 64-bit platform. We may+ * need to refill once, but then we can decode the whole+ * offset and preload the next litlen table entry.+ */+ if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {+ /* Offset codeword requires a subtable */+ if (unlikely((u8)bitsleft < OFFSET_MAXBITS ++ LITLEN_TABLEBITS - PRELOAD_SLACK))+ REFILL_BITS_IN_FASTLOOP();+ bitbuf >>= OFFSET_TABLEBITS;+ bitsleft -= OFFSET_TABLEBITS;+ entry = d->offset_decode_table[(entry >> 16) ++ EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];+ } else if (unlikely((u8)bitsleft < OFFSET_MAXFASTBITS ++ LITLEN_TABLEBITS - PRELOAD_SLACK))+ REFILL_BITS_IN_FASTLOOP();+ } else {+ /* Decoding a match offset on a 32-bit platform */+ REFILL_BITS_IN_FASTLOOP();+ if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {+ /* Offset codeword requires a subtable */+ bitbuf >>= OFFSET_TABLEBITS;+ bitsleft -= OFFSET_TABLEBITS;+ entry = d->offset_decode_table[(entry >> 16) ++ EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];+ REFILL_BITS_IN_FASTLOOP();+ /* No further refill needed before extra bits */+ STATIC_ASSERT(CAN_CONSUME(+ OFFSET_MAXBITS - OFFSET_TABLEBITS));+ } else {+ /* No refill needed before extra bits */+ STATIC_ASSERT(CAN_CONSUME(OFFSET_MAXFASTBITS));+ }+ }+ saved_bitbuf = bitbuf;+ bitbuf >>= (u8)entry;+ bitsleft -= entry; /* optimization: subtract full entry */+ offset = entry >> 16;+ offset += EXTRACT_VARBITS8(saved_bitbuf, entry) >> (u8)(entry >> 8);++ /* Validate the match offset; needed even in the fastloop. */+ SAFETY_CHECK(offset <= out_next - (const u8 *)out);+ src = out_next - offset;+ dst = out_next;+ out_next += length;++ /*+ * Before starting to issue the instructions to copy the match,+ * refill the bitbuffer and preload the litlen decode table+ * entry for the next loop iteration. This can increase+ * performance by allowing the latency of the match copy to+ * overlap with these other operations. To further reduce+ * latency, we've arranged for there to be enough bits remaining+ * to do the table preload independently of the refill, except+ * on 32-bit platforms using the byte-at-a-time refill method.+ */+ if (!CAN_CONSUME_AND_THEN_PRELOAD(+ MAX(OFFSET_MAXBITS - OFFSET_TABLEBITS,+ OFFSET_MAXFASTBITS),+ LITLEN_TABLEBITS) &&+ unlikely((u8)bitsleft < LITLEN_TABLEBITS - PRELOAD_SLACK))+ REFILL_BITS_IN_FASTLOOP();+ entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];+ REFILL_BITS_IN_FASTLOOP();++ /*+ * Copy the match. On most CPUs the fastest method is a+ * word-at-a-time copy, unconditionally copying about 5 words+ * since this is enough for most matches without being too much.+ *+ * The normal word-at-a-time copy works for offset >= WORDBYTES,+ * which is most cases. The case of offset == 1 is also common+ * and is worth optimizing for, since it is just RLE encoding of+ * the previous byte, which is the result of compressing long+ * runs of the same byte.+ *+ * Writing past the match 'length' is allowed here, since it's+ * been ensured there is enough output space left for a slight+ * overrun. FASTLOOP_MAX_BYTES_WRITTEN needs to be updated if+ * the maximum possible overrun here is changed.+ */+ if (UNALIGNED_ACCESS_IS_FAST && offset >= WORDBYTES) {+ store_word_unaligned(load_word_unaligned(src), dst);+ src += WORDBYTES;+ dst += WORDBYTES;+ store_word_unaligned(load_word_unaligned(src), dst);+ src += WORDBYTES;+ dst += WORDBYTES;+ store_word_unaligned(load_word_unaligned(src), dst);+ src += WORDBYTES;+ dst += WORDBYTES;+ store_word_unaligned(load_word_unaligned(src), dst);+ src += WORDBYTES;+ dst += WORDBYTES;+ store_word_unaligned(load_word_unaligned(src), dst);+ src += WORDBYTES;+ dst += WORDBYTES;+ while (dst < out_next) {+ store_word_unaligned(load_word_unaligned(src), dst);+ src += WORDBYTES;+ dst += WORDBYTES;+ store_word_unaligned(load_word_unaligned(src), dst);+ src += WORDBYTES;+ dst += WORDBYTES;+ store_word_unaligned(load_word_unaligned(src), dst);+ src += WORDBYTES;+ dst += WORDBYTES;+ store_word_unaligned(load_word_unaligned(src), dst);+ src += WORDBYTES;+ dst += WORDBYTES;+ store_word_unaligned(load_word_unaligned(src), dst);+ src += WORDBYTES;+ dst += WORDBYTES;+ }+ } else if (UNALIGNED_ACCESS_IS_FAST && offset == 1) {+ machine_word_t v;++ /*+ * This part tends to get auto-vectorized, so keep it+ * copying a multiple of 16 bytes at a time.+ */+ v = (machine_word_t)0x0101010101010101 * src[0];+ store_word_unaligned(v, dst);+ dst += WORDBYTES;+ store_word_unaligned(v, dst);+ dst += WORDBYTES;+ store_word_unaligned(v, dst);+ dst += WORDBYTES;+ store_word_unaligned(v, dst);+ dst += WORDBYTES;+ while (dst < out_next) {+ store_word_unaligned(v, dst);+ dst += WORDBYTES;+ store_word_unaligned(v, dst);+ dst += WORDBYTES;+ store_word_unaligned(v, dst);+ dst += WORDBYTES;+ store_word_unaligned(v, dst);+ dst += WORDBYTES;+ }+ } else if (UNALIGNED_ACCESS_IS_FAST) {+ store_word_unaligned(load_word_unaligned(src), dst);+ src += offset;+ dst += offset;+ store_word_unaligned(load_word_unaligned(src), dst);+ src += offset;+ dst += offset;+ do {+ store_word_unaligned(load_word_unaligned(src), dst);+ src += offset;+ dst += offset;+ store_word_unaligned(load_word_unaligned(src), dst);+ src += offset;+ dst += offset;+ } while (dst < out_next);+ } else {+ *dst++ = *src++;+ *dst++ = *src++;+ do {+ *dst++ = *src++;+ } while (dst < out_next);+ }+ } while (in_next < in_fastloop_end && out_next < out_fastloop_end);++ /*+ * This is the generic loop for decoding literals and matches. This+ * handles cases where in_next and out_next are close to the end of+ * their respective buffers. Usually this loop isn't performance-+ * critical, as most time is spent in the fastloop above instead. We+ * therefore omit some optimizations here in favor of smaller code.+ */+generic_loop:+ for (;;) {+ u32 length, offset;+ const u8 *src;+ u8 *dst;++ REFILL_BITS();+ entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];+ saved_bitbuf = bitbuf;+ bitbuf >>= (u8)entry;+ bitsleft -= entry;+ if (unlikely(entry & HUFFDEC_SUBTABLE_POINTER)) {+ entry = d->u.litlen_decode_table[(entry >> 16) ++ EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];+ saved_bitbuf = bitbuf;+ bitbuf >>= (u8)entry;+ bitsleft -= entry;+ }+ length = entry >> 16;+ if (entry & HUFFDEC_LITERAL) {+ if (unlikely(out_next == out_end))+ return LIBDEFLATE_INSUFFICIENT_SPACE;+ *out_next++ = length;+ continue;+ }+ if (unlikely(entry & HUFFDEC_END_OF_BLOCK))+ goto block_done;+ length += EXTRACT_VARBITS8(saved_bitbuf, entry) >> (u8)(entry >> 8);+ if (unlikely(length > out_end - out_next))+ return LIBDEFLATE_INSUFFICIENT_SPACE;++ if (!CAN_CONSUME(LENGTH_MAXBITS + OFFSET_MAXBITS))+ REFILL_BITS();+ entry = d->offset_decode_table[bitbuf & BITMASK(OFFSET_TABLEBITS)];+ if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {+ bitbuf >>= OFFSET_TABLEBITS;+ bitsleft -= OFFSET_TABLEBITS;+ entry = d->offset_decode_table[(entry >> 16) ++ EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];+ if (!CAN_CONSUME(OFFSET_MAXBITS))+ REFILL_BITS();+ }+ offset = entry >> 16;+ offset += EXTRACT_VARBITS8(bitbuf, entry) >> (u8)(entry >> 8);+ bitbuf >>= (u8)entry;+ bitsleft -= entry;++ SAFETY_CHECK(offset <= out_next - (const u8 *)out);+ src = out_next - offset;+ dst = out_next;+ out_next += length;++ STATIC_ASSERT(DEFLATE_MIN_MATCH_LEN == 3);+ *dst++ = *src++;+ *dst++ = *src++;+ do {+ *dst++ = *src++;+ } while (dst < out_next);+ }++block_done:+ /* Finished decoding a block */++ if (!is_final_block)+ goto next_block;++ /* That was the last block. */++ bitsleft = (u8)bitsleft;++ /*+ * If any of the implicit appended zero bytes were consumed (not just+ * refilled) before hitting end of stream, then the data is bad.+ */+ SAFETY_CHECK(overread_count <= (bitsleft >> 3));++ /* Optionally return the actual number of bytes consumed. */+ if (actual_in_nbytes_ret) {+ /* Don't count bytes that were refilled but not consumed. */+ in_next -= (bitsleft >> 3) - overread_count;++ *actual_in_nbytes_ret = in_next - (u8 *)in;+ }++ /* Optionally return the actual number of bytes written. */+ if (actual_out_nbytes_ret) {+ *actual_out_nbytes_ret = out_next - (u8 *)out;+ } else {+ if (out_next != out_end)+ return LIBDEFLATE_SHORT_OUTPUT;+ }+ return LIBDEFLATE_SUCCESS;+}++#undef FUNCNAME+#undef ATTRIBUTES+#undef EXTRACT_VARBITS+#undef EXTRACT_VARBITS8diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/deflate_constants.h b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/deflate_constants.hnew file mode 100644index 0000000..95c9e0a--- /dev/null+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/deflate_constants.h@@ -0,0 +1,56 @@+/*+ * deflate_constants.h - constants for the DEFLATE compression format+ */++#ifndef LIB_DEFLATE_CONSTANTS_H+#define LIB_DEFLATE_CONSTANTS_H++/* Valid block types */+#define DEFLATE_BLOCKTYPE_UNCOMPRESSED 0+#define DEFLATE_BLOCKTYPE_STATIC_HUFFMAN 1+#define DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN 2++/* Minimum and maximum supported match lengths (in bytes) */+#define DEFLATE_MIN_MATCH_LEN 3+#define DEFLATE_MAX_MATCH_LEN 258++/* Maximum supported match offset (in bytes) */+#define DEFLATE_MAX_MATCH_OFFSET 32768++/* log2 of DEFLATE_MAX_MATCH_OFFSET */+#define DEFLATE_WINDOW_ORDER 15++/* Number of symbols in each Huffman code. Note: for the literal/length+ * and offset codes, these are actually the maximum values; a given block+ * might use fewer symbols. */+#define DEFLATE_NUM_PRECODE_SYMS 19+#define DEFLATE_NUM_LITLEN_SYMS 288+#define DEFLATE_NUM_OFFSET_SYMS 32++/* The maximum number of symbols across all codes */+#define DEFLATE_MAX_NUM_SYMS 288++/* Division of symbols in the literal/length code */+#define DEFLATE_NUM_LITERALS 256+#define DEFLATE_END_OF_BLOCK 256+#define DEFLATE_FIRST_LEN_SYM 257++/* Maximum codeword length, in bits, within each Huffman code */+#define DEFLATE_MAX_PRE_CODEWORD_LEN 7+#define DEFLATE_MAX_LITLEN_CODEWORD_LEN 15+#define DEFLATE_MAX_OFFSET_CODEWORD_LEN 15++/* The maximum codeword length across all codes */+#define DEFLATE_MAX_CODEWORD_LEN 15++/* Maximum possible overrun when decoding codeword lengths */+#define DEFLATE_MAX_LENS_OVERRUN 137++/*+ * Maximum number of extra bits that may be required to represent a match+ * length or offset.+ */+#define DEFLATE_MAX_EXTRA_LENGTH_BITS 5+#define DEFLATE_MAX_EXTRA_OFFSET_BITS 13++#endif /* LIB_DEFLATE_CONSTANTS_H */diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/deflate_decompress.c b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/deflate_decompress.cnew file mode 100644index 0000000..63726c7--- /dev/null+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/deflate_decompress.c@@ -0,0 +1,1208 @@+/*+ * deflate_decompress.c - a decompressor for DEFLATE+ *+ * Copyright 2016 Eric Biggers+ *+ * Permission is hereby granted, free of charge, to any person+ * obtaining a copy of this software and associated documentation+ * files (the "Software"), to deal in the Software without+ * restriction, including without limitation the rights to use,+ * copy, modify, merge, publish, distribute, sublicense, and/or sell+ * copies of the Software, and to permit persons to whom the+ * Software is furnished to do so, subject to the following+ * conditions:+ *+ * The above copyright notice and this permission notice shall be+ * included in all copies or substantial portions of the Software.+ *+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR+ * OTHER DEALINGS IN THE SOFTWARE.+ *+ * ---------------------------------------------------------------------------+ *+ * This is a highly optimized DEFLATE decompressor. It is much faster than+ * vanilla zlib, typically well over twice as fast, though results vary by CPU.+ *+ * Why this is faster than vanilla zlib:+ *+ * - Word accesses rather than byte accesses when reading input+ * - Word accesses rather than byte accesses when copying matches+ * - Faster Huffman decoding combined with various DEFLATE-specific tricks+ * - Larger bitbuffer variable that doesn't need to be refilled as often+ * - Other optimizations to remove unnecessary branches+ * - Only full-buffer decompression is supported, so the code doesn't need to+ * support stopping and resuming decompression.+ * - On x86_64, a version of the decompression routine is compiled with BMI2+ * instructions enabled and is used automatically at runtime when supported.+ */++#include "lib_common.h"+#include "deflate_constants.h"++/*+ * If the expression passed to SAFETY_CHECK() evaluates to false, then the+ * decompression routine immediately returns LIBDEFLATE_BAD_DATA, indicating the+ * compressed data is invalid.+ *+ * Theoretically, these checks could be disabled for specialized applications+ * where all input to the decompressor will be trusted.+ */+#if 0+# pragma message("UNSAFE DECOMPRESSION IS ENABLED. THIS MUST ONLY BE USED IF THE DECOMPRESSOR INPUT WILL ALWAYS BE TRUSTED!")+# define SAFETY_CHECK(expr) (void)(expr)+#else+# define SAFETY_CHECK(expr) if (unlikely(!(expr))) return LIBDEFLATE_BAD_DATA+#endif++/*****************************************************************************+ * Input bitstream *+ *****************************************************************************/++/*+ * The state of the "input bitstream" consists of the following variables:+ *+ * - in_next: a pointer to the next unread byte in the input buffer+ *+ * - in_end: a pointer to just past the end of the input buffer+ *+ * - bitbuf: a word-sized variable containing bits that have been read from+ * the input buffer or from the implicit appended zero bytes+ *+ * - bitsleft: the number of bits in 'bitbuf' available to be consumed.+ * After REFILL_BITS_BRANCHLESS(), 'bitbuf' can actually+ * contain more bits than this. However, only the bits counted+ * by 'bitsleft' can actually be consumed; the rest can only be+ * used for preloading.+ *+ * As a micro-optimization, we allow bits 8 and higher of+ * 'bitsleft' to contain garbage. When consuming the bits+ * associated with a decode table entry, this allows us to do+ * 'bitsleft -= entry' instead of 'bitsleft -= (u8)entry'.+ * On some CPUs, this helps reduce instruction dependencies.+ * This does have the disadvantage that 'bitsleft' sometimes+ * needs to be cast to 'u8', such as when it's used as a shift+ * amount in REFILL_BITS_BRANCHLESS(). But that one happens+ * for free since most CPUs ignore high bits in shift amounts.+ *+ * - overread_count: the total number of implicit appended zero bytes that+ * have been loaded into the bitbuffer, including any+ * counted by 'bitsleft' and any already consumed+ */++/*+ * The type for the bitbuffer variable ('bitbuf' described above). For best+ * performance, this should have size equal to a machine word.+ *+ * 64-bit platforms have a significant advantage: they get a bigger bitbuffer+ * which they don't have to refill as often.+ */+typedef machine_word_t bitbuf_t;+#define BITBUF_NBITS (8 * (int)sizeof(bitbuf_t))++/* BITMASK(n) returns a bitmask of length 'n'. */+#define BITMASK(n) (((bitbuf_t)1 << (n)) - 1)++/*+ * MAX_BITSLEFT is the maximum number of consumable bits, i.e. the maximum value+ * of '(u8)bitsleft'. This is the size of the bitbuffer variable, minus 1 if+ * the branchless refill method is being used (see REFILL_BITS_BRANCHLESS()).+ */+#define MAX_BITSLEFT \+ (UNALIGNED_ACCESS_IS_FAST ? BITBUF_NBITS - 1 : BITBUF_NBITS)++/*+ * CONSUMABLE_NBITS is the minimum number of bits that are guaranteed to be+ * consumable (counted in 'bitsleft') immediately after refilling the bitbuffer.+ * Since only whole bytes can be added to 'bitsleft', the worst case is+ * 'MAX_BITSLEFT - 7': the smallest amount where another byte doesn't fit.+ */+#define CONSUMABLE_NBITS (MAX_BITSLEFT - 7)++/*+ * FASTLOOP_PRELOADABLE_NBITS is the minimum number of bits that are guaranteed+ * to be preloadable immediately after REFILL_BITS_IN_FASTLOOP(). (It is *not*+ * guaranteed after REFILL_BITS(), since REFILL_BITS() falls back to a+ * byte-at-a-time refill method near the end of input.) This may exceed the+ * number of consumable bits (counted by 'bitsleft'). Any bits not counted in+ * 'bitsleft' can only be used for precomputation and cannot be consumed.+ */+#define FASTLOOP_PRELOADABLE_NBITS \+ (UNALIGNED_ACCESS_IS_FAST ? BITBUF_NBITS : CONSUMABLE_NBITS)++/*+ * PRELOAD_SLACK is the minimum number of bits that are guaranteed to be+ * preloadable but not consumable, following REFILL_BITS_IN_FASTLOOP() and any+ * subsequent consumptions. This is 1 bit if the branchless refill method is+ * being used, and 0 bits otherwise.+ */+#define PRELOAD_SLACK MAX(0, FASTLOOP_PRELOADABLE_NBITS - MAX_BITSLEFT)++/*+ * CAN_CONSUME(n) is true if it's guaranteed that if the bitbuffer has just been+ * refilled, then it's always possible to consume 'n' bits from it. 'n' should+ * be a compile-time constant, to enable compile-time evaluation.+ */+#define CAN_CONSUME(n) (CONSUMABLE_NBITS >= (n))++/*+ * CAN_CONSUME_AND_THEN_PRELOAD(consume_nbits, preload_nbits) is true if it's+ * guaranteed that after REFILL_BITS_IN_FASTLOOP(), it's always possible to+ * consume 'consume_nbits' bits, then preload 'preload_nbits' bits. The+ * arguments should be compile-time constants to enable compile-time evaluation.+ */+#define CAN_CONSUME_AND_THEN_PRELOAD(consume_nbits, preload_nbits) \+ (CONSUMABLE_NBITS >= (consume_nbits) && \+ FASTLOOP_PRELOADABLE_NBITS >= (consume_nbits) + (preload_nbits))++/*+ * REFILL_BITS_BRANCHLESS() branchlessly refills the bitbuffer variable by+ * reading the next word from the input buffer and updating 'in_next' and+ * 'bitsleft' based on how many bits were refilled -- counting whole bytes only.+ * This is much faster than reading a byte at a time, at least if the CPU is+ * little endian and supports fast unaligned memory accesses.+ *+ * The simplest way of branchlessly updating 'bitsleft' would be:+ *+ * bitsleft += (MAX_BITSLEFT - bitsleft) & ~7;+ *+ * To make it faster, we define MAX_BITSLEFT to be 'WORDBITS - 1' rather than+ * WORDBITS, so that in binary it looks like 111111 or 11111. Then, we update+ * 'bitsleft' by just setting the bits above the low 3 bits:+ *+ * bitsleft |= MAX_BITSLEFT & ~7;+ *+ * That compiles down to a single instruction like 'or $0x38, %rbp'. Using+ * 'MAX_BITSLEFT == WORDBITS - 1' also has the advantage that refills can be+ * done when 'bitsleft == MAX_BITSLEFT' without invoking undefined behavior.+ *+ * The simplest way of branchlessly updating 'in_next' would be:+ *+ * in_next += (MAX_BITSLEFT - bitsleft) >> 3;+ *+ * With 'MAX_BITSLEFT == WORDBITS - 1' we could use an XOR instead, though this+ * isn't really better:+ *+ * in_next += (MAX_BITSLEFT ^ bitsleft) >> 3;+ *+ * An alternative which can be marginally better is the following:+ *+ * in_next += sizeof(bitbuf_t) - 1;+ * in_next -= (bitsleft >> 3) & 0x7;+ *+ * It seems this would increase the number of CPU instructions from 3 (sub, shr,+ * add) to 4 (add, shr, and, sub). However, if the CPU has a bitfield+ * extraction instruction (e.g. arm's ubfx), it stays at 3, and is potentially+ * more efficient because the length of the longest dependency chain decreases+ * from 3 to 2. This alternative also has the advantage that it ignores the+ * high bits in 'bitsleft', so it is compatible with the micro-optimization we+ * use where we let the high bits of 'bitsleft' contain garbage.+ */+#define REFILL_BITS_BRANCHLESS() \+do { \+ bitbuf |= get_unaligned_leword(in_next) << (u8)bitsleft; \+ in_next += sizeof(bitbuf_t) - 1; \+ in_next -= (bitsleft >> 3) & 0x7; \+ bitsleft |= MAX_BITSLEFT & ~7; \+} while (0)++/*+ * REFILL_BITS() loads bits from the input buffer until the bitbuffer variable+ * contains at least CONSUMABLE_NBITS consumable bits.+ *+ * This checks for the end of input, and it doesn't guarantee+ * FASTLOOP_PRELOADABLE_NBITS, so it can't be used in the fastloop.+ *+ * If we would overread the input buffer, we just don't read anything, leaving+ * the bits zeroed but marking them filled. This simplifies the decompressor+ * because it removes the need to always be able to distinguish between real+ * overreads and overreads caused only by the decompressor's own lookahead.+ *+ * We do still keep track of the number of bytes that have been overread, for+ * two reasons. First, it allows us to determine the exact number of bytes that+ * were consumed once the stream ends or an uncompressed block is reached.+ * Second, it allows us to stop early if the overread amount gets so large (more+ * than sizeof bitbuf) that it can only be caused by a real overread. (The+ * second part is arguably unneeded, since libdeflate is buffer-based; given+ * infinite zeroes, it will eventually either completely fill the output buffer+ * or return an error. However, we do it to be slightly more friendly to the+ * not-recommended use case of decompressing with an unknown output size.)+ */+#define REFILL_BITS() \+do { \+ if (UNALIGNED_ACCESS_IS_FAST && \+ likely(in_end - in_next >= sizeof(bitbuf_t))) { \+ REFILL_BITS_BRANCHLESS(); \+ } else { \+ while ((u8)bitsleft < CONSUMABLE_NBITS) { \+ if (likely(in_next != in_end)) { \+ bitbuf |= (bitbuf_t)*in_next++ << \+ (u8)bitsleft; \+ } else { \+ overread_count++; \+ SAFETY_CHECK(overread_count <= \+ sizeof(bitbuf_t)); \+ } \+ bitsleft += 8; \+ } \+ } \+} while (0)++/*+ * REFILL_BITS_IN_FASTLOOP() is like REFILL_BITS(), but it doesn't check for the+ * end of the input. It can only be used in the fastloop.+ */+#define REFILL_BITS_IN_FASTLOOP() \+do { \+ STATIC_ASSERT(UNALIGNED_ACCESS_IS_FAST || \+ FASTLOOP_PRELOADABLE_NBITS == CONSUMABLE_NBITS); \+ if (UNALIGNED_ACCESS_IS_FAST) { \+ REFILL_BITS_BRANCHLESS(); \+ } else { \+ while ((u8)bitsleft < CONSUMABLE_NBITS) { \+ bitbuf |= (bitbuf_t)*in_next++ << (u8)bitsleft; \+ bitsleft += 8; \+ } \+ } \+} while (0)++/*+ * This is the worst-case maximum number of output bytes that are written to+ * during each iteration of the fastloop. The worst case is 2 literals, then a+ * match of length DEFLATE_MAX_MATCH_LEN. Additionally, some slack space must+ * be included for the intentional overrun in the match copy implementation.+ */+#define FASTLOOP_MAX_BYTES_WRITTEN \+ (2 + DEFLATE_MAX_MATCH_LEN + (5 * WORDBYTES) - 1)++/*+ * This is the worst-case maximum number of input bytes that are read during+ * each iteration of the fastloop. To get this value, we first compute the+ * greatest number of bits that can be refilled during a loop iteration. The+ * refill at the beginning can add at most MAX_BITSLEFT, and the amount that can+ * be refilled later is no more than the maximum amount that can be consumed by+ * 2 literals that don't need a subtable, then a match. We convert this value+ * to bytes, rounding up; this gives the maximum number of bytes that 'in_next'+ * can be advanced. Finally, we add sizeof(bitbuf_t) to account for+ * REFILL_BITS_BRANCHLESS() reading a word past 'in_next'.+ */+#define FASTLOOP_MAX_BYTES_READ \+ (DIV_ROUND_UP(MAX_BITSLEFT + (2 * LITLEN_TABLEBITS) + \+ LENGTH_MAXBITS + OFFSET_MAXBITS, 8) + \+ sizeof(bitbuf_t))++/*****************************************************************************+ * Huffman decoding *+ *****************************************************************************/++/*+ * The fastest way to decode Huffman-encoded data is basically to use a decode+ * table that maps the next TABLEBITS bits of data to their symbol. Each entry+ * decode_table[i] maps to the symbol whose codeword is a prefix of 'i'. A+ * symbol with codeword length 'n' has '2**(TABLEBITS-n)' entries in the table.+ *+ * Ideally, TABLEBITS and the maximum codeword length would be the same; some+ * compression formats are designed with this goal in mind. Unfortunately, in+ * DEFLATE, the maximum litlen and offset codeword lengths are 15 bits, which is+ * too large for a practical TABLEBITS. It's not *that* much larger, though, so+ * the workaround is to use a single level of subtables. In the main table,+ * entries for prefixes of codewords longer than TABLEBITS contain a "pointer"+ * to the appropriate subtable along with the number of bits it is indexed with.+ *+ * The most efficient way to allocate subtables is to allocate them dynamically+ * after the main table. The worst-case number of table entries needed,+ * including subtables, is precomputable; see the ENOUGH constants below.+ *+ * A useful optimization is to store the codeword lengths in the decode table so+ * that they don't have to be looked up by indexing a separate table that maps+ * symbols to their codeword lengths. We basically do this; however, for the+ * litlen and offset codes we also implement some DEFLATE-specific optimizations+ * that build in the consideration of the "extra bits" and the+ * literal/length/end-of-block division. For the exact decode table entry+ * format we use, see the definitions of the *_decode_results[] arrays below.+ */+++/*+ * These are the TABLEBITS values we use for each of the DEFLATE Huffman codes,+ * along with their corresponding ENOUGH values.+ *+ * For the precode, we use PRECODE_TABLEBITS == 7 since this is the maximum+ * precode codeword length. This avoids ever needing subtables.+ *+ * For the litlen and offset codes, we cannot realistically avoid ever needing+ * subtables, since litlen and offset codewords can be up to 15 bits. A higher+ * TABLEBITS reduces the number of lookups that need a subtable, which increases+ * performance; however, it increases memory usage and makes building the table+ * take longer, which decreases performance. We choose values that work well in+ * practice, making subtables rarely needed without making the tables too large.+ *+ * Our choice of OFFSET_TABLEBITS == 8 is a bit low; without any special+ * considerations, 9 would fit the trade-off curve better. However, there is a+ * performance benefit to using exactly 8 bits when it is a compile-time+ * constant, as many CPUs can take the low byte more easily than the low 9 bits.+ *+ * zlib treats its equivalents of TABLEBITS as maximum values; whenever it+ * builds a table, it caps the actual table_bits to the longest codeword. This+ * makes sense in theory, as there's no need for the table to be any larger than+ * needed to support the longest codeword. However, having the table bits be a+ * compile-time constant is beneficial to the performance of the decode loop, so+ * there is a trade-off. libdeflate currently uses the dynamic table_bits+ * strategy for the litlen table only, due to its larger maximum size.+ * PRECODE_TABLEBITS and OFFSET_TABLEBITS are smaller, so going dynamic there+ * isn't as useful, and OFFSET_TABLEBITS=8 is useful as mentioned above.+ *+ * Each TABLEBITS value has a corresponding ENOUGH value that gives the+ * worst-case maximum number of decode table entries, including the main table+ * and all subtables. The ENOUGH value depends on three parameters:+ *+ * (1) the maximum number of symbols in the code (DEFLATE_NUM_*_SYMS)+ * (2) the maximum number of main table bits (*_TABLEBITS)+ * (3) the maximum allowed codeword length (DEFLATE_MAX_*_CODEWORD_LEN)+ *+ * The ENOUGH values were computed using the utility program 'enough' from zlib.+ */+#define PRECODE_TABLEBITS 7+#define PRECODE_ENOUGH 128 /* enough 19 7 7 */+#define LITLEN_TABLEBITS 11+#define LITLEN_ENOUGH 2342 /* enough 288 11 15 */+#define OFFSET_TABLEBITS 8+#define OFFSET_ENOUGH 402 /* enough 32 8 15 */++/*+ * make_decode_table_entry() creates a decode table entry for the given symbol+ * by combining the static part 'decode_results[sym]' with the dynamic part+ * 'len', which is the remaining codeword length (the codeword length for main+ * table entries, or the codeword length minus TABLEBITS for subtable entries).+ *+ * In all cases, we add 'len' to each of the two low-order bytes to create the+ * appropriately-formatted decode table entry. See the definitions of the+ * *_decode_results[] arrays below, where the entry format is described.+ */+static forceinline u32+make_decode_table_entry(const u32 decode_results[], u32 sym, u32 len)+{+ return decode_results[sym] + (len << 8) + len;+}++/*+ * Here is the format of our precode decode table entries. Bits not explicitly+ * described contain zeroes:+ *+ * Bit 20-16: presym+ * Bit 10-8: codeword length [not used]+ * Bit 2-0: codeword length+ *+ * The precode decode table never has subtables, since we use+ * PRECODE_TABLEBITS == DEFLATE_MAX_PRE_CODEWORD_LEN.+ *+ * precode_decode_results[] contains the static part of the entry for each+ * symbol. make_decode_table_entry() produces the final entries.+ */+static const u32 precode_decode_results[] = {+#define ENTRY(presym) ((u32)presym << 16)+ ENTRY(0) , ENTRY(1) , ENTRY(2) , ENTRY(3) ,+ ENTRY(4) , ENTRY(5) , ENTRY(6) , ENTRY(7) ,+ ENTRY(8) , ENTRY(9) , ENTRY(10) , ENTRY(11) ,+ ENTRY(12) , ENTRY(13) , ENTRY(14) , ENTRY(15) ,+ ENTRY(16) , ENTRY(17) , ENTRY(18) ,+#undef ENTRY+};++/* Litlen and offset decode table entry flags */++/* Indicates a literal entry in the litlen decode table */+#define HUFFDEC_LITERAL 0x80000000++/* Indicates that HUFFDEC_SUBTABLE_POINTER or HUFFDEC_END_OF_BLOCK is set */+#define HUFFDEC_EXCEPTIONAL 0x00008000++/* Indicates a subtable pointer entry in the litlen or offset decode table */+#define HUFFDEC_SUBTABLE_POINTER 0x00004000++/* Indicates an end-of-block entry in the litlen decode table */+#define HUFFDEC_END_OF_BLOCK 0x00002000++/* Maximum number of bits that can be consumed by decoding a match length */+#define LENGTH_MAXBITS (DEFLATE_MAX_LITLEN_CODEWORD_LEN + \+ DEFLATE_MAX_EXTRA_LENGTH_BITS)+#define LENGTH_MAXFASTBITS (LITLEN_TABLEBITS /* no subtable needed */ + \+ DEFLATE_MAX_EXTRA_LENGTH_BITS)++/*+ * Here is the format of our litlen decode table entries. Bits not explicitly+ * described contain zeroes:+ *+ * Literals:+ * Bit 31: 1 (HUFFDEC_LITERAL)+ * Bit 23-16: literal value+ * Bit 15: 0 (!HUFFDEC_EXCEPTIONAL)+ * Bit 14: 0 (!HUFFDEC_SUBTABLE_POINTER)+ * Bit 13: 0 (!HUFFDEC_END_OF_BLOCK)+ * Bit 11-8: remaining codeword length [not used]+ * Bit 3-0: remaining codeword length+ * Lengths:+ * Bit 31: 0 (!HUFFDEC_LITERAL)+ * Bit 24-16: length base value+ * Bit 15: 0 (!HUFFDEC_EXCEPTIONAL)+ * Bit 14: 0 (!HUFFDEC_SUBTABLE_POINTER)+ * Bit 13: 0 (!HUFFDEC_END_OF_BLOCK)+ * Bit 11-8: remaining codeword length+ * Bit 4-0: remaining codeword length + number of extra bits+ * End of block:+ * Bit 31: 0 (!HUFFDEC_LITERAL)+ * Bit 15: 1 (HUFFDEC_EXCEPTIONAL)+ * Bit 14: 0 (!HUFFDEC_SUBTABLE_POINTER)+ * Bit 13: 1 (HUFFDEC_END_OF_BLOCK)+ * Bit 11-8: remaining codeword length [not used]+ * Bit 3-0: remaining codeword length+ * Subtable pointer:+ * Bit 31: 0 (!HUFFDEC_LITERAL)+ * Bit 30-16: index of start of subtable+ * Bit 15: 1 (HUFFDEC_EXCEPTIONAL)+ * Bit 14: 1 (HUFFDEC_SUBTABLE_POINTER)+ * Bit 13: 0 (!HUFFDEC_END_OF_BLOCK)+ * Bit 11-8: number of subtable bits+ * Bit 3-0: number of main table bits+ *+ * This format has several desirable properties:+ *+ * - The codeword length, length slot base, and number of extra length bits+ * are all built in. This eliminates the need to separately look up this+ * information by indexing separate arrays by symbol or length slot.+ *+ * - The HUFFDEC_* flags enable easily distinguishing between the different+ * types of entries. The HUFFDEC_LITERAL flag enables a fast path for+ * literals; the high bit is used for this, as some CPUs can test the+ * high bit more easily than other bits. The HUFFDEC_EXCEPTIONAL flag+ * makes it possible to detect the two unlikely cases (subtable pointer+ * and end of block) in a single bit flag test.+ *+ * - The low byte is the number of bits that need to be removed from the+ * bitstream; this makes this value easily accessible, and it enables the+ * micro-optimization of doing 'bitsleft -= entry' instead of+ * 'bitsleft -= (u8)entry'. It also includes the number of extra bits,+ * so they don't need to be removed separately.+ *+ * - The flags in bits 15-13 are arranged to be 0 when the+ * "remaining codeword length" in bits 11-8 is needed, making this value+ * fairly easily accessible as well via a shift and downcast.+ *+ * - Similarly, bits 13-12 are 0 when the "subtable bits" in bits 11-8 are+ * needed, making it possible to extract this value with '& 0x3F' rather+ * than '& 0xF'. This value is only used as a shift amount, so this can+ * save an 'and' instruction as the masking by 0x3F happens implicitly.+ *+ * litlen_decode_results[] contains the static part of the entry for each+ * symbol. make_decode_table_entry() produces the final entries.+ */+static const u32 litlen_decode_results[] = {++ /* Literals */+#define ENTRY(literal) (HUFFDEC_LITERAL | ((u32)literal << 16))+ ENTRY(0) , ENTRY(1) , ENTRY(2) , ENTRY(3) ,+ ENTRY(4) , ENTRY(5) , ENTRY(6) , ENTRY(7) ,+ ENTRY(8) , ENTRY(9) , ENTRY(10) , ENTRY(11) ,+ ENTRY(12) , ENTRY(13) , ENTRY(14) , ENTRY(15) ,+ ENTRY(16) , ENTRY(17) , ENTRY(18) , ENTRY(19) ,+ ENTRY(20) , ENTRY(21) , ENTRY(22) , ENTRY(23) ,+ ENTRY(24) , ENTRY(25) , ENTRY(26) , ENTRY(27) ,+ ENTRY(28) , ENTRY(29) , ENTRY(30) , ENTRY(31) ,+ ENTRY(32) , ENTRY(33) , ENTRY(34) , ENTRY(35) ,+ ENTRY(36) , ENTRY(37) , ENTRY(38) , ENTRY(39) ,+ ENTRY(40) , ENTRY(41) , ENTRY(42) , ENTRY(43) ,+ ENTRY(44) , ENTRY(45) , ENTRY(46) , ENTRY(47) ,+ ENTRY(48) , ENTRY(49) , ENTRY(50) , ENTRY(51) ,+ ENTRY(52) , ENTRY(53) , ENTRY(54) , ENTRY(55) ,+ ENTRY(56) , ENTRY(57) , ENTRY(58) , ENTRY(59) ,+ ENTRY(60) , ENTRY(61) , ENTRY(62) , ENTRY(63) ,+ ENTRY(64) , ENTRY(65) , ENTRY(66) , ENTRY(67) ,+ ENTRY(68) , ENTRY(69) , ENTRY(70) , ENTRY(71) ,+ ENTRY(72) , ENTRY(73) , ENTRY(74) , ENTRY(75) ,+ ENTRY(76) , ENTRY(77) , ENTRY(78) , ENTRY(79) ,+ ENTRY(80) , ENTRY(81) , ENTRY(82) , ENTRY(83) ,+ ENTRY(84) , ENTRY(85) , ENTRY(86) , ENTRY(87) ,+ ENTRY(88) , ENTRY(89) , ENTRY(90) , ENTRY(91) ,+ ENTRY(92) , ENTRY(93) , ENTRY(94) , ENTRY(95) ,+ ENTRY(96) , ENTRY(97) , ENTRY(98) , ENTRY(99) ,+ ENTRY(100) , ENTRY(101) , ENTRY(102) , ENTRY(103) ,+ ENTRY(104) , ENTRY(105) , ENTRY(106) , ENTRY(107) ,+ ENTRY(108) , ENTRY(109) , ENTRY(110) , ENTRY(111) ,+ ENTRY(112) , ENTRY(113) , ENTRY(114) , ENTRY(115) ,+ ENTRY(116) , ENTRY(117) , ENTRY(118) , ENTRY(119) ,+ ENTRY(120) , ENTRY(121) , ENTRY(122) , ENTRY(123) ,+ ENTRY(124) , ENTRY(125) , ENTRY(126) , ENTRY(127) ,+ ENTRY(128) , ENTRY(129) , ENTRY(130) , ENTRY(131) ,+ ENTRY(132) , ENTRY(133) , ENTRY(134) , ENTRY(135) ,+ ENTRY(136) , ENTRY(137) , ENTRY(138) , ENTRY(139) ,+ ENTRY(140) , ENTRY(141) , ENTRY(142) , ENTRY(143) ,+ ENTRY(144) , ENTRY(145) , ENTRY(146) , ENTRY(147) ,+ ENTRY(148) , ENTRY(149) , ENTRY(150) , ENTRY(151) ,+ ENTRY(152) , ENTRY(153) , ENTRY(154) , ENTRY(155) ,+ ENTRY(156) , ENTRY(157) , ENTRY(158) , ENTRY(159) ,+ ENTRY(160) , ENTRY(161) , ENTRY(162) , ENTRY(163) ,+ ENTRY(164) , ENTRY(165) , ENTRY(166) , ENTRY(167) ,+ ENTRY(168) , ENTRY(169) , ENTRY(170) , ENTRY(171) ,+ ENTRY(172) , ENTRY(173) , ENTRY(174) , ENTRY(175) ,+ ENTRY(176) , ENTRY(177) , ENTRY(178) , ENTRY(179) ,+ ENTRY(180) , ENTRY(181) , ENTRY(182) , ENTRY(183) ,+ ENTRY(184) , ENTRY(185) , ENTRY(186) , ENTRY(187) ,+ ENTRY(188) , ENTRY(189) , ENTRY(190) , ENTRY(191) ,+ ENTRY(192) , ENTRY(193) , ENTRY(194) , ENTRY(195) ,+ ENTRY(196) , ENTRY(197) , ENTRY(198) , ENTRY(199) ,+ ENTRY(200) , ENTRY(201) , ENTRY(202) , ENTRY(203) ,+ ENTRY(204) , ENTRY(205) , ENTRY(206) , ENTRY(207) ,+ ENTRY(208) , ENTRY(209) , ENTRY(210) , ENTRY(211) ,+ ENTRY(212) , ENTRY(213) , ENTRY(214) , ENTRY(215) ,+ ENTRY(216) , ENTRY(217) , ENTRY(218) , ENTRY(219) ,+ ENTRY(220) , ENTRY(221) , ENTRY(222) , ENTRY(223) ,+ ENTRY(224) , ENTRY(225) , ENTRY(226) , ENTRY(227) ,+ ENTRY(228) , ENTRY(229) , ENTRY(230) , ENTRY(231) ,+ ENTRY(232) , ENTRY(233) , ENTRY(234) , ENTRY(235) ,+ ENTRY(236) , ENTRY(237) , ENTRY(238) , ENTRY(239) ,+ ENTRY(240) , ENTRY(241) , ENTRY(242) , ENTRY(243) ,+ ENTRY(244) , ENTRY(245) , ENTRY(246) , ENTRY(247) ,+ ENTRY(248) , ENTRY(249) , ENTRY(250) , ENTRY(251) ,+ ENTRY(252) , ENTRY(253) , ENTRY(254) , ENTRY(255) ,+#undef ENTRY++ /* End of block */+ HUFFDEC_EXCEPTIONAL | HUFFDEC_END_OF_BLOCK,++ /* Lengths */+#define ENTRY(length_base, num_extra_bits) \+ (((u32)(length_base) << 16) | (num_extra_bits))+ ENTRY(3 , 0) , ENTRY(4 , 0) , ENTRY(5 , 0) , ENTRY(6 , 0),+ ENTRY(7 , 0) , ENTRY(8 , 0) , ENTRY(9 , 0) , ENTRY(10 , 0),+ ENTRY(11 , 1) , ENTRY(13 , 1) , ENTRY(15 , 1) , ENTRY(17 , 1),+ ENTRY(19 , 2) , ENTRY(23 , 2) , ENTRY(27 , 2) , ENTRY(31 , 2),+ ENTRY(35 , 3) , ENTRY(43 , 3) , ENTRY(51 , 3) , ENTRY(59 , 3),+ ENTRY(67 , 4) , ENTRY(83 , 4) , ENTRY(99 , 4) , ENTRY(115, 4),+ ENTRY(131, 5) , ENTRY(163, 5) , ENTRY(195, 5) , ENTRY(227, 5),+ ENTRY(258, 0) , ENTRY(258, 0) , ENTRY(258, 0) ,+#undef ENTRY+};++/* Maximum number of bits that can be consumed by decoding a match offset */+#define OFFSET_MAXBITS (DEFLATE_MAX_OFFSET_CODEWORD_LEN + \+ DEFLATE_MAX_EXTRA_OFFSET_BITS)+#define OFFSET_MAXFASTBITS (OFFSET_TABLEBITS /* no subtable needed */ + \+ DEFLATE_MAX_EXTRA_OFFSET_BITS)++/*+ * Here is the format of our offset decode table entries. Bits not explicitly+ * described contain zeroes:+ *+ * Offsets:+ * Bit 31-16: offset base value+ * Bit 15: 0 (!HUFFDEC_EXCEPTIONAL)+ * Bit 14: 0 (!HUFFDEC_SUBTABLE_POINTER)+ * Bit 11-8: remaining codeword length+ * Bit 4-0: remaining codeword length + number of extra bits+ * Subtable pointer:+ * Bit 31-16: index of start of subtable+ * Bit 15: 1 (HUFFDEC_EXCEPTIONAL)+ * Bit 14: 1 (HUFFDEC_SUBTABLE_POINTER)+ * Bit 11-8: number of subtable bits+ * Bit 3-0: number of main table bits+ *+ * These work the same way as the length entries and subtable pointer entries in+ * the litlen decode table; see litlen_decode_results[] above.+ */+static const u32 offset_decode_results[] = {+#define ENTRY(offset_base, num_extra_bits) \+ (((u32)(offset_base) << 16) | (num_extra_bits))+ ENTRY(1 , 0) , ENTRY(2 , 0) , ENTRY(3 , 0) , ENTRY(4 , 0) ,+ ENTRY(5 , 1) , ENTRY(7 , 1) , ENTRY(9 , 2) , ENTRY(13 , 2) ,+ ENTRY(17 , 3) , ENTRY(25 , 3) , ENTRY(33 , 4) , ENTRY(49 , 4) ,+ ENTRY(65 , 5) , ENTRY(97 , 5) , ENTRY(129 , 6) , ENTRY(193 , 6) ,+ ENTRY(257 , 7) , ENTRY(385 , 7) , ENTRY(513 , 8) , ENTRY(769 , 8) ,+ ENTRY(1025 , 9) , ENTRY(1537 , 9) , ENTRY(2049 , 10) , ENTRY(3073 , 10) ,+ ENTRY(4097 , 11) , ENTRY(6145 , 11) , ENTRY(8193 , 12) , ENTRY(12289 , 12) ,+ ENTRY(16385 , 13) , ENTRY(24577 , 13) , ENTRY(24577 , 13) , ENTRY(24577 , 13) ,+#undef ENTRY+};++/*+ * The main DEFLATE decompressor structure. Since libdeflate only supports+ * full-buffer decompression, this structure doesn't store the entire+ * decompression state, most of which is in stack variables. Instead, this+ * struct just contains the decode tables and some temporary arrays used for+ * building them, as these are too large to comfortably allocate on the stack.+ *+ * Storing the decode tables in the decompressor struct also allows the decode+ * tables for the static codes to be reused whenever two static Huffman blocks+ * are decoded without an intervening dynamic block, even across streams.+ */+struct libdeflate_decompressor {++ /*+ * The arrays aren't all needed at the same time. 'precode_lens' and+ * 'precode_decode_table' are unneeded after 'lens' has been filled.+ * Furthermore, 'lens' need not be retained after building the litlen+ * and offset decode tables. In fact, 'lens' can be in union with+ * 'litlen_decode_table' provided that 'offset_decode_table' is separate+ * and is built first.+ */++ union {+ u8 precode_lens[DEFLATE_NUM_PRECODE_SYMS];++ struct {+ u8 lens[DEFLATE_NUM_LITLEN_SYMS ++ DEFLATE_NUM_OFFSET_SYMS ++ DEFLATE_MAX_LENS_OVERRUN];++ u32 precode_decode_table[PRECODE_ENOUGH];+ } l;++ u32 litlen_decode_table[LITLEN_ENOUGH];+ } u;++ u32 offset_decode_table[OFFSET_ENOUGH];++ /* used only during build_decode_table() */+ u16 sorted_syms[DEFLATE_MAX_NUM_SYMS];++ bool static_codes_loaded;+ unsigned litlen_tablebits;++ /* The free() function for this struct, chosen at allocation time */+ free_func_t free_func;+};++/*+ * Build a table for fast decoding of symbols from a Huffman code. As input,+ * this function takes the codeword length of each symbol which may be used in+ * the code. As output, it produces a decode table for the canonical Huffman+ * code described by the codeword lengths. The decode table is built with the+ * assumption that it will be indexed with "bit-reversed" codewords, where the+ * low-order bit is the first bit of the codeword. This format is used for all+ * Huffman codes in DEFLATE.+ *+ * @decode_table+ * The array in which the decode table will be generated. This array must+ * have sufficient length; see the definition of the ENOUGH numbers.+ * @lens+ * An array which provides, for each symbol, the length of the+ * corresponding codeword in bits, or 0 if the symbol is unused. This may+ * alias @decode_table, since nothing is written to @decode_table until all+ * @lens have been consumed. All codeword lengths are assumed to be <=+ * @max_codeword_len but are otherwise considered untrusted. If they do+ * not form a valid Huffman code, then the decode table is not built and+ * %false is returned.+ * @num_syms+ * The number of symbols in the code, including all unused symbols.+ * @decode_results+ * An array which gives the incomplete decode result for each symbol. The+ * needed values in this array will be combined with codeword lengths to+ * make the final decode table entries using make_decode_table_entry().+ * @table_bits+ * The log base-2 of the number of main table entries to use.+ * If @table_bits_ret != NULL, then @table_bits is treated as a maximum+ * value and it will be decreased if a smaller table would be sufficient.+ * @max_codeword_len+ * The maximum allowed codeword length for this Huffman code.+ * Must be <= DEFLATE_MAX_CODEWORD_LEN.+ * @sorted_syms+ * A temporary array of length @num_syms.+ * @table_bits_ret+ * If non-NULL, then the dynamic table_bits is enabled, and the actual+ * table_bits value will be returned here.+ *+ * Returns %true if successful; %false if the codeword lengths do not form a+ * valid Huffman code.+ */+static bool+build_decode_table(u32 decode_table[],+ const u8 lens[],+ const unsigned num_syms,+ const u32 decode_results[],+ unsigned table_bits,+ unsigned max_codeword_len,+ u16 *sorted_syms,+ unsigned *table_bits_ret)+{+ unsigned len_counts[DEFLATE_MAX_CODEWORD_LEN + 1];+ unsigned offsets[DEFLATE_MAX_CODEWORD_LEN + 1];+ unsigned sym; /* current symbol */+ unsigned codeword; /* current codeword, bit-reversed */+ unsigned len; /* current codeword length in bits */+ unsigned count; /* num codewords remaining with this length */+ u32 codespace_used; /* codespace used out of '2^max_codeword_len' */+ unsigned cur_table_end; /* end index of current table */+ unsigned subtable_prefix; /* codeword prefix of current subtable */+ unsigned subtable_start; /* start index of current subtable */+ unsigned subtable_bits; /* log2 of current subtable length */++ /* Count how many codewords have each length, including 0. */+ for (len = 0; len <= max_codeword_len; len++)+ len_counts[len] = 0;+ for (sym = 0; sym < num_syms; sym++)+ len_counts[lens[sym]]++;++ /*+ * Determine the actual maximum codeword length that was used, and+ * decrease table_bits to it if allowed.+ */+ while (max_codeword_len > 1 && len_counts[max_codeword_len] == 0)+ max_codeword_len--;+ if (table_bits_ret != NULL) {+ table_bits = MIN(table_bits, max_codeword_len);+ *table_bits_ret = table_bits;+ }++ /*+ * Sort the symbols primarily by increasing codeword length and+ * secondarily by increasing symbol value; or equivalently by their+ * codewords in lexicographic order, since a canonical code is assumed.+ *+ * For efficiency, also compute 'codespace_used' in the same pass over+ * 'len_counts[]' used to build 'offsets[]' for sorting.+ */++ /* Ensure that 'codespace_used' cannot overflow. */+ STATIC_ASSERT(sizeof(codespace_used) == 4);+ STATIC_ASSERT(UINT32_MAX / (1U << (DEFLATE_MAX_CODEWORD_LEN - 1)) >=+ DEFLATE_MAX_NUM_SYMS);++ offsets[0] = 0;+ offsets[1] = len_counts[0];+ codespace_used = 0;+ for (len = 1; len < max_codeword_len; len++) {+ offsets[len + 1] = offsets[len] + len_counts[len];+ codespace_used = (codespace_used << 1) + len_counts[len];+ }+ codespace_used = (codespace_used << 1) + len_counts[len];++ for (sym = 0; sym < num_syms; sym++)+ sorted_syms[offsets[lens[sym]]++] = sym;++ sorted_syms += offsets[0]; /* Skip unused symbols */++ /* lens[] is done being used, so we can write to decode_table[] now. */++ /*+ * Check whether the lengths form a complete code (exactly fills the+ * codespace), an incomplete code (doesn't fill the codespace), or an+ * overfull code (overflows the codespace). A codeword of length 'n'+ * uses proportion '1/(2^n)' of the codespace. An overfull code is+ * nonsensical, so is considered invalid. An incomplete code is+ * considered valid only in two specific cases; see below.+ */++ /* overfull code? */+ if (unlikely(codespace_used > (1U << max_codeword_len)))+ return false;++ /* incomplete code? */+ if (unlikely(codespace_used < (1U << max_codeword_len))) {+ u32 entry;+ unsigned i;++ /*+ * The DEFLATE RFC explicitly allows the offset code to be+ * incomplete in two cases: a code containing just 1 codeword,+ * if that codeword has length 1; and a code containing no+ * codewords. Note: the list of offset codeword lengths is+ * always nonempty, but lengths of 0 don't count as codewords.+ *+ * The RFC doesn't say whether the same cases are allowed for+ * the litlen and pre codes. It's actually impossible for no+ * symbols to be used from these codes; however, it's+ * technically possible for only one symbol to be used. zlib+ * allows 1 codeword for the litlen code, but not the precode.+ * The RFC also doesn't say whether, when there is 1 codeword,+ * that codeword is '0' or '1'. zlib uses '0'.+ *+ * We accept what zlib accepts, plus a bit more. First, we+ * don't treat the precode more strictly than the litlen and+ * offset codes. There's no convincing reason to add a special+ * case for the precode here.+ *+ * Second, we just map each allowed incompete code to a complete+ * code with only real symbols. To do this, we choose a symbol,+ * either the used symbol (for codes with 1 codeword) or an+ * arbitrary symbol (for empty codes), and give it both+ * codewords '0' and '1'. zlib instead uses a special ERROR+ * symbol in the part of the codespace the code doesn't use.+ * However, having an ERROR symbol reduces the performance of+ * the Huffman decoder, for no real benefit. Our approach also+ * avoids having to decide whether '0' or '1' is correct.+ *+ * Like zlib, we still reject all incomplete codes that contain+ * more than 1 codeword or a codeword length greater than 1.+ */+ if (codespace_used == 0) {+ sym = 0; /* arbitrary */+ } else {+ if (codespace_used != (1U << (max_codeword_len - 1)) ||+ len_counts[1] != 1)+ return false;+ sym = sorted_syms[0];+ }+ entry = make_decode_table_entry(decode_results, sym, 1);+ for (i = 0; i < (1U << table_bits); i++)+ decode_table[i] = entry;+ return true;+ }++ /*+ * The lengths form a complete code. Now, enumerate the codewords in+ * lexicographic order and fill the decode table entries for each one.+ *+ * First, process all codewords with len <= table_bits. Each one gets+ * '2^(table_bits-len)' direct entries in the table.+ *+ * Since DEFLATE uses bit-reversed codewords, these entries aren't+ * consecutive but rather are spaced '2^len' entries apart. This makes+ * filling them naively somewhat awkward and inefficient, since strided+ * stores are less cache-friendly and preclude the use of word or+ * vector-at-a-time stores to fill multiple entries per instruction.+ *+ * To optimize this, we incrementally double the table size. When+ * processing codewords with length 'len', the table is treated as+ * having only '2^len' entries, so each codeword uses just one entry.+ * Then, each time 'len' is incremented, the table size is doubled and+ * the first half is copied to the second half. This significantly+ * improves performance over naively doing strided stores.+ *+ * Note that some entries copied for each table doubling may not have+ * been initialized yet, but it doesn't matter since they're guaranteed+ * to be initialized later (because the Huffman code is complete).+ */+ codeword = 0;+ len = 1;+ while ((count = len_counts[len]) == 0)+ len++;+ cur_table_end = 1U << len;+ while (len <= table_bits) {+ /* Process all 'count' codewords with length 'len' bits. */+ do {+ unsigned bit;++ /* Fill the first entry for the current codeword. */+ decode_table[codeword] =+ make_decode_table_entry(decode_results,+ *sorted_syms++, len);++ if (codeword == cur_table_end - 1) {+ /* Last codeword (all 1's) */+ for (; len < table_bits; len++) {+ memcpy(&decode_table[cur_table_end],+ decode_table,+ cur_table_end *+ sizeof(decode_table[0]));+ cur_table_end <<= 1;+ }+ return true;+ }+ /*+ * To advance to the lexicographically next codeword in+ * the canonical code, the codeword must be incremented,+ * then 0's must be appended to the codeword as needed+ * to match the next codeword's length.+ *+ * Since the codeword is bit-reversed, appending 0's is+ * a no-op. However, incrementing it is nontrivial. To+ * do so efficiently, use the 'bsr' instruction to find+ * the last (highest order) 0 bit in the codeword, set+ * it, and clear any later (higher order) 1 bits. But+ * 'bsr' actually finds the highest order 1 bit, so to+ * use it first flip all bits in the codeword by XOR'ing+ * it with (1U << len) - 1 == cur_table_end - 1.+ */+ bit = 1U << bsr32(codeword ^ (cur_table_end - 1));+ codeword &= bit - 1;+ codeword |= bit;+ } while (--count);++ /* Advance to the next codeword length. */+ do {+ if (++len <= table_bits) {+ memcpy(&decode_table[cur_table_end],+ decode_table,+ cur_table_end * sizeof(decode_table[0]));+ cur_table_end <<= 1;+ }+ } while ((count = len_counts[len]) == 0);+ }++ /* Process codewords with len > table_bits. These require subtables. */+ cur_table_end = 1U << table_bits;+ subtable_prefix = -1;+ subtable_start = 0;+ for (;;) {+ u32 entry;+ unsigned i;+ unsigned stride;+ unsigned bit;++ /*+ * Start a new subtable if the first 'table_bits' bits of the+ * codeword don't match the prefix of the current subtable.+ */+ if ((codeword & ((1U << table_bits) - 1)) != subtable_prefix) {+ subtable_prefix = (codeword & ((1U << table_bits) - 1));+ subtable_start = cur_table_end;+ /*+ * Calculate the subtable length. If the codeword has+ * length 'table_bits + n', then the subtable needs+ * '2^n' entries. But it may need more; if fewer than+ * '2^n' codewords of length 'table_bits + n' remain,+ * then the length will need to be incremented to bring+ * in longer codewords until the subtable can be+ * completely filled. Note that because the Huffman+ * code is complete, it will always be possible to fill+ * the subtable eventually.+ */+ subtable_bits = len - table_bits;+ codespace_used = count;+ while (codespace_used < (1U << subtable_bits)) {+ subtable_bits++;+ codespace_used = (codespace_used << 1) ++ len_counts[table_bits + subtable_bits];+ }+ cur_table_end = subtable_start + (1U << subtable_bits);++ /*+ * Create the entry that points from the main table to+ * the subtable.+ */+ decode_table[subtable_prefix] =+ ((u32)subtable_start << 16) |+ HUFFDEC_EXCEPTIONAL |+ HUFFDEC_SUBTABLE_POINTER |+ (subtable_bits << 8) | table_bits;+ }++ /* Fill the subtable entries for the current codeword. */+ entry = make_decode_table_entry(decode_results, *sorted_syms++,+ len - table_bits);+ i = subtable_start + (codeword >> table_bits);+ stride = 1U << (len - table_bits);+ do {+ decode_table[i] = entry;+ i += stride;+ } while (i < cur_table_end);++ /* Advance to the next codeword. */+ if (codeword == (1U << len) - 1) /* last codeword (all 1's)? */+ return true;+ bit = 1U << bsr32(codeword ^ ((1U << len) - 1));+ codeword &= bit - 1;+ codeword |= bit;+ count--;+ while (count == 0)+ count = len_counts[++len];+ }+}++/* Build the decode table for the precode. */+static bool+build_precode_decode_table(struct libdeflate_decompressor *d)+{+ /* When you change TABLEBITS, you must change ENOUGH, and vice versa! */+ STATIC_ASSERT(PRECODE_TABLEBITS == 7 && PRECODE_ENOUGH == 128);++ STATIC_ASSERT(ARRAY_LEN(precode_decode_results) ==+ DEFLATE_NUM_PRECODE_SYMS);++ return build_decode_table(d->u.l.precode_decode_table,+ d->u.precode_lens,+ DEFLATE_NUM_PRECODE_SYMS,+ precode_decode_results,+ PRECODE_TABLEBITS,+ DEFLATE_MAX_PRE_CODEWORD_LEN,+ d->sorted_syms,+ NULL);+}++/* Build the decode table for the literal/length code. */+static bool+build_litlen_decode_table(struct libdeflate_decompressor *d,+ unsigned num_litlen_syms, unsigned num_offset_syms)+{+ /* When you change TABLEBITS, you must change ENOUGH, and vice versa! */+ STATIC_ASSERT(LITLEN_TABLEBITS == 11 && LITLEN_ENOUGH == 2342);++ STATIC_ASSERT(ARRAY_LEN(litlen_decode_results) ==+ DEFLATE_NUM_LITLEN_SYMS);++ return build_decode_table(d->u.litlen_decode_table,+ d->u.l.lens,+ num_litlen_syms,+ litlen_decode_results,+ LITLEN_TABLEBITS,+ DEFLATE_MAX_LITLEN_CODEWORD_LEN,+ d->sorted_syms,+ &d->litlen_tablebits);+}++/* Build the decode table for the offset code. */+static bool+build_offset_decode_table(struct libdeflate_decompressor *d,+ unsigned num_litlen_syms, unsigned num_offset_syms)+{+ /* When you change TABLEBITS, you must change ENOUGH, and vice versa! */+ STATIC_ASSERT(OFFSET_TABLEBITS == 8 && OFFSET_ENOUGH == 402);++ STATIC_ASSERT(ARRAY_LEN(offset_decode_results) ==+ DEFLATE_NUM_OFFSET_SYMS);++ return build_decode_table(d->offset_decode_table,+ d->u.l.lens + num_litlen_syms,+ num_offset_syms,+ offset_decode_results,+ OFFSET_TABLEBITS,+ DEFLATE_MAX_OFFSET_CODEWORD_LEN,+ d->sorted_syms,+ NULL);+}++/*****************************************************************************+ * Main decompression routine+ *****************************************************************************/++typedef enum libdeflate_result (*decompress_func_t)+ (struct libdeflate_decompressor * restrict d,+ const void * restrict in, size_t in_nbytes,+ void * restrict out, size_t out_nbytes_avail,+ size_t *actual_in_nbytes_ret, size_t *actual_out_nbytes_ret);++#define FUNCNAME deflate_decompress_default+#undef ATTRIBUTES+#undef EXTRACT_VARBITS+#undef EXTRACT_VARBITS8+#include "decompress_template.h"++/* Include architecture-specific implementation(s) if available. */+#undef DEFAULT_IMPL+#undef arch_select_decompress_func+#if defined(ARCH_X86_32) || defined(ARCH_X86_64)+# include "x86/decompress_impl.h"+#endif++#ifndef DEFAULT_IMPL+# define DEFAULT_IMPL deflate_decompress_default+#endif++#ifdef arch_select_decompress_func+static enum libdeflate_result+dispatch_decomp(struct libdeflate_decompressor *d,+ const void *in, size_t in_nbytes,+ void *out, size_t out_nbytes_avail,+ size_t *actual_in_nbytes_ret, size_t *actual_out_nbytes_ret);++static volatile decompress_func_t decompress_impl = dispatch_decomp;++/* Choose the best implementation at runtime. */+static enum libdeflate_result+dispatch_decomp(struct libdeflate_decompressor *d,+ const void *in, size_t in_nbytes,+ void *out, size_t out_nbytes_avail,+ size_t *actual_in_nbytes_ret, size_t *actual_out_nbytes_ret)+{+ decompress_func_t f = arch_select_decompress_func();++ if (f == NULL)+ f = DEFAULT_IMPL;++ decompress_impl = f;+ return f(d, in, in_nbytes, out, out_nbytes_avail,+ actual_in_nbytes_ret, actual_out_nbytes_ret);+}+#else+/* The best implementation is statically known, so call it directly. */+# define decompress_impl DEFAULT_IMPL+#endif++/*+ * This is the main DEFLATE decompression routine. See libdeflate.h for the+ * documentation.+ *+ * Note that the real code is in decompress_template.h. The part here just+ * handles calling the appropriate implementation depending on the CPU features+ * at runtime.+ */+LIBDEFLATEAPI enum libdeflate_result+libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *d,+ const void *in, size_t in_nbytes,+ void *out, size_t out_nbytes_avail,+ size_t *actual_in_nbytes_ret,+ size_t *actual_out_nbytes_ret)+{+ return decompress_impl(d, in, in_nbytes, out, out_nbytes_avail,+ actual_in_nbytes_ret, actual_out_nbytes_ret);+}++LIBDEFLATEAPI enum libdeflate_result+libdeflate_deflate_decompress(struct libdeflate_decompressor *d,+ const void *in, size_t in_nbytes,+ void *out, size_t out_nbytes_avail,+ size_t *actual_out_nbytes_ret)+{+ return libdeflate_deflate_decompress_ex(d, in, in_nbytes,+ out, out_nbytes_avail,+ NULL, actual_out_nbytes_ret);+}++LIBDEFLATEAPI struct libdeflate_decompressor *+libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options)+{+ struct libdeflate_decompressor *d;++ /*+ * Note: if more fields are added to libdeflate_options, this code will+ * need to be updated to support both the old and new structs.+ */+ if (options->sizeof_options != sizeof(*options))+ return NULL;++ d = (options->malloc_func ? options->malloc_func :+ libdeflate_default_malloc_func)(sizeof(*d));+ if (d == NULL)+ return NULL;+ /*+ * Note that only certain parts of the decompressor actually must be+ * initialized here:+ *+ * - 'static_codes_loaded' must be initialized to false.+ *+ * - The first half of the main portion of each decode table must be+ * initialized to any value, to avoid reading from uninitialized+ * memory during table expansion in build_decode_table(). (Although,+ * this is really just to avoid warnings with dynamic tools like+ * valgrind, since build_decode_table() is guaranteed to initialize+ * all entries eventually anyway.)+ *+ * - 'free_func' must be set.+ *+ * But for simplicity, we currently just zero the whole decompressor.+ */+ memset(d, 0, sizeof(*d));+ d->free_func = options->free_func ?+ options->free_func : libdeflate_default_free_func;+ return d;+}++LIBDEFLATEAPI struct libdeflate_decompressor *+libdeflate_alloc_decompressor(void)+{+ static const struct libdeflate_options defaults = {+ .sizeof_options = sizeof(defaults),+ };+ return libdeflate_alloc_decompressor_ex(&defaults);+}++LIBDEFLATEAPI void+libdeflate_free_decompressor(struct libdeflate_decompressor *d)+{+ if (d)+ d->free_func(d);+}diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/lib_common.h b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/lib_common.hnew file mode 100644index 0000000..faedfcb--- /dev/null+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/lib_common.h@@ -0,0 +1,106 @@+/*+ * lib_common.h - internal header included by all library code+ */++#ifndef LIB_LIB_COMMON_H+#define LIB_LIB_COMMON_H++#ifdef LIBDEFLATE_H+ /*+ * When building the library, LIBDEFLATEAPI needs to be defined properly before+ * including libdeflate.h.+ */+# error "lib_common.h must always be included before libdeflate.h"+#endif++#if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))+# define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)+#elif defined(__GNUC__)+# define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))+#else+# define LIBDEFLATE_EXPORT_SYM+#endif++/*+ * On i386, gcc assumes that the stack is 16-byte aligned at function entry.+ * However, some compilers (e.g. MSVC) and programming languages (e.g. Delphi)+ * only guarantee 4-byte alignment when calling functions. This is mainly an+ * issue on Windows, but it has been seen on Linux too. Work around this ABI+ * incompatibility by realigning the stack pointer when entering libdeflate.+ * This prevents crashes in SSE/AVX code.+ */+#if defined(__GNUC__) && defined(__i386__)+# define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))+#else+# define LIBDEFLATE_ALIGN_STACK+#endif++#define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK++#include "../common_defs.h"++typedef void *(*malloc_func_t)(size_t);+typedef void (*free_func_t)(void *);++extern malloc_func_t libdeflate_default_malloc_func;+extern free_func_t libdeflate_default_free_func;++void *libdeflate_aligned_malloc(malloc_func_t malloc_func,+ size_t alignment, size_t size);+void libdeflate_aligned_free(free_func_t free_func, void *ptr);++#ifdef FREESTANDING+/*+ * With -ffreestanding, <string.h> may be missing, and we must provide+ * implementations of memset(), memcpy(), memmove(), and memcmp().+ * See https://gcc.gnu.org/onlinedocs/gcc/Standards.html+ *+ * Also, -ffreestanding disables interpreting calls to these functions as+ * built-ins. E.g., calling memcpy(&v, p, WORDBYTES) will make a function call,+ * not be optimized to a single load instruction. For performance reasons we+ * don't want that. So, declare these functions as macros that expand to the+ * corresponding built-ins. This approach is recommended in the gcc man page.+ * We still need the actual function definitions in case gcc calls them.+ */+void *memset(void *s, int c, size_t n);+#define memset(s, c, n) __builtin_memset((s), (c), (n))++void *memcpy(void *dest, const void *src, size_t n);+#define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))++void *memmove(void *dest, const void *src, size_t n);+#define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))++int memcmp(const void *s1, const void *s2, size_t n);+#define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))++#undef LIBDEFLATE_ENABLE_ASSERTIONS+#else+# include <string.h>+ /*+ * To prevent false positive static analyzer warnings, ensure that assertions+ * are visible to the static analyzer.+ */+# ifdef __clang_analyzer__+# define LIBDEFLATE_ENABLE_ASSERTIONS+# endif+#endif++/*+ * Runtime assertion support. Don't enable this in production builds; it may+ * hurt performance significantly.+ */+#ifdef LIBDEFLATE_ENABLE_ASSERTIONS+NORETURN void+libdeflate_assertion_failed(const char *expr, const char *file, int line);+#define ASSERT(expr) { if (unlikely(!(expr))) \+ libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }+#else+#define ASSERT(expr) (void)(expr)+#endif++#define CONCAT_IMPL(a, b) a##b+#define CONCAT(a, b) CONCAT_IMPL(a, b)+#define ADD_SUFFIX(name) CONCAT(name, SUFFIX)++#endif /* LIB_LIB_COMMON_H */diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/zlib_constants.h b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/zlib_constants.hnew file mode 100644index 0000000..f304310--- /dev/null+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/zlib_constants.h@@ -0,0 +1,21 @@+/*+ * zlib_constants.h - constants for the zlib wrapper format+ */++#ifndef LIB_ZLIB_CONSTANTS_H+#define LIB_ZLIB_CONSTANTS_H++#define ZLIB_MIN_HEADER_SIZE 2+#define ZLIB_FOOTER_SIZE 4+#define ZLIB_MIN_OVERHEAD (ZLIB_MIN_HEADER_SIZE + ZLIB_FOOTER_SIZE)++#define ZLIB_CM_DEFLATE 8++#define ZLIB_CINFO_32K_WINDOW 7++#define ZLIB_FASTEST_COMPRESSION 0+#define ZLIB_FAST_COMPRESSION 1+#define ZLIB_DEFAULT_COMPRESSION 2+#define ZLIB_SLOWEST_COMPRESSION 3++#endif /* LIB_ZLIB_CONSTANTS_H */diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/zlib_decompress.c b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/zlib_decompress.cnew file mode 100644index 0000000..f5e43ea--- /dev/null+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/zlib_decompress.c@@ -0,0 +1,104 @@+/*+ * zlib_decompress.c - decompress with a zlib wrapper+ *+ * Copyright 2016 Eric Biggers+ *+ * Permission is hereby granted, free of charge, to any person+ * obtaining a copy of this software and associated documentation+ * files (the "Software"), to deal in the Software without+ * restriction, including without limitation the rights to use,+ * copy, modify, merge, publish, distribute, sublicense, and/or sell+ * copies of the Software, and to permit persons to whom the+ * Software is furnished to do so, subject to the following+ * conditions:+ *+ * The above copyright notice and this permission notice shall be+ * included in all copies or substantial portions of the Software.+ *+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR+ * OTHER DEALINGS IN THE SOFTWARE.+ */++#include "lib_common.h"+#include "zlib_constants.h"++LIBDEFLATEAPI enum libdeflate_result+libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *d,+ const void *in, size_t in_nbytes,+ void *out, size_t out_nbytes_avail,+ size_t *actual_in_nbytes_ret,+ size_t *actual_out_nbytes_ret)+{+ const u8 *in_next = in;+ const u8 * const in_end = in_next + in_nbytes;+ u16 hdr;+ size_t actual_in_nbytes;+ size_t actual_out_nbytes;+ enum libdeflate_result result;++ if (in_nbytes < ZLIB_MIN_OVERHEAD)+ return LIBDEFLATE_BAD_DATA;++ /* 2 byte header: CMF and FLG */+ hdr = get_unaligned_be16(in_next);+ in_next += 2;++ /* FCHECK */+ if ((hdr % 31) != 0)+ return LIBDEFLATE_BAD_DATA;++ /* CM */+ if (((hdr >> 8) & 0xF) != ZLIB_CM_DEFLATE)+ return LIBDEFLATE_BAD_DATA;++ /* CINFO */+ if ((hdr >> 12) > ZLIB_CINFO_32K_WINDOW)+ return LIBDEFLATE_BAD_DATA;++ /* FDICT */+ if ((hdr >> 5) & 1)+ return LIBDEFLATE_BAD_DATA;++ /* Compressed data */+ result = libdeflate_deflate_decompress_ex(d, in_next,+ in_end - ZLIB_FOOTER_SIZE - in_next,+ out, out_nbytes_avail,+ &actual_in_nbytes, actual_out_nbytes_ret);+ if (result != LIBDEFLATE_SUCCESS)+ return result;++ if (actual_out_nbytes_ret)+ actual_out_nbytes = *actual_out_nbytes_ret;+ else+ actual_out_nbytes = out_nbytes_avail;++ in_next += actual_in_nbytes;++ /* ADLER32 */+ if (libdeflate_adler32(1, out, actual_out_nbytes) !=+ get_unaligned_be32(in_next))+ return LIBDEFLATE_BAD_DATA;+ in_next += 4;++ if (actual_in_nbytes_ret)+ *actual_in_nbytes_ret = in_next - (u8 *)in;++ return LIBDEFLATE_SUCCESS;+}++LIBDEFLATEAPI enum libdeflate_result+libdeflate_zlib_decompress(struct libdeflate_decompressor *d,+ const void *in, size_t in_nbytes,+ void *out, size_t out_nbytes_avail,+ size_t *actual_out_nbytes_ret)+{+ return libdeflate_zlib_decompress_ex(d, in, in_nbytes,+ out, out_nbytes_avail,+ NULL, actual_out_nbytes_ret);+}diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/libdeflate.h b/xnu/files/bsd/miscfs/apfs/libdeflate/libdeflate.hnew file mode 100644index 0000000..52236da--- /dev/null+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/libdeflate.h@@ -0,0 +1,411 @@+/*+ * libdeflate.h - public header for libdeflate+ */++#ifndef LIBDEFLATE_H+#define LIBDEFLATE_H++#include <stddef.h>+#include <stdint.h>++#ifdef __cplusplus+extern "C" {+#endif++#define LIBDEFLATE_VERSION_MAJOR 1+#define LIBDEFLATE_VERSION_MINOR 24+#define LIBDEFLATE_VERSION_STRING "1.24"++/*+ * Users of libdeflate.dll on Windows can define LIBDEFLATE_DLL to cause+ * __declspec(dllimport) to be used. This should be done when it's easy to do.+ * Otherwise it's fine to skip it, since it is a very minor performance+ * optimization that is irrelevant for most use cases of libdeflate.+ */+#ifndef LIBDEFLATEAPI+# if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))+# define LIBDEFLATEAPI __declspec(dllimport)+# else+# define LIBDEFLATEAPI+# endif+#endif++/* ========================================================================== */+/* Compression */+/* ========================================================================== */++struct libdeflate_compressor;+struct libdeflate_options;++/*+ * libdeflate_alloc_compressor() allocates a new compressor that supports+ * DEFLATE, zlib, and gzip compression. 'compression_level' is the compression+ * level on a zlib-like scale but with a higher maximum value (1 = fastest, 6 =+ * medium/default, 9 = slow, 12 = slowest). Level 0 is also supported and means+ * "no compression", specifically "create a valid stream, but only emit+ * uncompressed blocks" (this will expand the data slightly).+ *+ * The return value is a pointer to the new compressor, or NULL if out of memory+ * or if the compression level is invalid (i.e. outside the range [0, 12]).+ *+ * Note: for compression, the sliding window size is defined at compilation time+ * to 32768, the largest size permissible in the DEFLATE format. It cannot be+ * changed at runtime.+ *+ * A single compressor is not safe to use by multiple threads concurrently.+ * However, different threads may use different compressors concurrently.+ */+LIBDEFLATEAPI struct libdeflate_compressor *+libdeflate_alloc_compressor(int compression_level);++/*+ * Like libdeflate_alloc_compressor(), but adds the 'options' argument.+ */+LIBDEFLATEAPI struct libdeflate_compressor *+libdeflate_alloc_compressor_ex(int compression_level,+ const struct libdeflate_options *options);++/*+ * libdeflate_deflate_compress() performs raw DEFLATE compression on a buffer of+ * data. It attempts to compress 'in_nbytes' bytes of data located at 'in' and+ * write the result to 'out', which has space for 'out_nbytes_avail' bytes. The+ * return value is the compressed size in bytes, or 0 if the data could not be+ * compressed to 'out_nbytes_avail' bytes or fewer.+ *+ * If compression is successful, then the output data is guaranteed to be a+ * valid DEFLATE stream that decompresses to the input data. No other+ * guarantees are made about the output data. Notably, different versions of+ * libdeflate can produce different compressed data for the same uncompressed+ * data, even at the same compression level. Do ***NOT*** do things like+ * writing tests that compare compressed data to a golden output, as this can+ * break when libdeflate is updated. (This property isn't specific to+ * libdeflate; the same is true for zlib and other compression libraries too.)+ */+LIBDEFLATEAPI size_t+libdeflate_deflate_compress(struct libdeflate_compressor *compressor,+ const void *in, size_t in_nbytes,+ void *out, size_t out_nbytes_avail);++/*+ * libdeflate_deflate_compress_bound() returns a worst-case upper bound on the+ * number of bytes of compressed data that may be produced by compressing any+ * buffer of length less than or equal to 'in_nbytes' using+ * libdeflate_deflate_compress() with the specified compressor. This bound will+ * necessarily be a number greater than or equal to 'in_nbytes'. It may be an+ * overestimate of the true upper bound. The return value is guaranteed to be+ * the same for all invocations with the same compressor and same 'in_nbytes'.+ *+ * As a special case, 'compressor' may be NULL. This causes the bound to be+ * taken across *any* libdeflate_compressor that could ever be allocated with+ * this build of the library, with any options.+ *+ * Note that this function is not necessary in many applications. With+ * block-based compression, it is usually preferable to separately store the+ * uncompressed size of each block and to store any blocks that did not compress+ * to less than their original size uncompressed. In that scenario, there is no+ * need to know the worst-case compressed size, since the maximum number of+ * bytes of compressed data that may be used would always be one less than the+ * input length. You can just pass a buffer of that size to+ * libdeflate_deflate_compress() and store the data uncompressed if+ * libdeflate_deflate_compress() returns 0, indicating that the compressed data+ * did not fit into the provided output buffer.+ */+LIBDEFLATEAPI size_t+libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,+ size_t in_nbytes);++/*+ * Like libdeflate_deflate_compress(), but uses the zlib wrapper format instead+ * of raw DEFLATE.+ */+LIBDEFLATEAPI size_t+libdeflate_zlib_compress(struct libdeflate_compressor *compressor,+ const void *in, size_t in_nbytes,+ void *out, size_t out_nbytes_avail);++/*+ * Like libdeflate_deflate_compress_bound(), but assumes the data will be+ * compressed with libdeflate_zlib_compress() rather than with+ * libdeflate_deflate_compress().+ */+LIBDEFLATEAPI size_t+libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,+ size_t in_nbytes);++/*+ * Like libdeflate_deflate_compress(), but uses the gzip wrapper format instead+ * of raw DEFLATE.+ */+LIBDEFLATEAPI size_t+libdeflate_gzip_compress(struct libdeflate_compressor *compressor,+ const void *in, size_t in_nbytes,+ void *out, size_t out_nbytes_avail);++/*+ * Like libdeflate_deflate_compress_bound(), but assumes the data will be+ * compressed with libdeflate_gzip_compress() rather than with+ * libdeflate_deflate_compress().+ */+LIBDEFLATEAPI size_t+libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,+ size_t in_nbytes);++/*+ * libdeflate_free_compressor() frees a compressor that was allocated with+ * libdeflate_alloc_compressor(). If a NULL pointer is passed in, no action is+ * taken.+ */+LIBDEFLATEAPI void+libdeflate_free_compressor(struct libdeflate_compressor *compressor);++/* ========================================================================== */+/* Decompression */+/* ========================================================================== */++struct libdeflate_decompressor;+struct libdeflate_options;++/*+ * libdeflate_alloc_decompressor() allocates a new decompressor that can be used+ * for DEFLATE, zlib, and gzip decompression. The return value is a pointer to+ * the new decompressor, or NULL if out of memory.+ *+ * This function takes no parameters, and the returned decompressor is valid for+ * decompressing data that was compressed at any compression level and with any+ * sliding window size.+ *+ * A single decompressor is not safe to use by multiple threads concurrently.+ * However, different threads may use different decompressors concurrently.+ */+LIBDEFLATEAPI struct libdeflate_decompressor *+libdeflate_alloc_decompressor(void);++/*+ * Like libdeflate_alloc_decompressor(), but adds the 'options' argument.+ */+LIBDEFLATEAPI struct libdeflate_decompressor *+libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);++/*+ * Result of a call to libdeflate_deflate_decompress(),+ * libdeflate_zlib_decompress(), or libdeflate_gzip_decompress().+ */+enum libdeflate_result {+ /* Decompression was successful. */+ LIBDEFLATE_SUCCESS = 0,++ /* Decompression failed because the compressed data was invalid,+ * corrupt, or otherwise unsupported. */+ LIBDEFLATE_BAD_DATA = 1,++ /* A NULL 'actual_out_nbytes_ret' was provided, but the data would have+ * decompressed to fewer than 'out_nbytes_avail' bytes. */+ LIBDEFLATE_SHORT_OUTPUT = 2,++ /* The data would have decompressed to more than 'out_nbytes_avail'+ * bytes. */+ LIBDEFLATE_INSUFFICIENT_SPACE = 3,+};++/*+ * libdeflate_deflate_decompress() decompresses a DEFLATE stream from the buffer+ * 'in' with compressed size up to 'in_nbytes' bytes. The uncompressed data is+ * written to 'out', a buffer with size 'out_nbytes_avail' bytes. If+ * decompression succeeds, then 0 (LIBDEFLATE_SUCCESS) is returned. Otherwise,+ * a nonzero result code such as LIBDEFLATE_BAD_DATA is returned, and the+ * contents of the output buffer are undefined.+ *+ * Decompression stops at the end of the DEFLATE stream (as indicated by the+ * BFINAL flag), even if it is actually shorter than 'in_nbytes' bytes.+ *+ * libdeflate_deflate_decompress() can be used in cases where the actual+ * uncompressed size is known (recommended) or unknown (not recommended):+ *+ * - If the actual uncompressed size is known, then pass the actual+ * uncompressed size as 'out_nbytes_avail' and pass NULL for+ * 'actual_out_nbytes_ret'. This makes libdeflate_deflate_decompress() fail+ * with LIBDEFLATE_SHORT_OUTPUT if the data decompressed to fewer than the+ * specified number of bytes.+ *+ * - If the actual uncompressed size is unknown, then provide a non-NULL+ * 'actual_out_nbytes_ret' and provide a buffer with some size+ * 'out_nbytes_avail' that you think is large enough to hold all the+ * uncompressed data. In this case, if the data decompresses to less than+ * or equal to 'out_nbytes_avail' bytes, then+ * libdeflate_deflate_decompress() will write the actual uncompressed size+ * to *actual_out_nbytes_ret and return 0 (LIBDEFLATE_SUCCESS). Otherwise,+ * it will return LIBDEFLATE_INSUFFICIENT_SPACE if the provided buffer was+ * not large enough but no other problems were encountered, or another+ * nonzero result code if decompression failed for another reason.+ */+LIBDEFLATEAPI enum libdeflate_result+libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,+ const void *in, size_t in_nbytes,+ void *out, size_t out_nbytes_avail,+ size_t *actual_out_nbytes_ret);++/*+ * Like libdeflate_deflate_decompress(), but adds the 'actual_in_nbytes_ret'+ * argument. If decompression succeeds and 'actual_in_nbytes_ret' is not NULL,+ * then the actual compressed size of the DEFLATE stream (aligned to the next+ * byte boundary) is written to *actual_in_nbytes_ret.+ */+LIBDEFLATEAPI enum libdeflate_result+libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,+ const void *in, size_t in_nbytes,+ void *out, size_t out_nbytes_avail,+ size_t *actual_in_nbytes_ret,+ size_t *actual_out_nbytes_ret);++/*+ * Like libdeflate_deflate_decompress(), but assumes the zlib wrapper format+ * instead of raw DEFLATE.+ *+ * Decompression will stop at the end of the zlib stream, even if it is shorter+ * than 'in_nbytes'. If you need to know exactly where the zlib stream ended,+ * use libdeflate_zlib_decompress_ex().+ */+LIBDEFLATEAPI enum libdeflate_result+libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,+ const void *in, size_t in_nbytes,+ void *out, size_t out_nbytes_avail,+ size_t *actual_out_nbytes_ret);++/*+ * Like libdeflate_zlib_decompress(), but adds the 'actual_in_nbytes_ret'+ * argument. If 'actual_in_nbytes_ret' is not NULL and the decompression+ * succeeds (indicating that the first zlib-compressed stream in the input+ * buffer was decompressed), then the actual number of input bytes consumed is+ * written to *actual_in_nbytes_ret.+ */+LIBDEFLATEAPI enum libdeflate_result+libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,+ const void *in, size_t in_nbytes,+ void *out, size_t out_nbytes_avail,+ size_t *actual_in_nbytes_ret,+ size_t *actual_out_nbytes_ret);++/*+ * Like libdeflate_deflate_decompress(), but assumes the gzip wrapper format+ * instead of raw DEFLATE.+ *+ * If multiple gzip-compressed members are concatenated, then only the first+ * will be decompressed. Use libdeflate_gzip_decompress_ex() if you need+ * multi-member support.+ */+LIBDEFLATEAPI enum libdeflate_result+libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,+ const void *in, size_t in_nbytes,+ void *out, size_t out_nbytes_avail,+ size_t *actual_out_nbytes_ret);++/*+ * Like libdeflate_gzip_decompress(), but adds the 'actual_in_nbytes_ret'+ * argument. If 'actual_in_nbytes_ret' is not NULL and the decompression+ * succeeds (indicating that the first gzip-compressed member in the input+ * buffer was decompressed), then the actual number of input bytes consumed is+ * written to *actual_in_nbytes_ret.+ */+LIBDEFLATEAPI enum libdeflate_result+libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,+ const void *in, size_t in_nbytes,+ void *out, size_t out_nbytes_avail,+ size_t *actual_in_nbytes_ret,+ size_t *actual_out_nbytes_ret);++/*+ * libdeflate_free_decompressor() frees a decompressor that was allocated with+ * libdeflate_alloc_decompressor(). If a NULL pointer is passed in, no action+ * is taken.+ */+LIBDEFLATEAPI void+libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);++/* ========================================================================== */+/* Checksums */+/* ========================================================================== */++/*+ * libdeflate_adler32() updates a running Adler-32 checksum with 'len' bytes of+ * data and returns the updated checksum. When starting a new checksum, the+ * required initial value for 'adler' is 1. This value is also returned when+ * 'buffer' is specified as NULL.+ */+LIBDEFLATEAPI uint32_t+libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);+++/*+ * libdeflate_crc32() updates a running CRC-32 checksum with 'len' bytes of data+ * and returns the updated checksum. When starting a new checksum, the required+ * initial value for 'crc' is 0. This value is also returned when 'buffer' is+ * specified as NULL.+ */+LIBDEFLATEAPI uint32_t+libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);++/* ========================================================================== */+/* Custom memory allocator */+/* ========================================================================== */++/*+ * Install a custom memory allocator which libdeflate will use for all memory+ * allocations by default. 'malloc_func' is a function that must behave like+ * malloc(), and 'free_func' is a function that must behave like free().+ *+ * The per-(de)compressor custom memory allocator that can be specified in+ * 'struct libdeflate_options' takes priority over this.+ *+ * This doesn't affect the free() function that will be used to free+ * (de)compressors that were already in existence when this is called.+ */+LIBDEFLATEAPI void+libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),+ void (*free_func)(void *));++/*+ * Advanced options. This is the options structure that+ * libdeflate_alloc_compressor_ex() and libdeflate_alloc_decompressor_ex()+ * require. Most users won't need this and should just use the non-"_ex"+ * functions instead. If you do need this, it should be initialized like this:+ *+ * struct libdeflate_options options;+ *+ * memset(&options, 0, sizeof(options));+ * options.sizeof_options = sizeof(options);+ * // Then set the fields that you need to override the defaults for.+ */+struct libdeflate_options {++ /*+ * This field must be set to the struct size. This field exists for+ * extensibility, so that fields can be appended to this struct in+ * future versions of libdeflate while still supporting old binaries.+ */+ size_t sizeof_options;++ /*+ * An optional custom memory allocator to use for this (de)compressor.+ * 'malloc_func' must be a function that behaves like malloc(), and+ * 'free_func' must be a function that behaves like free().+ *+ * This is useful in cases where a process might have multiple users of+ * libdeflate who want to use different memory allocators. For example,+ * a library might want to use libdeflate with a custom memory allocator+ * without interfering with user code that might use libdeflate too.+ *+ * This takes priority over the "global" memory allocator (which by+ * default is malloc() and free(), but can be changed by+ * libdeflate_set_memory_allocator()). Moreover, libdeflate will never+ * call the "global" memory allocator if a per-(de)compressor custom+ * allocator is always given.+ */+ void *(*malloc_func)(size_t);+ void (*free_func)(void *);+};++#ifdef __cplusplus+}+#endif++#endif /* LIBDEFLATE_H */