Skip to content

Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.

C

APFS: libdeflate's decompressor, as released, for the kernel to build in

vcvkk53940ea05e3c
diff --git a/tools/mk-libdeflate.sh b/tools/mk-libdeflate.sh
new file mode 100755
index 0000000..d057488
--- /dev/null
+++ b/tools/mk-libdeflate.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+# Put the decompressing half of libdeflate into the kernel tree, as bsd/miscfs/apfs/libdeflate/:
+# the APFS driver inflates the zlib pieces of compressed files with it instead of libkern's zlib.
+#
+# Why: on the restore-ramdisk volume -- 19149 pieces, 180 MB in, 1229 MB out -- libkern's zlib
+# inflates at 344 MB/s on the build machine and libdeflate at 740 MB/s, to the same bytes; and
+# on the phone zlib was 3819 of the 4227 ms one reader spent in vnop_read over that volume.
+#
+# Taken as it is from the release, files unchanged; what the kernel lacks (an allocator, no C
+# library) is in bsd/miscfs/apfs/apfs_deflate.c, which compiles these with -DFREESTANDING.
+# libdeflate is Copyright 2016 Eric Biggers and 2024 Google LLC, under the MIT licence (COPYING,
+# copied alongside).
+set -e
+VERSION=1.24
+SHA256=a0dda1c4b804742066db07b9510876edd09cc0ca06cdc32c5dfe1b2016a26463
+ROOT=$(cd "$(dirname "$(readlink -f "$0")")/.." && pwd)
+OUT=${XNU_SRC:-$ROOT/xnu-src/xnu-8796.121.2}/bsd/miscfs/apfs/libdeflate
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+curl -sL --fail -o "$TMP/l.tar.gz" \
+ https://github.com/ebiggers/libdeflate/releases/download/v$VERSION/libdeflate-$VERSION.tar.gz
+echo "$SHA256 $TMP/l.tar.gz" | sha256sum -c --quiet
+tar xzf "$TMP/l.tar.gz" -C "$TMP"
+S=$TMP/libdeflate-$VERSION
+rm -rf "$OUT"
+mkdir -p "$OUT/lib/arm"
+cp "$S/COPYING" "$S/libdeflate.h" "$S/common_defs.h" "$OUT/"
+for f in lib_common.h cpu_features_common.h deflate_constants.h zlib_constants.h \
+ decompress_template.h deflate_decompress.c zlib_decompress.c adler32.c; do
+ cp "$S/lib/$f" "$OUT/lib/"
+done
+# the architecture dispatch is included unconditionally; in the kernel it settles on NEON for
+# the Adler-32 and the portable code for the rest (see apfs_deflate.c)
+cp "$S/lib/arm/"*.h "$OUT/lib/arm/"
+echo "mk-libdeflate.sh: $OUT ($(du -sk "$OUT" | cut -f1) KB, libdeflate $VERSION)"
diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/COPYING b/xnu/files/bsd/miscfs/apfs/libdeflate/COPYING
new file mode 100644
index 0000000..9b620b3
--- /dev/null
+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/COPYING
@@ -0,0 +1,22 @@
+Copyright 2016 Eric Biggers
+Copyright 2024 Google LLC
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation files
+(the "Software"), to deal in the Software without restriction,
+including without limitation the rights to use, copy, modify, merge,
+publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/common_defs.h b/xnu/files/bsd/miscfs/apfs/libdeflate/common_defs.h
new file mode 100644
index 0000000..a0868d4
--- /dev/null
+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/common_defs.h
@@ -0,0 +1,748 @@
+/*
+ * common_defs.h
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef COMMON_DEFS_H
+#define COMMON_DEFS_H
+
+#include "libdeflate.h"
+
+#include <stdbool.h>
+#include <stddef.h> /* for size_t */
+#include <stdint.h>
+#ifdef _MSC_VER
+# include <intrin.h> /* for _BitScan*() and other intrinsics */
+# include <stdlib.h> /* for _byteswap_*() */
+ /* Disable MSVC warnings that are expected. */
+ /* /W2 */
+# pragma warning(disable : 4146) /* unary minus on unsigned type */
+ /* /W3 */
+# pragma warning(disable : 4018) /* signed/unsigned mismatch */
+# pragma warning(disable : 4244) /* possible loss of data */
+# pragma warning(disable : 4267) /* possible loss of precision */
+# pragma warning(disable : 4310) /* cast truncates constant value */
+ /* /W4 */
+# pragma warning(disable : 4100) /* unreferenced formal parameter */
+# pragma warning(disable : 4127) /* conditional expression is constant */
+# pragma warning(disable : 4189) /* local variable initialized but not referenced */
+# pragma warning(disable : 4232) /* nonstandard extension used */
+# pragma warning(disable : 4245) /* conversion from 'int' to 'unsigned int' */
+# pragma warning(disable : 4295) /* array too small to include terminating null */
+#endif
+#ifndef FREESTANDING
+# include <string.h> /* for memcpy() */
+#endif
+
+/* ========================================================================== */
+/* Target architecture */
+/* ========================================================================== */
+
+/* If possible, define a compiler-independent ARCH_* macro. */
+#undef ARCH_X86_64
+#undef ARCH_X86_32
+#undef ARCH_ARM64
+#undef ARCH_ARM32
+#undef ARCH_RISCV
+#ifdef _MSC_VER
+ /* Way too many things are broken in ARM64EC to pretend that it is x86_64. */
+# if defined(_M_X64) && !defined(_M_ARM64EC)
+# define ARCH_X86_64
+# elif defined(_M_IX86)
+# define ARCH_X86_32
+# elif defined(_M_ARM64)
+# define ARCH_ARM64
+# elif defined(_M_ARM)
+# define ARCH_ARM32
+# endif
+#else
+# if defined(__x86_64__)
+# define ARCH_X86_64
+# elif defined(__i386__)
+# define ARCH_X86_32
+# elif defined(__aarch64__)
+# define ARCH_ARM64
+# elif defined(__arm__)
+# define ARCH_ARM32
+# elif defined(__riscv)
+# define ARCH_RISCV
+# endif
+#endif
+
+/* ========================================================================== */
+/* Type definitions */
+/* ========================================================================== */
+
+/* Fixed-width integer types */
+typedef uint8_t u8;
+typedef uint16_t u16;
+typedef uint32_t u32;
+typedef uint64_t u64;
+typedef int8_t s8;
+typedef int16_t s16;
+typedef int32_t s32;
+typedef int64_t s64;
+
+/* ssize_t, if not available in <sys/types.h> */
+#ifdef _MSC_VER
+# ifdef _WIN64
+ typedef long long ssize_t;
+# else
+ typedef long ssize_t;
+# endif
+#endif
+
+/*
+ * Word type of the target architecture. Use 'size_t' instead of
+ * 'unsigned long' to account for platforms such as Windows that use 32-bit
+ * 'unsigned long' on 64-bit architectures.
+ */
+typedef size_t machine_word_t;
+
+/* Number of bytes in a word */
+#define WORDBYTES ((int)sizeof(machine_word_t))
+
+/* Number of bits in a word */
+#define WORDBITS (8 * WORDBYTES)
+
+/* ========================================================================== */
+/* Optional compiler features */
+/* ========================================================================== */
+
+/* Compiler version checks. Only use when absolutely necessary. */
+#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
+# define GCC_PREREQ(major, minor) \
+ (__GNUC__ > (major) || \
+ (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
+# if !GCC_PREREQ(4, 9)
+# error "gcc versions older than 4.9 are no longer supported"
+# endif
+#else
+# define GCC_PREREQ(major, minor) 0
+#endif
+#ifdef __clang__
+# ifdef __apple_build_version__
+# define CLANG_PREREQ(major, minor, apple_version) \
+ (__apple_build_version__ >= (apple_version))
+# else
+# define CLANG_PREREQ(major, minor, apple_version) \
+ (__clang_major__ > (major) || \
+ (__clang_major__ == (major) && __clang_minor__ >= (minor)))
+# endif
+# if !CLANG_PREREQ(3, 9, 8000000)
+# error "clang versions older than 3.9 are no longer supported"
+# endif
+#else
+# define CLANG_PREREQ(major, minor, apple_version) 0
+#endif
+#ifdef _MSC_VER
+# define MSVC_PREREQ(version) (_MSC_VER >= (version))
+# if !MSVC_PREREQ(1900)
+# error "MSVC versions older than Visual Studio 2015 are no longer supported"
+# endif
+#else
+# define MSVC_PREREQ(version) 0
+#endif
+
+/*
+ * __has_attribute(attribute) - check whether the compiler supports the given
+ * attribute (and also supports doing the check in the first place). Mostly
+ * useful just for clang, since gcc didn't add this macro until gcc 5.
+ */
+#ifndef __has_attribute
+# define __has_attribute(attribute) 0
+#endif
+
+/*
+ * __has_builtin(builtin) - check whether the compiler supports the given
+ * builtin (and also supports doing the check in the first place). Mostly
+ * useful just for clang, since gcc didn't add this macro until gcc 10.
+ */
+#ifndef __has_builtin
+# define __has_builtin(builtin) 0
+#endif
+
+/* inline - suggest that a function be inlined */
+#ifdef _MSC_VER
+# define inline __inline
+#endif /* else assume 'inline' is usable as-is */
+
+/* forceinline - force a function to be inlined, if possible */
+#if defined(__GNUC__) || __has_attribute(always_inline)
+# define forceinline inline __attribute__((always_inline))
+#elif defined(_MSC_VER)
+# define forceinline __forceinline
+#else
+# define forceinline inline
+#endif
+
+/* MAYBE_UNUSED - mark a function or variable as maybe unused */
+#if defined(__GNUC__) || __has_attribute(unused)
+# define MAYBE_UNUSED __attribute__((unused))
+#else
+# define MAYBE_UNUSED
+#endif
+
+/* NORETURN - mark a function as never returning, e.g. due to calling abort() */
+#if defined(__GNUC__) || __has_attribute(noreturn)
+# define NORETURN __attribute__((noreturn))
+#else
+# define NORETURN
+#endif
+
+/*
+ * restrict - hint that writes only occur through the given pointer.
+ *
+ * Don't use MSVC's __restrict, since it has nonstandard behavior.
+ * Standard restrict is okay, if it is supported.
+ */
+#if !defined(__STDC_VERSION__) || (__STDC_VERSION__ < 201112L)
+# if defined(__GNUC__) || defined(__clang__)
+# define restrict __restrict__
+# else
+# define restrict
+# endif
+#endif /* else assume 'restrict' is usable as-is */
+
+/* likely(expr) - hint that an expression is usually true */
+#if defined(__GNUC__) || __has_builtin(__builtin_expect)
+# define likely(expr) __builtin_expect(!!(expr), 1)
+#else
+# define likely(expr) (expr)
+#endif
+
+/* unlikely(expr) - hint that an expression is usually false */
+#if defined(__GNUC__) || __has_builtin(__builtin_expect)
+# define unlikely(expr) __builtin_expect(!!(expr), 0)
+#else
+# define unlikely(expr) (expr)
+#endif
+
+/* prefetchr(addr) - prefetch into L1 cache for read */
+#undef prefetchr
+#if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
+# define prefetchr(addr) __builtin_prefetch((addr), 0)
+#elif defined(_MSC_VER)
+# if defined(ARCH_X86_32) || defined(ARCH_X86_64)
+# define prefetchr(addr) _mm_prefetch((addr), _MM_HINT_T0)
+# elif defined(ARCH_ARM64)
+# define prefetchr(addr) __prefetch2((addr), 0x00 /* prfop=PLDL1KEEP */)
+# elif defined(ARCH_ARM32)
+# define prefetchr(addr) __prefetch(addr)
+# endif
+#endif
+#ifndef prefetchr
+# define prefetchr(addr)
+#endif
+
+/* prefetchw(addr) - prefetch into L1 cache for write */
+#undef prefetchw
+#if defined(__GNUC__) || __has_builtin(__builtin_prefetch)
+# define prefetchw(addr) __builtin_prefetch((addr), 1)
+#elif defined(_MSC_VER)
+# if defined(ARCH_X86_32) || defined(ARCH_X86_64)
+# define prefetchw(addr) _m_prefetchw(addr)
+# elif defined(ARCH_ARM64)
+# define prefetchw(addr) __prefetch2((addr), 0x10 /* prfop=PSTL1KEEP */)
+# elif defined(ARCH_ARM32)
+# define prefetchw(addr) __prefetchw(addr)
+# endif
+#endif
+#ifndef prefetchw
+# define prefetchw(addr)
+#endif
+
+/*
+ * _aligned_attribute(n) - declare that the annotated variable, or variables of
+ * the annotated type, must be aligned on n-byte boundaries.
+ */
+#undef _aligned_attribute
+#if defined(__GNUC__) || __has_attribute(aligned)
+# define _aligned_attribute(n) __attribute__((aligned(n)))
+#elif defined(_MSC_VER)
+# define _aligned_attribute(n) __declspec(align(n))
+#endif
+
+/*
+ * _target_attribute(attrs) - override the compilation target for a function.
+ *
+ * This accepts one or more comma-separated suffixes to the -m prefix jointly
+ * forming the name of a machine-dependent option. On gcc-like compilers, this
+ * enables codegen for the given targets, including arbitrary compiler-generated
+ * code as well as the corresponding intrinsics. On other compilers this macro
+ * expands to nothing, though MSVC allows intrinsics to be used anywhere anyway.
+ */
+#if defined(__GNUC__) || __has_attribute(target)
+# define _target_attribute(attrs) __attribute__((target(attrs)))
+#else
+# define _target_attribute(attrs)
+#endif
+
+/* ========================================================================== */
+/* Miscellaneous macros */
+/* ========================================================================== */
+
+#define ARRAY_LEN(A) (sizeof(A) / sizeof((A)[0]))
+#define MIN(a, b) ((a) <= (b) ? (a) : (b))
+#define MAX(a, b) ((a) >= (b) ? (a) : (b))
+#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
+#define STATIC_ASSERT(expr) ((void)sizeof(char[1 - 2 * !(expr)]))
+#define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1))
+#define ROUND_UP(n, d) ((d) * DIV_ROUND_UP((n), (d)))
+
+/* ========================================================================== */
+/* Endianness handling */
+/* ========================================================================== */
+
+/*
+ * CPU_IS_LITTLE_ENDIAN() - 1 if the CPU is little endian, or 0 if it is big
+ * endian. When possible this is a compile-time macro that can be used in
+ * preprocessor conditionals. As a fallback, a generic method is used that
+ * can't be used in preprocessor conditionals but should still be optimized out.
+ */
+#if defined(__BYTE_ORDER__) /* gcc v4.6+ and clang */
+# define CPU_IS_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+#elif defined(_MSC_VER)
+# define CPU_IS_LITTLE_ENDIAN() true
+#else
+static forceinline bool CPU_IS_LITTLE_ENDIAN(void)
+{
+ union {
+ u32 w;
+ u8 b;
+ } u;
+
+ u.w = 1;
+ return u.b;
+}
+#endif
+
+/* bswap16(v) - swap the bytes of a 16-bit integer */
+static forceinline u16 bswap16(u16 v)
+{
+#if defined(__GNUC__) || __has_builtin(__builtin_bswap16)
+ return __builtin_bswap16(v);
+#elif defined(_MSC_VER)
+ return _byteswap_ushort(v);
+#else
+ return (v << 8) | (v >> 8);
+#endif
+}
+
+/* bswap32(v) - swap the bytes of a 32-bit integer */
+static forceinline u32 bswap32(u32 v)
+{
+#if defined(__GNUC__) || __has_builtin(__builtin_bswap32)
+ return __builtin_bswap32(v);
+#elif defined(_MSC_VER)
+ return _byteswap_ulong(v);
+#else
+ return ((v & 0x000000FF) << 24) |
+ ((v & 0x0000FF00) << 8) |
+ ((v & 0x00FF0000) >> 8) |
+ ((v & 0xFF000000) >> 24);
+#endif
+}
+
+/* bswap64(v) - swap the bytes of a 64-bit integer */
+static forceinline u64 bswap64(u64 v)
+{
+#if defined(__GNUC__) || __has_builtin(__builtin_bswap64)
+ return __builtin_bswap64(v);
+#elif defined(_MSC_VER)
+ return _byteswap_uint64(v);
+#else
+ return ((v & 0x00000000000000FF) << 56) |
+ ((v & 0x000000000000FF00) << 40) |
+ ((v & 0x0000000000FF0000) << 24) |
+ ((v & 0x00000000FF000000) << 8) |
+ ((v & 0x000000FF00000000) >> 8) |
+ ((v & 0x0000FF0000000000) >> 24) |
+ ((v & 0x00FF000000000000) >> 40) |
+ ((v & 0xFF00000000000000) >> 56);
+#endif
+}
+
+#define le16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap16(v))
+#define le32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap32(v))
+#define le64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? (v) : bswap64(v))
+#define be16_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap16(v) : (v))
+#define be32_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap32(v) : (v))
+#define be64_bswap(v) (CPU_IS_LITTLE_ENDIAN() ? bswap64(v) : (v))
+
+/* ========================================================================== */
+/* Unaligned memory accesses */
+/* ========================================================================== */
+
+/*
+ * UNALIGNED_ACCESS_IS_FAST() - 1 if unaligned memory accesses can be performed
+ * efficiently on the target platform, otherwise 0.
+ */
+#if (defined(__GNUC__) || defined(__clang__)) && \
+ (defined(ARCH_X86_64) || defined(ARCH_X86_32) || \
+ defined(__ARM_FEATURE_UNALIGNED) || defined(__powerpc64__) || \
+ defined(__riscv_misaligned_fast) || \
+ /*
+ * For all compilation purposes, WebAssembly behaves like any other CPU
+ * instruction set. Even though WebAssembly engine might be running on
+ * top of different actual CPU architectures, the WebAssembly spec
+ * itself permits unaligned access and it will be fast on most of those
+ * platforms, and simulated at the engine level on others, so it's
+ * worth treating it as a CPU architecture with fast unaligned access.
+ */ defined(__wasm__))
+# define UNALIGNED_ACCESS_IS_FAST 1
+#elif defined(_MSC_VER)
+# define UNALIGNED_ACCESS_IS_FAST 1
+#else
+# define UNALIGNED_ACCESS_IS_FAST 0
+#endif
+
+/*
+ * Implementing unaligned memory accesses using memcpy() is portable, and it
+ * usually gets optimized appropriately by modern compilers. I.e., each
+ * memcpy() of 1, 2, 4, or WORDBYTES bytes gets compiled to a load or store
+ * instruction, not to an actual function call.
+ *
+ * We no longer use the "packed struct" approach to unaligned accesses, as that
+ * is nonstandard, has unclear semantics, and doesn't receive enough testing
+ * (see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94994).
+ *
+ * arm32 with __ARM_FEATURE_UNALIGNED in gcc 5 and earlier is a known exception
+ * where memcpy() generates inefficient code
+ * (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67366). However, we no longer
+ * consider that one case important enough to maintain different code for.
+ * If you run into it, please just use a newer version of gcc (or use clang).
+ */
+
+#ifdef FREESTANDING
+# define MEMCOPY __builtin_memcpy
+#else
+# define MEMCOPY memcpy
+#endif
+
+/* Unaligned loads and stores without endianness conversion */
+
+#define DEFINE_UNALIGNED_TYPE(type) \
+static forceinline type \
+load_##type##_unaligned(const void *p) \
+{ \
+ type v; \
+ \
+ MEMCOPY(&v, p, sizeof(v)); \
+ return v; \
+} \
+ \
+static forceinline void \
+store_##type##_unaligned(type v, void *p) \
+{ \
+ MEMCOPY(p, &v, sizeof(v)); \
+}
+
+DEFINE_UNALIGNED_TYPE(u16)
+DEFINE_UNALIGNED_TYPE(u32)
+DEFINE_UNALIGNED_TYPE(u64)
+DEFINE_UNALIGNED_TYPE(machine_word_t)
+
+#undef MEMCOPY
+
+#define load_word_unaligned load_machine_word_t_unaligned
+#define store_word_unaligned store_machine_word_t_unaligned
+
+/* Unaligned loads with endianness conversion */
+
+static forceinline u16
+get_unaligned_le16(const u8 *p)
+{
+ if (UNALIGNED_ACCESS_IS_FAST)
+ return le16_bswap(load_u16_unaligned(p));
+ else
+ return ((u16)p[1] << 8) | p[0];
+}
+
+static forceinline u16
+get_unaligned_be16(const u8 *p)
+{
+ if (UNALIGNED_ACCESS_IS_FAST)
+ return be16_bswap(load_u16_unaligned(p));
+ else
+ return ((u16)p[0] << 8) | p[1];
+}
+
+static forceinline u32
+get_unaligned_le32(const u8 *p)
+{
+ if (UNALIGNED_ACCESS_IS_FAST)
+ return le32_bswap(load_u32_unaligned(p));
+ else
+ return ((u32)p[3] << 24) | ((u32)p[2] << 16) |
+ ((u32)p[1] << 8) | p[0];
+}
+
+static forceinline u32
+get_unaligned_be32(const u8 *p)
+{
+ if (UNALIGNED_ACCESS_IS_FAST)
+ return be32_bswap(load_u32_unaligned(p));
+ else
+ return ((u32)p[0] << 24) | ((u32)p[1] << 16) |
+ ((u32)p[2] << 8) | p[3];
+}
+
+static forceinline u64
+get_unaligned_le64(const u8 *p)
+{
+ if (UNALIGNED_ACCESS_IS_FAST)
+ return le64_bswap(load_u64_unaligned(p));
+ else
+ return ((u64)p[7] << 56) | ((u64)p[6] << 48) |
+ ((u64)p[5] << 40) | ((u64)p[4] << 32) |
+ ((u64)p[3] << 24) | ((u64)p[2] << 16) |
+ ((u64)p[1] << 8) | p[0];
+}
+
+static forceinline machine_word_t
+get_unaligned_leword(const u8 *p)
+{
+ STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
+ if (WORDBITS == 32)
+ return get_unaligned_le32(p);
+ else
+ return get_unaligned_le64(p);
+}
+
+/* Unaligned stores with endianness conversion */
+
+static forceinline void
+put_unaligned_le16(u16 v, u8 *p)
+{
+ if (UNALIGNED_ACCESS_IS_FAST) {
+ store_u16_unaligned(le16_bswap(v), p);
+ } else {
+ p[0] = (u8)(v >> 0);
+ p[1] = (u8)(v >> 8);
+ }
+}
+
+static forceinline void
+put_unaligned_be16(u16 v, u8 *p)
+{
+ if (UNALIGNED_ACCESS_IS_FAST) {
+ store_u16_unaligned(be16_bswap(v), p);
+ } else {
+ p[0] = (u8)(v >> 8);
+ p[1] = (u8)(v >> 0);
+ }
+}
+
+static forceinline void
+put_unaligned_le32(u32 v, u8 *p)
+{
+ if (UNALIGNED_ACCESS_IS_FAST) {
+ store_u32_unaligned(le32_bswap(v), p);
+ } else {
+ p[0] = (u8)(v >> 0);
+ p[1] = (u8)(v >> 8);
+ p[2] = (u8)(v >> 16);
+ p[3] = (u8)(v >> 24);
+ }
+}
+
+static forceinline void
+put_unaligned_be32(u32 v, u8 *p)
+{
+ if (UNALIGNED_ACCESS_IS_FAST) {
+ store_u32_unaligned(be32_bswap(v), p);
+ } else {
+ p[0] = (u8)(v >> 24);
+ p[1] = (u8)(v >> 16);
+ p[2] = (u8)(v >> 8);
+ p[3] = (u8)(v >> 0);
+ }
+}
+
+static forceinline void
+put_unaligned_le64(u64 v, u8 *p)
+{
+ if (UNALIGNED_ACCESS_IS_FAST) {
+ store_u64_unaligned(le64_bswap(v), p);
+ } else {
+ p[0] = (u8)(v >> 0);
+ p[1] = (u8)(v >> 8);
+ p[2] = (u8)(v >> 16);
+ p[3] = (u8)(v >> 24);
+ p[4] = (u8)(v >> 32);
+ p[5] = (u8)(v >> 40);
+ p[6] = (u8)(v >> 48);
+ p[7] = (u8)(v >> 56);
+ }
+}
+
+static forceinline void
+put_unaligned_leword(machine_word_t v, u8 *p)
+{
+ STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
+ if (WORDBITS == 32)
+ put_unaligned_le32(v, p);
+ else
+ put_unaligned_le64(v, p);
+}
+
+/* ========================================================================== */
+/* Bit manipulation functions */
+/* ========================================================================== */
+
+/*
+ * Bit Scan Reverse (BSR) - find the 0-based index (relative to the least
+ * significant end) of the *most* significant 1 bit in the input value. The
+ * input value must be nonzero!
+ */
+
+static forceinline unsigned
+bsr32(u32 v)
+{
+#if defined(__GNUC__) || __has_builtin(__builtin_clz)
+ return 31 - __builtin_clz(v);
+#elif defined(_MSC_VER)
+ unsigned long i;
+
+ _BitScanReverse(&i, v);
+ return i;
+#else
+ unsigned i = 0;
+
+ while ((v >>= 1) != 0)
+ i++;
+ return i;
+#endif
+}
+
+static forceinline unsigned
+bsr64(u64 v)
+{
+#if defined(__GNUC__) || __has_builtin(__builtin_clzll)
+ return 63 - __builtin_clzll(v);
+#elif defined(_MSC_VER) && defined(_WIN64)
+ unsigned long i;
+
+ _BitScanReverse64(&i, v);
+ return i;
+#else
+ unsigned i = 0;
+
+ while ((v >>= 1) != 0)
+ i++;
+ return i;
+#endif
+}
+
+static forceinline unsigned
+bsrw(machine_word_t v)
+{
+ STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
+ if (WORDBITS == 32)
+ return bsr32(v);
+ else
+ return bsr64(v);
+}
+
+/*
+ * Bit Scan Forward (BSF) - find the 0-based index (relative to the least
+ * significant end) of the *least* significant 1 bit in the input value. The
+ * input value must be nonzero!
+ */
+
+static forceinline unsigned
+bsf32(u32 v)
+{
+#if defined(__GNUC__) || __has_builtin(__builtin_ctz)
+ return __builtin_ctz(v);
+#elif defined(_MSC_VER)
+ unsigned long i;
+
+ _BitScanForward(&i, v);
+ return i;
+#else
+ unsigned i = 0;
+
+ for (; (v & 1) == 0; v >>= 1)
+ i++;
+ return i;
+#endif
+}
+
+static forceinline unsigned
+bsf64(u64 v)
+{
+#if defined(__GNUC__) || __has_builtin(__builtin_ctzll)
+ return __builtin_ctzll(v);
+#elif defined(_MSC_VER) && defined(_WIN64)
+ unsigned long i;
+
+ _BitScanForward64(&i, v);
+ return i;
+#else
+ unsigned i = 0;
+
+ for (; (v & 1) == 0; v >>= 1)
+ i++;
+ return i;
+#endif
+}
+
+static forceinline unsigned
+bsfw(machine_word_t v)
+{
+ STATIC_ASSERT(WORDBITS == 32 || WORDBITS == 64);
+ if (WORDBITS == 32)
+ return bsf32(v);
+ else
+ return bsf64(v);
+}
+
+/*
+ * rbit32(v): reverse the bits in a 32-bit integer. This doesn't have a
+ * fallback implementation; use '#ifdef rbit32' to check if this is available.
+ */
+#undef rbit32
+#if (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM32) && \
+ (__ARM_ARCH >= 7 || (__ARM_ARCH == 6 && defined(__ARM_ARCH_6T2__)))
+static forceinline u32
+rbit32(u32 v)
+{
+ __asm__("rbit %0, %1" : "=r" (v) : "r" (v));
+ return v;
+}
+#define rbit32 rbit32
+#elif (defined(__GNUC__) || defined(__clang__)) && defined(ARCH_ARM64)
+static forceinline u32
+rbit32(u32 v)
+{
+ __asm__("rbit %w0, %w1" : "=r" (v) : "r" (v));
+ return v;
+}
+#define rbit32 rbit32
+#endif
+
+#endif /* COMMON_DEFS_H */
diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/adler32.c b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/adler32.c
new file mode 100644
index 0000000..d5f39d8
--- /dev/null
+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/adler32.c
@@ -0,0 +1,162 @@
+/*
+ * adler32.c - Adler-32 checksum algorithm
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "lib_common.h"
+
+/* The Adler-32 divisor, or "base", value */
+#define DIVISOR 65521
+
+/*
+ * MAX_CHUNK_LEN is the most bytes that can be processed without the possibility
+ * of s2 overflowing when it is represented as an unsigned 32-bit integer. This
+ * value was computed using the following Python script:
+ *
+ * divisor = 65521
+ * count = 0
+ * s1 = divisor - 1
+ * s2 = divisor - 1
+ * while True:
+ * s1 += 0xFF
+ * s2 += s1
+ * if s2 > 0xFFFFFFFF:
+ * break
+ * count += 1
+ * print(count)
+ *
+ * Note that to get the correct worst-case value, we must assume that every byte
+ * has value 0xFF and that s1 and s2 started with the highest possible values
+ * modulo the divisor.
+ */
+#define MAX_CHUNK_LEN 5552
+
+/*
+ * Update the Adler-32 values s1 and s2 using n bytes from p, update p to p + n,
+ * update n to 0, and reduce s1 and s2 mod DIVISOR. It is assumed that neither
+ * s1 nor s2 can overflow before the reduction at the end, i.e. n plus any bytes
+ * already processed after the last reduction must not exceed MAX_CHUNK_LEN.
+ *
+ * This uses only portable C code. This is used as a fallback when a vectorized
+ * implementation of Adler-32 (e.g. AVX2) is unavailable on the platform.
+ *
+ * Some of the vectorized implementations also use this to handle the end of the
+ * data when the data isn't evenly divisible by the length the vectorized code
+ * works on. To avoid compiler errors about target-specific option mismatches
+ * when this is used in that way, this is a macro rather than a function.
+ *
+ * Although this is unvectorized, this does include an optimization where the
+ * main loop processes four bytes at a time using a strategy similar to that
+ * used by vectorized implementations. This provides increased instruction-
+ * level parallelism compared to the traditional 's1 += *p++; s2 += s1;'.
+ */
+#define ADLER32_CHUNK(s1, s2, p, n) \
+do { \
+ if (n >= 4) { \
+ u32 s1_sum = 0; \
+ u32 byte_0_sum = 0; \
+ u32 byte_1_sum = 0; \
+ u32 byte_2_sum = 0; \
+ u32 byte_3_sum = 0; \
+ \
+ do { \
+ s1_sum += s1; \
+ s1 += p[0] + p[1] + p[2] + p[3]; \
+ byte_0_sum += p[0]; \
+ byte_1_sum += p[1]; \
+ byte_2_sum += p[2]; \
+ byte_3_sum += p[3]; \
+ p += 4; \
+ n -= 4; \
+ } while (n >= 4); \
+ s2 += (4 * (s1_sum + byte_0_sum)) + (3 * byte_1_sum) + \
+ (2 * byte_2_sum) + byte_3_sum; \
+ } \
+ for (; n; n--, p++) { \
+ s1 += *p; \
+ s2 += s1; \
+ } \
+ s1 %= DIVISOR; \
+ s2 %= DIVISOR; \
+} while (0)
+
+static u32 MAYBE_UNUSED
+adler32_generic(u32 adler, const u8 *p, size_t len)
+{
+ u32 s1 = adler & 0xFFFF;
+ u32 s2 = adler >> 16;
+
+ while (len) {
+ size_t n = MIN(len, MAX_CHUNK_LEN & ~3);
+
+ len -= n;
+ ADLER32_CHUNK(s1, s2, p, n);
+ }
+
+ return (s2 << 16) | s1;
+}
+
+/* Include architecture-specific implementation(s) if available. */
+#undef DEFAULT_IMPL
+#undef arch_select_adler32_func
+typedef u32 (*adler32_func_t)(u32 adler, const u8 *p, size_t len);
+#if defined(ARCH_ARM32) || defined(ARCH_ARM64)
+# include "arm/adler32_impl.h"
+#elif defined(ARCH_X86_32) || defined(ARCH_X86_64)
+# include "x86/adler32_impl.h"
+#endif
+
+#ifndef DEFAULT_IMPL
+# define DEFAULT_IMPL adler32_generic
+#endif
+
+#ifdef arch_select_adler32_func
+static u32 dispatch_adler32(u32 adler, const u8 *p, size_t len);
+
+static volatile adler32_func_t adler32_impl = dispatch_adler32;
+
+/* Choose the best implementation at runtime. */
+static u32 dispatch_adler32(u32 adler, const u8 *p, size_t len)
+{
+ adler32_func_t f = arch_select_adler32_func();
+
+ if (f == NULL)
+ f = DEFAULT_IMPL;
+
+ adler32_impl = f;
+ return f(adler, p, len);
+}
+#else
+/* The best implementation is statically known, so call it directly. */
+#define adler32_impl DEFAULT_IMPL
+#endif
+
+LIBDEFLATEAPI u32
+libdeflate_adler32(u32 adler, const void *buffer, size_t len)
+{
+ if (buffer == NULL) /* Return initial value. */
+ return 1;
+ return adler32_impl(adler, buffer, len);
+}
diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/adler32_impl.h b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/adler32_impl.h
new file mode 100644
index 0000000..66d2f2b
--- /dev/null
+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/adler32_impl.h
@@ -0,0 +1,365 @@
+/*
+ * arm/adler32_impl.h - ARM implementations of Adler-32 checksum algorithm
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_ARM_ADLER32_IMPL_H
+#define LIB_ARM_ADLER32_IMPL_H
+
+#include "cpu_features.h"
+
+/* Regular NEON implementation */
+#if HAVE_NEON_INTRIN && CPU_IS_LITTLE_ENDIAN()
+# define adler32_arm_neon adler32_arm_neon
+# if HAVE_NEON_NATIVE
+ /*
+ * Use no attributes if none are needed, to support old versions of clang
+ * that don't accept the simd target attribute.
+ */
+# define ATTRIBUTES
+# elif defined(ARCH_ARM32)
+# define ATTRIBUTES _target_attribute("fpu=neon")
+# elif defined(__clang__)
+# define ATTRIBUTES _target_attribute("simd")
+# else
+# define ATTRIBUTES _target_attribute("+simd")
+# endif
+static ATTRIBUTES MAYBE_UNUSED u32
+adler32_arm_neon(u32 adler, const u8 *p, size_t len)
+{
+ static const u16 _aligned_attribute(16) mults[64] = {
+ 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49,
+ 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33,
+ 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
+ 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1,
+ };
+ const uint16x8_t mults_a = vld1q_u16(&mults[0]);
+ const uint16x8_t mults_b = vld1q_u16(&mults[8]);
+ const uint16x8_t mults_c = vld1q_u16(&mults[16]);
+ const uint16x8_t mults_d = vld1q_u16(&mults[24]);
+ const uint16x8_t mults_e = vld1q_u16(&mults[32]);
+ const uint16x8_t mults_f = vld1q_u16(&mults[40]);
+ const uint16x8_t mults_g = vld1q_u16(&mults[48]);
+ const uint16x8_t mults_h = vld1q_u16(&mults[56]);
+ u32 s1 = adler & 0xFFFF;
+ u32 s2 = adler >> 16;
+
+ /*
+ * If the length is large and the pointer is misaligned, align it.
+ * For smaller lengths, just take the misaligned load penalty.
+ */
+ if (unlikely(len > 32768 && ((uintptr_t)p & 15))) {
+ do {
+ s1 += *p++;
+ s2 += s1;
+ len--;
+ } while ((uintptr_t)p & 15);
+ s1 %= DIVISOR;
+ s2 %= DIVISOR;
+ }
+
+ while (len) {
+ /*
+ * Calculate the length of the next data chunk such that s1 and
+ * s2 are guaranteed to not exceed UINT32_MAX.
+ */
+ size_t n = MIN(len, MAX_CHUNK_LEN & ~63);
+
+ len -= n;
+
+ if (n >= 64) {
+ uint32x4_t v_s1 = vdupq_n_u32(0);
+ uint32x4_t v_s2 = vdupq_n_u32(0);
+ /*
+ * v_byte_sums_* contain the sum of the bytes at index i
+ * across all 64-byte segments, for each index 0..63.
+ */
+ uint16x8_t v_byte_sums_a = vdupq_n_u16(0);
+ uint16x8_t v_byte_sums_b = vdupq_n_u16(0);
+ uint16x8_t v_byte_sums_c = vdupq_n_u16(0);
+ uint16x8_t v_byte_sums_d = vdupq_n_u16(0);
+ uint16x8_t v_byte_sums_e = vdupq_n_u16(0);
+ uint16x8_t v_byte_sums_f = vdupq_n_u16(0);
+ uint16x8_t v_byte_sums_g = vdupq_n_u16(0);
+ uint16x8_t v_byte_sums_h = vdupq_n_u16(0);
+
+ s2 += s1 * (n & ~63);
+
+ do {
+ /* Load the next 64 data bytes. */
+ const uint8x16_t data_a = vld1q_u8(p + 0);
+ const uint8x16_t data_b = vld1q_u8(p + 16);
+ const uint8x16_t data_c = vld1q_u8(p + 32);
+ const uint8x16_t data_d = vld1q_u8(p + 48);
+ uint16x8_t tmp;
+
+ /*
+ * Accumulate the previous s1 counters into the
+ * s2 counters. The needed multiplication by 64
+ * is delayed to later.
+ */
+ v_s2 = vaddq_u32(v_s2, v_s1);
+
+ /*
+ * Add the 64 data bytes to their v_byte_sums
+ * counters, while also accumulating the sums of
+ * each adjacent set of 4 bytes into v_s1.
+ */
+ tmp = vpaddlq_u8(data_a);
+ v_byte_sums_a = vaddw_u8(v_byte_sums_a,
+ vget_low_u8(data_a));
+ v_byte_sums_b = vaddw_u8(v_byte_sums_b,
+ vget_high_u8(data_a));
+ tmp = vpadalq_u8(tmp, data_b);
+ v_byte_sums_c = vaddw_u8(v_byte_sums_c,
+ vget_low_u8(data_b));
+ v_byte_sums_d = vaddw_u8(v_byte_sums_d,
+ vget_high_u8(data_b));
+ tmp = vpadalq_u8(tmp, data_c);
+ v_byte_sums_e = vaddw_u8(v_byte_sums_e,
+ vget_low_u8(data_c));
+ v_byte_sums_f = vaddw_u8(v_byte_sums_f,
+ vget_high_u8(data_c));
+ tmp = vpadalq_u8(tmp, data_d);
+ v_byte_sums_g = vaddw_u8(v_byte_sums_g,
+ vget_low_u8(data_d));
+ v_byte_sums_h = vaddw_u8(v_byte_sums_h,
+ vget_high_u8(data_d));
+ v_s1 = vpadalq_u16(v_s1, tmp);
+
+ p += 64;
+ n -= 64;
+ } while (n >= 64);
+
+ /* s2 = 64*s2 + (64*bytesum0 + 63*bytesum1 + ... + 1*bytesum63) */
+ #ifdef ARCH_ARM32
+ # define umlal2(a, b, c) vmlal_u16((a), vget_high_u16(b), vget_high_u16(c))
+ #else
+ # define umlal2 vmlal_high_u16
+ #endif
+ v_s2 = vqshlq_n_u32(v_s2, 6);
+ v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_a),
+ vget_low_u16(mults_a));
+ v_s2 = umlal2(v_s2, v_byte_sums_a, mults_a);
+ v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_b),
+ vget_low_u16(mults_b));
+ v_s2 = umlal2(v_s2, v_byte_sums_b, mults_b);
+ v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_c),
+ vget_low_u16(mults_c));
+ v_s2 = umlal2(v_s2, v_byte_sums_c, mults_c);
+ v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_d),
+ vget_low_u16(mults_d));
+ v_s2 = umlal2(v_s2, v_byte_sums_d, mults_d);
+ v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_e),
+ vget_low_u16(mults_e));
+ v_s2 = umlal2(v_s2, v_byte_sums_e, mults_e);
+ v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_f),
+ vget_low_u16(mults_f));
+ v_s2 = umlal2(v_s2, v_byte_sums_f, mults_f);
+ v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_g),
+ vget_low_u16(mults_g));
+ v_s2 = umlal2(v_s2, v_byte_sums_g, mults_g);
+ v_s2 = vmlal_u16(v_s2, vget_low_u16(v_byte_sums_h),
+ vget_low_u16(mults_h));
+ v_s2 = umlal2(v_s2, v_byte_sums_h, mults_h);
+ #undef umlal2
+
+ /* Horizontal sum to finish up */
+ #ifdef ARCH_ARM32
+ s1 += vgetq_lane_u32(v_s1, 0) + vgetq_lane_u32(v_s1, 1) +
+ vgetq_lane_u32(v_s1, 2) + vgetq_lane_u32(v_s1, 3);
+ s2 += vgetq_lane_u32(v_s2, 0) + vgetq_lane_u32(v_s2, 1) +
+ vgetq_lane_u32(v_s2, 2) + vgetq_lane_u32(v_s2, 3);
+ #else
+ s1 += vaddvq_u32(v_s1);
+ s2 += vaddvq_u32(v_s2);
+ #endif
+ }
+ /*
+ * Process the last 0 <= n < 64 bytes of the chunk using
+ * scalar instructions and reduce s1 and s2 mod DIVISOR.
+ */
+ ADLER32_CHUNK(s1, s2, p, n);
+ }
+ return (s2 << 16) | s1;
+}
+#undef ATTRIBUTES
+#endif /* Regular NEON implementation */
+
+/* NEON+dotprod implementation */
+#if HAVE_DOTPROD_INTRIN && CPU_IS_LITTLE_ENDIAN() && \
+ !defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_DOTPROD)
+# define adler32_arm_neon_dotprod adler32_arm_neon_dotprod
+# ifdef __clang__
+# define ATTRIBUTES _target_attribute("dotprod")
+ /*
+ * Both gcc and binutils originally considered dotprod to depend on
+ * arch=armv8.2-a or later. This was fixed in gcc 13.2 by commit
+ * 9aac37ab8a7b ("aarch64: Remove architecture dependencies from intrinsics")
+ * and in binutils 2.41 by commit 205e4380c800 ("aarch64: Remove version
+ * dependencies from features"). Unfortunately, always using arch=armv8.2-a
+ * causes build errors with some compiler options because it may reduce the
+ * arch rather than increase it. Therefore we try to omit the arch whenever
+ * possible. If gcc is 14 or later, then both gcc and binutils are probably
+ * fixed, so we omit the arch. We also omit the arch if a feature that
+ * depends on armv8.2-a or later (in gcc 13.1 and earlier) is present.
+ */
+# elif GCC_PREREQ(14, 0) || defined(__ARM_FEATURE_JCVT) \
+ || defined(__ARM_FEATURE_DOTPROD)
+# define ATTRIBUTES _target_attribute("+dotprod")
+# else
+# define ATTRIBUTES _target_attribute("arch=armv8.2-a+dotprod")
+# endif
+static ATTRIBUTES u32
+adler32_arm_neon_dotprod(u32 adler, const u8 *p, size_t len)
+{
+ static const u8 _aligned_attribute(16) mults[64] = {
+ 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49,
+ 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33,
+ 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17,
+ 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1,
+ };
+ const uint8x16_t mults_a = vld1q_u8(&mults[0]);
+ const uint8x16_t mults_b = vld1q_u8(&mults[16]);
+ const uint8x16_t mults_c = vld1q_u8(&mults[32]);
+ const uint8x16_t mults_d = vld1q_u8(&mults[48]);
+ const uint8x16_t ones = vdupq_n_u8(1);
+ u32 s1 = adler & 0xFFFF;
+ u32 s2 = adler >> 16;
+
+ /*
+ * If the length is large and the pointer is misaligned, align it.
+ * For smaller lengths, just take the misaligned load penalty.
+ */
+ if (unlikely(len > 32768 && ((uintptr_t)p & 15))) {
+ do {
+ s1 += *p++;
+ s2 += s1;
+ len--;
+ } while ((uintptr_t)p & 15);
+ s1 %= DIVISOR;
+ s2 %= DIVISOR;
+ }
+
+ while (len) {
+ /*
+ * Calculate the length of the next data chunk such that s1 and
+ * s2 are guaranteed to not exceed UINT32_MAX.
+ */
+ size_t n = MIN(len, MAX_CHUNK_LEN & ~63);
+
+ len -= n;
+
+ if (n >= 64) {
+ uint32x4_t v_s1_a = vdupq_n_u32(0);
+ uint32x4_t v_s1_b = vdupq_n_u32(0);
+ uint32x4_t v_s1_c = vdupq_n_u32(0);
+ uint32x4_t v_s1_d = vdupq_n_u32(0);
+ uint32x4_t v_s2_a = vdupq_n_u32(0);
+ uint32x4_t v_s2_b = vdupq_n_u32(0);
+ uint32x4_t v_s2_c = vdupq_n_u32(0);
+ uint32x4_t v_s2_d = vdupq_n_u32(0);
+ uint32x4_t v_s1_sums_a = vdupq_n_u32(0);
+ uint32x4_t v_s1_sums_b = vdupq_n_u32(0);
+ uint32x4_t v_s1_sums_c = vdupq_n_u32(0);
+ uint32x4_t v_s1_sums_d = vdupq_n_u32(0);
+ uint32x4_t v_s1;
+ uint32x4_t v_s2;
+ uint32x4_t v_s1_sums;
+
+ s2 += s1 * (n & ~63);
+
+ do {
+ uint8x16_t data_a = vld1q_u8(p + 0);
+ uint8x16_t data_b = vld1q_u8(p + 16);
+ uint8x16_t data_c = vld1q_u8(p + 32);
+ uint8x16_t data_d = vld1q_u8(p + 48);
+
+ v_s1_sums_a = vaddq_u32(v_s1_sums_a, v_s1_a);
+ v_s1_a = vdotq_u32(v_s1_a, data_a, ones);
+ v_s2_a = vdotq_u32(v_s2_a, data_a, mults_a);
+
+ v_s1_sums_b = vaddq_u32(v_s1_sums_b, v_s1_b);
+ v_s1_b = vdotq_u32(v_s1_b, data_b, ones);
+ v_s2_b = vdotq_u32(v_s2_b, data_b, mults_b);
+
+ v_s1_sums_c = vaddq_u32(v_s1_sums_c, v_s1_c);
+ v_s1_c = vdotq_u32(v_s1_c, data_c, ones);
+ v_s2_c = vdotq_u32(v_s2_c, data_c, mults_c);
+
+ v_s1_sums_d = vaddq_u32(v_s1_sums_d, v_s1_d);
+ v_s1_d = vdotq_u32(v_s1_d, data_d, ones);
+ v_s2_d = vdotq_u32(v_s2_d, data_d, mults_d);
+
+ p += 64;
+ n -= 64;
+ } while (n >= 64);
+
+ v_s1 = vaddq_u32(vaddq_u32(v_s1_a, v_s1_b),
+ vaddq_u32(v_s1_c, v_s1_d));
+ v_s2 = vaddq_u32(vaddq_u32(v_s2_a, v_s2_b),
+ vaddq_u32(v_s2_c, v_s2_d));
+ v_s1_sums = vaddq_u32(vaddq_u32(v_s1_sums_a,
+ v_s1_sums_b),
+ vaddq_u32(v_s1_sums_c,
+ v_s1_sums_d));
+ v_s2 = vaddq_u32(v_s2, vqshlq_n_u32(v_s1_sums, 6));
+
+ s1 += vaddvq_u32(v_s1);
+ s2 += vaddvq_u32(v_s2);
+ }
+ /*
+ * Process the last 0 <= n < 64 bytes of the chunk using
+ * scalar instructions and reduce s1 and s2 mod DIVISOR.
+ */
+ ADLER32_CHUNK(s1, s2, p, n);
+ }
+ return (s2 << 16) | s1;
+}
+#undef ATTRIBUTES
+#endif /* NEON+dotprod implementation */
+
+#if defined(adler32_arm_neon_dotprod) && defined(__ARM_FEATURE_DOTPROD)
+#define DEFAULT_IMPL adler32_arm_neon_dotprod
+#else
+static inline adler32_func_t
+arch_select_adler32_func(void)
+{
+ const u32 features MAYBE_UNUSED = get_arm_cpu_features();
+
+#ifdef adler32_arm_neon_dotprod
+ if (HAVE_NEON(features) && HAVE_DOTPROD(features))
+ return adler32_arm_neon_dotprod;
+#endif
+#ifdef adler32_arm_neon
+ if (HAVE_NEON(features))
+ return adler32_arm_neon;
+#endif
+ return NULL;
+}
+#define arch_select_adler32_func arch_select_adler32_func
+#endif
+
+#endif /* LIB_ARM_ADLER32_IMPL_H */
diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/cpu_features.h b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/cpu_features.h
new file mode 100644
index 0000000..dc9ab8a
--- /dev/null
+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/cpu_features.h
@@ -0,0 +1,214 @@
+/*
+ * arm/cpu_features.h - feature detection for ARM CPUs
+ *
+ * Copyright 2018 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_ARM_CPU_FEATURES_H
+#define LIB_ARM_CPU_FEATURES_H
+
+#include "../lib_common.h"
+
+#if defined(ARCH_ARM32) || defined(ARCH_ARM64)
+
+#define ARM_CPU_FEATURE_NEON (1 << 0)
+#define ARM_CPU_FEATURE_PMULL (1 << 1)
+/*
+ * PREFER_PMULL indicates that the CPU has very high pmull throughput, and so
+ * the 12x wide pmull-based CRC-32 implementation is likely to be faster than an
+ * implementation based on the crc32 instructions.
+ */
+#define ARM_CPU_FEATURE_PREFER_PMULL (1 << 2)
+#define ARM_CPU_FEATURE_CRC32 (1 << 3)
+#define ARM_CPU_FEATURE_SHA3 (1 << 4)
+#define ARM_CPU_FEATURE_DOTPROD (1 << 5)
+
+#if !defined(FREESTANDING) && \
+ (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)) && \
+ (defined(__linux__) || \
+ (defined(__APPLE__) && defined(ARCH_ARM64)) || \
+ (defined(_WIN32) && defined(ARCH_ARM64)))
+/* Runtime ARM CPU feature detection is supported. */
+# define ARM_CPU_FEATURES_KNOWN (1U << 31)
+extern volatile u32 libdeflate_arm_cpu_features;
+
+void libdeflate_init_arm_cpu_features(void);
+
+static inline u32 get_arm_cpu_features(void)
+{
+ if (libdeflate_arm_cpu_features == 0)
+ libdeflate_init_arm_cpu_features();
+ return libdeflate_arm_cpu_features;
+}
+#else
+static inline u32 get_arm_cpu_features(void) { return 0; }
+#endif
+
+/* NEON */
+#if defined(__ARM_NEON) || (defined(_MSC_VER) && defined(ARCH_ARM64))
+# define HAVE_NEON(features) 1
+# define HAVE_NEON_NATIVE 1
+#else
+# define HAVE_NEON(features) ((features) & ARM_CPU_FEATURE_NEON)
+# define HAVE_NEON_NATIVE 0
+#endif
+/*
+ * With both gcc and clang, NEON intrinsics require that the main target has
+ * NEON enabled already. Exception: with gcc 6.1 and later (r230411 for arm32,
+ * r226563 for arm64), hardware floating point support is sufficient.
+ */
+#if (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER)) && \
+ (HAVE_NEON_NATIVE || (GCC_PREREQ(6, 1) && defined(__ARM_FP)))
+# define HAVE_NEON_INTRIN 1
+# include <arm_neon.h>
+#else
+# define HAVE_NEON_INTRIN 0
+#endif
+
+/* PMULL */
+#ifdef __ARM_FEATURE_CRYPTO
+# define HAVE_PMULL(features) 1
+#else
+# define HAVE_PMULL(features) ((features) & ARM_CPU_FEATURE_PMULL)
+#endif
+#if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
+ (GCC_PREREQ(7, 1) || defined(__clang__) || defined(_MSC_VER)) && \
+ CPU_IS_LITTLE_ENDIAN() /* untested on big endian */
+# define HAVE_PMULL_INTRIN 1
+ /* Work around MSVC's vmull_p64() taking poly64x1_t instead of poly64_t */
+# ifdef _MSC_VER
+# define compat_vmull_p64(a, b) vmull_p64(vcreate_p64(a), vcreate_p64(b))
+# else
+# define compat_vmull_p64(a, b) vmull_p64((a), (b))
+# endif
+#else
+# define HAVE_PMULL_INTRIN 0
+#endif
+
+/* CRC32 */
+#ifdef __ARM_FEATURE_CRC32
+# define HAVE_CRC32(features) 1
+#else
+# define HAVE_CRC32(features) ((features) & ARM_CPU_FEATURE_CRC32)
+#endif
+#if defined(ARCH_ARM64) && \
+ (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER))
+# define HAVE_CRC32_INTRIN 1
+# if defined(__GNUC__) || defined(__clang__)
+# include <arm_acle.h>
+# endif
+ /*
+ * Use an inline assembly fallback for clang 15 and earlier, which only
+ * defined the crc32 intrinsics when crc32 is enabled in the main target.
+ */
+# if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
+ !defined(__ARM_FEATURE_CRC32)
+# undef __crc32b
+# define __crc32b(a, b) \
+ ({ uint32_t res; \
+ __asm__("crc32b %w0, %w1, %w2" \
+ : "=r" (res) : "r" (a), "r" (b)); \
+ res; })
+# undef __crc32h
+# define __crc32h(a, b) \
+ ({ uint32_t res; \
+ __asm__("crc32h %w0, %w1, %w2" \
+ : "=r" (res) : "r" (a), "r" (b)); \
+ res; })
+# undef __crc32w
+# define __crc32w(a, b) \
+ ({ uint32_t res; \
+ __asm__("crc32w %w0, %w1, %w2" \
+ : "=r" (res) : "r" (a), "r" (b)); \
+ res; })
+# undef __crc32d
+# define __crc32d(a, b) \
+ ({ uint32_t res; \
+ __asm__("crc32x %w0, %w1, %2" \
+ : "=r" (res) : "r" (a), "r" (b)); \
+ res; })
+# pragma clang diagnostic ignored "-Wgnu-statement-expression"
+# endif
+#else
+# define HAVE_CRC32_INTRIN 0
+#endif
+
+/* SHA3 (needed for the eor3 instruction) */
+#ifdef __ARM_FEATURE_SHA3
+# define HAVE_SHA3(features) 1
+#else
+# define HAVE_SHA3(features) ((features) & ARM_CPU_FEATURE_SHA3)
+#endif
+#if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
+ (GCC_PREREQ(9, 1) /* r268049 */ || \
+ CLANG_PREREQ(7, 0, 10010463) /* r338010 */)
+# define HAVE_SHA3_INTRIN 1
+ /*
+ * Use an inline assembly fallback for clang 15 and earlier, which only
+ * defined the sha3 intrinsics when sha3 is enabled in the main target.
+ */
+# if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
+ !defined(__ARM_FEATURE_SHA3)
+# undef veor3q_u8
+# define veor3q_u8(a, b, c) \
+ ({ uint8x16_t res; \
+ __asm__("eor3 %0.16b, %1.16b, %2.16b, %3.16b" \
+ : "=w" (res) : "w" (a), "w" (b), "w" (c)); \
+ res; })
+# pragma clang diagnostic ignored "-Wgnu-statement-expression"
+# endif
+#else
+# define HAVE_SHA3_INTRIN 0
+#endif
+
+/* dotprod */
+#ifdef __ARM_FEATURE_DOTPROD
+# define HAVE_DOTPROD(features) 1
+#else
+# define HAVE_DOTPROD(features) ((features) & ARM_CPU_FEATURE_DOTPROD)
+#endif
+#if defined(ARCH_ARM64) && HAVE_NEON_INTRIN && \
+ (GCC_PREREQ(8, 1) || CLANG_PREREQ(7, 0, 10010000) || defined(_MSC_VER))
+# define HAVE_DOTPROD_INTRIN 1
+ /*
+ * Use an inline assembly fallback for clang 15 and earlier, which only
+ * defined the dotprod intrinsics when dotprod is enabled in the main target.
+ */
+# if defined(__clang__) && !CLANG_PREREQ(16, 0, 16000000) && \
+ !defined(__ARM_FEATURE_DOTPROD)
+# undef vdotq_u32
+# define vdotq_u32(a, b, c) \
+ ({ uint32x4_t res = (a); \
+ __asm__("udot %0.4s, %1.16b, %2.16b" \
+ : "+w" (res) : "w" (b), "w" (c)); \
+ res; })
+# pragma clang diagnostic ignored "-Wgnu-statement-expression"
+# endif
+#else
+# define HAVE_DOTPROD_INTRIN 0
+#endif
+
+#endif /* ARCH_ARM32 || ARCH_ARM64 */
+
+#endif /* LIB_ARM_CPU_FEATURES_H */
diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/crc32_impl.h b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/crc32_impl.h
new file mode 100644
index 0000000..696af5a
--- /dev/null
+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/crc32_impl.h
@@ -0,0 +1,594 @@
+/*
+ * arm/crc32_impl.h - ARM implementations of the gzip CRC-32 algorithm
+ *
+ * Copyright 2022 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_ARM_CRC32_IMPL_H
+#define LIB_ARM_CRC32_IMPL_H
+
+#include "cpu_features.h"
+
+/*
+ * crc32_arm_crc() - implementation using crc32 instructions (only)
+ *
+ * In general this implementation is straightforward. However, naive use of the
+ * crc32 instructions is serial: one of the two inputs to each crc32 instruction
+ * is the output of the previous one. To take advantage of CPUs that can
+ * execute multiple crc32 instructions in parallel, when possible we interleave
+ * the checksumming of several adjacent chunks, then combine their CRCs.
+ *
+ * However, without pmull, combining CRCs is fairly slow. So in this pmull-less
+ * version, we only use a large chunk length, and thus we only do chunked
+ * processing if there is a lot of data to checksum. This also means that a
+ * variable chunk length wouldn't help much, so we just support a fixed length.
+ */
+#if HAVE_CRC32_INTRIN
+# ifdef __clang__
+# define ATTRIBUTES _target_attribute("crc")
+# else
+# define ATTRIBUTES _target_attribute("+crc")
+# endif
+
+/*
+ * Combine the CRCs for 4 adjacent chunks of length L = CRC32_FIXED_CHUNK_LEN
+ * bytes each by computing:
+ *
+ * [ crc0*x^(3*8*L) + crc1*x^(2*8*L) + crc2*x^(1*8*L) + crc3 ] mod G(x)
+ *
+ * This has been optimized in several ways:
+ *
+ * - The needed multipliers (x to some power, reduced mod G(x)) were
+ * precomputed.
+ *
+ * - The 3 multiplications are interleaved.
+ *
+ * - The reduction mod G(x) is delayed to the end and done using __crc32d.
+ * Note that the use of __crc32d introduces an extra factor of x^32. To
+ * cancel that out along with the extra factor of x^1 that gets introduced
+ * because of how the 63-bit products are aligned in their 64-bit integers,
+ * the multipliers are actually x^(j*8*L - 33) instead of x^(j*8*L).
+ */
+static forceinline ATTRIBUTES u32
+combine_crcs_slow(u32 crc0, u32 crc1, u32 crc2, u32 crc3)
+{
+ u64 res0 = 0, res1 = 0, res2 = 0;
+ int i;
+
+ /* Multiply crc{0,1,2} by CRC32_FIXED_CHUNK_MULT_{3,2,1}. */
+ for (i = 0; i < 32; i++) {
+ if (CRC32_FIXED_CHUNK_MULT_3 & (1U << i))
+ res0 ^= (u64)crc0 << i;
+ if (CRC32_FIXED_CHUNK_MULT_2 & (1U << i))
+ res1 ^= (u64)crc1 << i;
+ if (CRC32_FIXED_CHUNK_MULT_1 & (1U << i))
+ res2 ^= (u64)crc2 << i;
+ }
+ /* Add the different parts and reduce mod G(x). */
+ return __crc32d(0, res0 ^ res1 ^ res2) ^ crc3;
+}
+
+#define crc32_arm_crc crc32_arm_crc
+static ATTRIBUTES u32
+crc32_arm_crc(u32 crc, const u8 *p, size_t len)
+{
+ if (len >= 64) {
+ const size_t align = -(uintptr_t)p & 7;
+
+ /* Align p to the next 8-byte boundary. */
+ if (align) {
+ if (align & 1)
+ crc = __crc32b(crc, *p++);
+ if (align & 2) {
+ crc = __crc32h(crc, le16_bswap(*(u16 *)p));
+ p += 2;
+ }
+ if (align & 4) {
+ crc = __crc32w(crc, le32_bswap(*(u32 *)p));
+ p += 4;
+ }
+ len -= align;
+ }
+ /*
+ * Interleave the processing of multiple adjacent data chunks to
+ * take advantage of instruction-level parallelism.
+ *
+ * Some CPUs don't prefetch the data if it's being fetched in
+ * multiple interleaved streams, so do explicit prefetching.
+ */
+ while (len >= CRC32_NUM_CHUNKS * CRC32_FIXED_CHUNK_LEN) {
+ const u64 *wp0 = (const u64 *)p;
+ const u64 * const wp0_end =
+ (const u64 *)(p + CRC32_FIXED_CHUNK_LEN);
+ u32 crc1 = 0, crc2 = 0, crc3 = 0;
+
+ STATIC_ASSERT(CRC32_NUM_CHUNKS == 4);
+ STATIC_ASSERT(CRC32_FIXED_CHUNK_LEN % (4 * 8) == 0);
+ do {
+ prefetchr(&wp0[64 + 0*CRC32_FIXED_CHUNK_LEN/8]);
+ prefetchr(&wp0[64 + 1*CRC32_FIXED_CHUNK_LEN/8]);
+ prefetchr(&wp0[64 + 2*CRC32_FIXED_CHUNK_LEN/8]);
+ prefetchr(&wp0[64 + 3*CRC32_FIXED_CHUNK_LEN/8]);
+ crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_FIXED_CHUNK_LEN/8]));
+ crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_FIXED_CHUNK_LEN/8]));
+ crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_FIXED_CHUNK_LEN/8]));
+ crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_FIXED_CHUNK_LEN/8]));
+ wp0++;
+ crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_FIXED_CHUNK_LEN/8]));
+ crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_FIXED_CHUNK_LEN/8]));
+ crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_FIXED_CHUNK_LEN/8]));
+ crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_FIXED_CHUNK_LEN/8]));
+ wp0++;
+ crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_FIXED_CHUNK_LEN/8]));
+ crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_FIXED_CHUNK_LEN/8]));
+ crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_FIXED_CHUNK_LEN/8]));
+ crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_FIXED_CHUNK_LEN/8]));
+ wp0++;
+ crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_FIXED_CHUNK_LEN/8]));
+ crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_FIXED_CHUNK_LEN/8]));
+ crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_FIXED_CHUNK_LEN/8]));
+ crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_FIXED_CHUNK_LEN/8]));
+ wp0++;
+ } while (wp0 != wp0_end);
+ crc = combine_crcs_slow(crc, crc1, crc2, crc3);
+ p += CRC32_NUM_CHUNKS * CRC32_FIXED_CHUNK_LEN;
+ len -= CRC32_NUM_CHUNKS * CRC32_FIXED_CHUNK_LEN;
+ }
+ /*
+ * Due to the large fixed chunk length used above, there might
+ * still be a lot of data left. So use a 64-byte loop here,
+ * instead of a loop that is less unrolled.
+ */
+ while (len >= 64) {
+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 0)));
+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 8)));
+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 16)));
+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 24)));
+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 32)));
+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 40)));
+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 48)));
+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 56)));
+ p += 64;
+ len -= 64;
+ }
+ }
+ if (len & 32) {
+ crc = __crc32d(crc, get_unaligned_le64(p + 0));
+ crc = __crc32d(crc, get_unaligned_le64(p + 8));
+ crc = __crc32d(crc, get_unaligned_le64(p + 16));
+ crc = __crc32d(crc, get_unaligned_le64(p + 24));
+ p += 32;
+ }
+ if (len & 16) {
+ crc = __crc32d(crc, get_unaligned_le64(p + 0));
+ crc = __crc32d(crc, get_unaligned_le64(p + 8));
+ p += 16;
+ }
+ if (len & 8) {
+ crc = __crc32d(crc, get_unaligned_le64(p));
+ p += 8;
+ }
+ if (len & 4) {
+ crc = __crc32w(crc, get_unaligned_le32(p));
+ p += 4;
+ }
+ if (len & 2) {
+ crc = __crc32h(crc, get_unaligned_le16(p));
+ p += 2;
+ }
+ if (len & 1)
+ crc = __crc32b(crc, *p);
+ return crc;
+}
+#undef ATTRIBUTES
+#endif /* crc32_arm_crc() */
+
+/*
+ * crc32_arm_crc_pmullcombine() - implementation using crc32 instructions, plus
+ * pmull instructions for CRC combining
+ *
+ * This is similar to crc32_arm_crc(), but it enables the use of pmull
+ * (carryless multiplication) instructions for the steps where the CRCs of
+ * adjacent data chunks are combined. As this greatly speeds up CRC
+ * combination, this implementation also differs from crc32_arm_crc() in that it
+ * uses a variable chunk length which can get fairly small. The precomputed
+ * multipliers needed for the selected chunk length are loaded from a table.
+ *
+ * Note that pmull is used here only for combining the CRCs of separately
+ * checksummed chunks, not for folding the data itself. See crc32_arm_pmull*()
+ * for implementations that use pmull for folding the data itself.
+ */
+#if HAVE_CRC32_INTRIN && HAVE_PMULL_INTRIN
+# ifdef __clang__
+# define ATTRIBUTES _target_attribute("crc,aes")
+# else
+# define ATTRIBUTES _target_attribute("+crc,+crypto")
+# endif
+
+/* Do carryless multiplication of two 32-bit values. */
+static forceinline ATTRIBUTES u64
+clmul_u32(u32 a, u32 b)
+{
+ uint64x2_t res = vreinterpretq_u64_p128(
+ compat_vmull_p64((poly64_t)a, (poly64_t)b));
+
+ return vgetq_lane_u64(res, 0);
+}
+
+/*
+ * Like combine_crcs_slow(), but uses vmull_p64 to do the multiplications more
+ * quickly, and supports a variable chunk length. The chunk length is
+ * 'i * CRC32_MIN_VARIABLE_CHUNK_LEN'
+ * where 1 <= i < ARRAY_LEN(crc32_mults_for_chunklen).
+ */
+static forceinline ATTRIBUTES u32
+combine_crcs_fast(u32 crc0, u32 crc1, u32 crc2, u32 crc3, size_t i)
+{
+ u64 res0 = clmul_u32(crc0, crc32_mults_for_chunklen[i][0]);
+ u64 res1 = clmul_u32(crc1, crc32_mults_for_chunklen[i][1]);
+ u64 res2 = clmul_u32(crc2, crc32_mults_for_chunklen[i][2]);
+
+ return __crc32d(0, res0 ^ res1 ^ res2) ^ crc3;
+}
+
+#define crc32_arm_crc_pmullcombine crc32_arm_crc_pmullcombine
+static ATTRIBUTES u32
+crc32_arm_crc_pmullcombine(u32 crc, const u8 *p, size_t len)
+{
+ const size_t align = -(uintptr_t)p & 7;
+
+ if (len >= align + CRC32_NUM_CHUNKS * CRC32_MIN_VARIABLE_CHUNK_LEN) {
+ /* Align p to the next 8-byte boundary. */
+ if (align) {
+ if (align & 1)
+ crc = __crc32b(crc, *p++);
+ if (align & 2) {
+ crc = __crc32h(crc, le16_bswap(*(u16 *)p));
+ p += 2;
+ }
+ if (align & 4) {
+ crc = __crc32w(crc, le32_bswap(*(u32 *)p));
+ p += 4;
+ }
+ len -= align;
+ }
+ /*
+ * Handle CRC32_MAX_VARIABLE_CHUNK_LEN specially, so that better
+ * code is generated for it.
+ */
+ while (len >= CRC32_NUM_CHUNKS * CRC32_MAX_VARIABLE_CHUNK_LEN) {
+ const u64 *wp0 = (const u64 *)p;
+ const u64 * const wp0_end =
+ (const u64 *)(p + CRC32_MAX_VARIABLE_CHUNK_LEN);
+ u32 crc1 = 0, crc2 = 0, crc3 = 0;
+
+ STATIC_ASSERT(CRC32_NUM_CHUNKS == 4);
+ STATIC_ASSERT(CRC32_MAX_VARIABLE_CHUNK_LEN % (4 * 8) == 0);
+ do {
+ prefetchr(&wp0[64 + 0*CRC32_MAX_VARIABLE_CHUNK_LEN/8]);
+ prefetchr(&wp0[64 + 1*CRC32_MAX_VARIABLE_CHUNK_LEN/8]);
+ prefetchr(&wp0[64 + 2*CRC32_MAX_VARIABLE_CHUNK_LEN/8]);
+ prefetchr(&wp0[64 + 3*CRC32_MAX_VARIABLE_CHUNK_LEN/8]);
+ crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+ crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+ crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+ crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+ wp0++;
+ crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+ crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+ crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+ crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+ wp0++;
+ crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+ crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+ crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+ crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+ wp0++;
+ crc = __crc32d(crc, le64_bswap(wp0[0*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+ crc1 = __crc32d(crc1, le64_bswap(wp0[1*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+ crc2 = __crc32d(crc2, le64_bswap(wp0[2*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+ crc3 = __crc32d(crc3, le64_bswap(wp0[3*CRC32_MAX_VARIABLE_CHUNK_LEN/8]));
+ wp0++;
+ } while (wp0 != wp0_end);
+ crc = combine_crcs_fast(crc, crc1, crc2, crc3,
+ ARRAY_LEN(crc32_mults_for_chunklen) - 1);
+ p += CRC32_NUM_CHUNKS * CRC32_MAX_VARIABLE_CHUNK_LEN;
+ len -= CRC32_NUM_CHUNKS * CRC32_MAX_VARIABLE_CHUNK_LEN;
+ }
+ /* Handle up to one variable-length chunk. */
+ if (len >= CRC32_NUM_CHUNKS * CRC32_MIN_VARIABLE_CHUNK_LEN) {
+ const size_t i = len / (CRC32_NUM_CHUNKS *
+ CRC32_MIN_VARIABLE_CHUNK_LEN);
+ const size_t chunk_len =
+ i * CRC32_MIN_VARIABLE_CHUNK_LEN;
+ const u64 *wp0 = (const u64 *)(p + 0*chunk_len);
+ const u64 *wp1 = (const u64 *)(p + 1*chunk_len);
+ const u64 *wp2 = (const u64 *)(p + 2*chunk_len);
+ const u64 *wp3 = (const u64 *)(p + 3*chunk_len);
+ const u64 * const wp0_end = wp1;
+ u32 crc1 = 0, crc2 = 0, crc3 = 0;
+
+ STATIC_ASSERT(CRC32_NUM_CHUNKS == 4);
+ STATIC_ASSERT(CRC32_MIN_VARIABLE_CHUNK_LEN % (4 * 8) == 0);
+ do {
+ prefetchr(wp0 + 64);
+ prefetchr(wp1 + 64);
+ prefetchr(wp2 + 64);
+ prefetchr(wp3 + 64);
+ crc = __crc32d(crc, le64_bswap(*wp0++));
+ crc1 = __crc32d(crc1, le64_bswap(*wp1++));
+ crc2 = __crc32d(crc2, le64_bswap(*wp2++));
+ crc3 = __crc32d(crc3, le64_bswap(*wp3++));
+ crc = __crc32d(crc, le64_bswap(*wp0++));
+ crc1 = __crc32d(crc1, le64_bswap(*wp1++));
+ crc2 = __crc32d(crc2, le64_bswap(*wp2++));
+ crc3 = __crc32d(crc3, le64_bswap(*wp3++));
+ crc = __crc32d(crc, le64_bswap(*wp0++));
+ crc1 = __crc32d(crc1, le64_bswap(*wp1++));
+ crc2 = __crc32d(crc2, le64_bswap(*wp2++));
+ crc3 = __crc32d(crc3, le64_bswap(*wp3++));
+ crc = __crc32d(crc, le64_bswap(*wp0++));
+ crc1 = __crc32d(crc1, le64_bswap(*wp1++));
+ crc2 = __crc32d(crc2, le64_bswap(*wp2++));
+ crc3 = __crc32d(crc3, le64_bswap(*wp3++));
+ } while (wp0 != wp0_end);
+ crc = combine_crcs_fast(crc, crc1, crc2, crc3, i);
+ p += CRC32_NUM_CHUNKS * chunk_len;
+ len -= CRC32_NUM_CHUNKS * chunk_len;
+ }
+
+ while (len >= 32) {
+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 0)));
+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 8)));
+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 16)));
+ crc = __crc32d(crc, le64_bswap(*(u64 *)(p + 24)));
+ p += 32;
+ len -= 32;
+ }
+ } else {
+ while (len >= 32) {
+ crc = __crc32d(crc, get_unaligned_le64(p + 0));
+ crc = __crc32d(crc, get_unaligned_le64(p + 8));
+ crc = __crc32d(crc, get_unaligned_le64(p + 16));
+ crc = __crc32d(crc, get_unaligned_le64(p + 24));
+ p += 32;
+ len -= 32;
+ }
+ }
+ if (len & 16) {
+ crc = __crc32d(crc, get_unaligned_le64(p + 0));
+ crc = __crc32d(crc, get_unaligned_le64(p + 8));
+ p += 16;
+ }
+ if (len & 8) {
+ crc = __crc32d(crc, get_unaligned_le64(p));
+ p += 8;
+ }
+ if (len & 4) {
+ crc = __crc32w(crc, get_unaligned_le32(p));
+ p += 4;
+ }
+ if (len & 2) {
+ crc = __crc32h(crc, get_unaligned_le16(p));
+ p += 2;
+ }
+ if (len & 1)
+ crc = __crc32b(crc, *p);
+ return crc;
+}
+#undef ATTRIBUTES
+#endif /* crc32_arm_crc_pmullcombine() */
+
+/*
+ * crc32_arm_pmullx4() - implementation using "folding" with pmull instructions
+ *
+ * This implementation is intended for CPUs that support pmull instructions but
+ * not crc32 instructions.
+ */
+#if HAVE_PMULL_INTRIN
+# define crc32_arm_pmullx4 crc32_arm_pmullx4
+# define SUFFIX _pmullx4
+# ifdef __clang__
+ /*
+ * This used to use "crypto", but that stopped working with clang 16.
+ * Now only "aes" works. "aes" works with older versions too, so use
+ * that. No "+" prefix; clang 15 and earlier doesn't accept that.
+ */
+# define ATTRIBUTES _target_attribute("aes")
+# else
+ /*
+ * With gcc, only "+crypto" works. Both the "+" prefix and the
+ * "crypto" (not "aes") are essential...
+ */
+# define ATTRIBUTES _target_attribute("+crypto")
+# endif
+# define ENABLE_EOR3 0
+# include "crc32_pmull_helpers.h"
+
+static ATTRIBUTES u32
+crc32_arm_pmullx4(u32 crc, const u8 *p, size_t len)
+{
+ static const u64 _aligned_attribute(16) mults[3][2] = {
+ { CRC32_X159_MODG, CRC32_X95_MODG }, /* 1 vecs */
+ { CRC32_X543_MODG, CRC32_X479_MODG }, /* 4 vecs */
+ { CRC32_X287_MODG, CRC32_X223_MODG }, /* 2 vecs */
+ };
+ static const u64 _aligned_attribute(16) barrett_consts[3][2] = {
+ { CRC32_X95_MODG, },
+ { CRC32_BARRETT_CONSTANT_1, },
+ { CRC32_BARRETT_CONSTANT_2, },
+ };
+ const poly64x2_t multipliers_1 = load_multipliers(mults[0]);
+ uint8x16_t v0, v1, v2, v3;
+
+ if (len < 64 + 15) {
+ if (len < 16)
+ return crc32_slice1(crc, p, len);
+ v0 = veorq_u8(vld1q_u8(p), u32_to_bytevec(crc));
+ p += 16;
+ len -= 16;
+ while (len >= 16) {
+ v0 = fold_vec(v0, vld1q_u8(p), multipliers_1);
+ p += 16;
+ len -= 16;
+ }
+ } else {
+ const poly64x2_t multipliers_4 = load_multipliers(mults[1]);
+ const poly64x2_t multipliers_2 = load_multipliers(mults[2]);
+ const size_t align = -(uintptr_t)p & 15;
+ const uint8x16_t *vp;
+
+ v0 = veorq_u8(vld1q_u8(p), u32_to_bytevec(crc));
+ p += 16;
+ /* Align p to the next 16-byte boundary. */
+ if (align) {
+ v0 = fold_partial_vec(v0, p, align, multipliers_1);
+ p += align;
+ len -= align;
+ }
+ vp = (const uint8x16_t *)p;
+ v1 = *vp++;
+ v2 = *vp++;
+ v3 = *vp++;
+ while (len >= 64 + 64) {
+ v0 = fold_vec(v0, *vp++, multipliers_4);
+ v1 = fold_vec(v1, *vp++, multipliers_4);
+ v2 = fold_vec(v2, *vp++, multipliers_4);
+ v3 = fold_vec(v3, *vp++, multipliers_4);
+ len -= 64;
+ }
+ v0 = fold_vec(v0, v2, multipliers_2);
+ v1 = fold_vec(v1, v3, multipliers_2);
+ if (len & 32) {
+ v0 = fold_vec(v0, *vp++, multipliers_2);
+ v1 = fold_vec(v1, *vp++, multipliers_2);
+ }
+ v0 = fold_vec(v0, v1, multipliers_1);
+ if (len & 16)
+ v0 = fold_vec(v0, *vp++, multipliers_1);
+ p = (const u8 *)vp;
+ len &= 15;
+ }
+
+ /* Handle any remaining partial block now before reducing to 32 bits. */
+ if (len)
+ v0 = fold_partial_vec(v0, p, len, multipliers_1);
+
+ /* Reduce to 32 bits, following lib/x86/crc32_pclmul_template.h */
+ v0 = veorq_u8(clmul_low(v0, load_multipliers(barrett_consts[0])),
+ vextq_u8(v0, vdupq_n_u8(0), 8));
+ v1 = clmul_low(v0, load_multipliers(barrett_consts[1]));
+ v1 = clmul_low(v1, load_multipliers(barrett_consts[2]));
+ v0 = veorq_u8(v0, v1);
+ return vgetq_lane_u32(vreinterpretq_u32_u8(v0), 2);
+}
+#undef SUFFIX
+#undef ATTRIBUTES
+#undef ENABLE_EOR3
+#endif /* crc32_arm_pmullx4() */
+
+/*
+ * crc32_arm_pmullx12_crc() - large-stride implementation using "folding" with
+ * pmull instructions, where crc32 instructions are also available
+ *
+ * See crc32_pmull_wide.h for explanation.
+ */
+#if HAVE_PMULL_INTRIN && HAVE_CRC32_INTRIN
+# define crc32_arm_pmullx12_crc crc32_arm_pmullx12_crc
+# define SUFFIX _pmullx12_crc
+# ifdef __clang__
+# define ATTRIBUTES _target_attribute("aes,crc")
+# else
+# define ATTRIBUTES _target_attribute("+crypto,+crc")
+# endif
+# define ENABLE_EOR3 0
+# include "crc32_pmull_wide.h"
+#endif
+
+/*
+ * crc32_arm_pmullx12_crc_eor3()
+ *
+ * This like crc32_arm_pmullx12_crc(), but it adds the eor3 instruction (from
+ * the sha3 extension) for even better performance.
+ */
+#if HAVE_PMULL_INTRIN && HAVE_CRC32_INTRIN && HAVE_SHA3_INTRIN && \
+ !defined(LIBDEFLATE_ASSEMBLER_DOES_NOT_SUPPORT_SHA3)
+# define crc32_arm_pmullx12_crc_eor3 crc32_arm_pmullx12_crc_eor3
+# define SUFFIX _pmullx12_crc_eor3
+# ifdef __clang__
+# define ATTRIBUTES _target_attribute("aes,crc,sha3")
+ /*
+ * Both gcc and binutils originally considered sha3 to depend on
+ * arch=armv8.2-a or later. This was fixed in gcc 13.2 by commit
+ * 9aac37ab8a7b ("aarch64: Remove architecture dependencies from intrinsics")
+ * and in binutils 2.41 by commit 205e4380c800 ("aarch64: Remove version
+ * dependencies from features"). Unfortunately, always using arch=armv8.2-a
+ * causes build errors with some compiler options because it may reduce the
+ * arch rather than increase it. Therefore we try to omit the arch whenever
+ * possible. If gcc is 14 or later, then both gcc and binutils are probably
+ * fixed, so we omit the arch. We also omit the arch if a feature that
+ * depends on armv8.2-a or later (in gcc 13.1 and earlier) is present.
+ */
+# elif GCC_PREREQ(14, 0) || defined(__ARM_FEATURE_JCVT) \
+ || defined(__ARM_FEATURE_DOTPROD)
+# define ATTRIBUTES _target_attribute("+crypto,+crc,+sha3")
+# else
+# define ATTRIBUTES _target_attribute("arch=armv8.2-a+crypto+crc+sha3")
+# endif
+# define ENABLE_EOR3 1
+# include "crc32_pmull_wide.h"
+#endif
+
+static inline crc32_func_t
+arch_select_crc32_func(void)
+{
+ const u32 features MAYBE_UNUSED = get_arm_cpu_features();
+
+#ifdef crc32_arm_pmullx12_crc_eor3
+ if ((features & ARM_CPU_FEATURE_PREFER_PMULL) &&
+ HAVE_PMULL(features) && HAVE_CRC32(features) && HAVE_SHA3(features))
+ return crc32_arm_pmullx12_crc_eor3;
+#endif
+#ifdef crc32_arm_pmullx12_crc
+ if ((features & ARM_CPU_FEATURE_PREFER_PMULL) &&
+ HAVE_PMULL(features) && HAVE_CRC32(features))
+ return crc32_arm_pmullx12_crc;
+#endif
+#ifdef crc32_arm_crc_pmullcombine
+ if (HAVE_CRC32(features) && HAVE_PMULL(features))
+ return crc32_arm_crc_pmullcombine;
+#endif
+#ifdef crc32_arm_crc
+ if (HAVE_CRC32(features))
+ return crc32_arm_crc;
+#endif
+#ifdef crc32_arm_pmullx4
+ if (HAVE_PMULL(features))
+ return crc32_arm_pmullx4;
+#endif
+ return NULL;
+}
+#define arch_select_crc32_func arch_select_crc32_func
+
+#endif /* LIB_ARM_CRC32_IMPL_H */
diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/crc32_pmull_helpers.h b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/crc32_pmull_helpers.h
new file mode 100644
index 0000000..023b904
--- /dev/null
+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/crc32_pmull_helpers.h
@@ -0,0 +1,156 @@
+/*
+ * arm/crc32_pmull_helpers.h - helper functions for CRC-32 folding with PMULL
+ *
+ * Copyright 2022 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * This file is a "template" for instantiating helper functions for CRC folding
+ * with pmull instructions. It accepts the following parameters:
+ *
+ * SUFFIX:
+ * Name suffix to append to all instantiated functions.
+ * ATTRIBUTES:
+ * Target function attributes to use.
+ * ENABLE_EOR3:
+ * Use the eor3 instruction (from the sha3 extension).
+ */
+
+/* Create a vector with 'a' in the first 4 bytes, and the rest zeroed out. */
+#undef u32_to_bytevec
+static forceinline ATTRIBUTES uint8x16_t
+ADD_SUFFIX(u32_to_bytevec)(u32 a)
+{
+ return vreinterpretq_u8_u32(vsetq_lane_u32(a, vdupq_n_u32(0), 0));
+}
+#define u32_to_bytevec ADD_SUFFIX(u32_to_bytevec)
+
+/* Load two 64-bit values into a vector. */
+#undef load_multipliers
+static forceinline ATTRIBUTES poly64x2_t
+ADD_SUFFIX(load_multipliers)(const u64 p[2])
+{
+ return vreinterpretq_p64_u64(vld1q_u64(p));
+}
+#define load_multipliers ADD_SUFFIX(load_multipliers)
+
+/* Do carryless multiplication of the low halves of two vectors. */
+#undef clmul_low
+static forceinline ATTRIBUTES uint8x16_t
+ADD_SUFFIX(clmul_low)(uint8x16_t a, poly64x2_t b)
+{
+ return vreinterpretq_u8_p128(
+ compat_vmull_p64(vgetq_lane_p64(vreinterpretq_p64_u8(a), 0),
+ vgetq_lane_p64(b, 0)));
+}
+#define clmul_low ADD_SUFFIX(clmul_low)
+
+/* Do carryless multiplication of the high halves of two vectors. */
+#undef clmul_high
+static forceinline ATTRIBUTES uint8x16_t
+ADD_SUFFIX(clmul_high)(uint8x16_t a, poly64x2_t b)
+{
+#ifdef __clang__
+ /*
+ * Use inline asm to ensure that pmull2 is really used. This works
+ * around clang bug https://github.com/llvm/llvm-project/issues/52868.
+ */
+ uint8x16_t res;
+
+ __asm__("pmull2 %0.1q, %1.2d, %2.2d" : "=w" (res) : "w" (a), "w" (b));
+ return res;
+#else
+ return vreinterpretq_u8_p128(vmull_high_p64(vreinterpretq_p64_u8(a), b));
+#endif
+}
+#define clmul_high ADD_SUFFIX(clmul_high)
+
+#undef eor3
+static forceinline ATTRIBUTES uint8x16_t
+ADD_SUFFIX(eor3)(uint8x16_t a, uint8x16_t b, uint8x16_t c)
+{
+#if ENABLE_EOR3
+ return veor3q_u8(a, b, c);
+#else
+ return veorq_u8(veorq_u8(a, b), c);
+#endif
+}
+#define eor3 ADD_SUFFIX(eor3)
+
+#undef fold_vec
+static forceinline ATTRIBUTES uint8x16_t
+ADD_SUFFIX(fold_vec)(uint8x16_t src, uint8x16_t dst, poly64x2_t multipliers)
+{
+ uint8x16_t a = clmul_low(src, multipliers);
+ uint8x16_t b = clmul_high(src, multipliers);
+
+ return eor3(a, b, dst);
+}
+#define fold_vec ADD_SUFFIX(fold_vec)
+
+/*
+ * Given v containing a 16-byte polynomial, and a pointer 'p' that points to the
+ * next '1 <= len <= 15' data bytes, rearrange the concatenation of v and the
+ * data into vectors x0 and x1 that contain 'len' bytes and 16 bytes,
+ * respectively. Then fold x0 into x1 and return the result. Assumes that
+ * 'p + len - 16' is in-bounds.
+ */
+#undef fold_partial_vec
+static forceinline ATTRIBUTES MAYBE_UNUSED uint8x16_t
+ADD_SUFFIX(fold_partial_vec)(uint8x16_t v, const u8 *p, size_t len,
+ poly64x2_t multipliers_1)
+{
+ /*
+ * vqtbl1q_u8(v, shift_tab[len..len+15]) left shifts v by 16-len bytes.
+ * vqtbl1q_u8(v, shift_tab[len+16..len+31]) right shifts v by len bytes.
+ */
+ static const u8 shift_tab[48] = {
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ };
+ const uint8x16_t lshift = vld1q_u8(&shift_tab[len]);
+ const uint8x16_t rshift = vld1q_u8(&shift_tab[len + 16]);
+ uint8x16_t x0, x1, bsl_mask;
+
+ /* x0 = v left-shifted by '16 - len' bytes */
+ x0 = vqtbl1q_u8(v, lshift);
+
+ /* Create a vector of '16 - len' 0x00 bytes, then 'len' 0xff bytes. */
+ bsl_mask = vreinterpretq_u8_s8(
+ vshrq_n_s8(vreinterpretq_s8_u8(rshift), 7));
+
+ /*
+ * x1 = the last '16 - len' bytes from v (i.e. v right-shifted by 'len'
+ * bytes) followed by the remaining data.
+ */
+ x1 = vbslq_u8(bsl_mask /* 0 bits select from arg3, 1 bits from arg2 */,
+ vld1q_u8(p + len - 16), vqtbl1q_u8(v, rshift));
+
+ return fold_vec(x0, x1, multipliers_1);
+}
+#define fold_partial_vec ADD_SUFFIX(fold_partial_vec)
diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/crc32_pmull_wide.h b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/crc32_pmull_wide.h
new file mode 100644
index 0000000..300e282
--- /dev/null
+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/crc32_pmull_wide.h
@@ -0,0 +1,226 @@
+/*
+ * arm/crc32_pmull_wide.h - gzip CRC-32 with PMULL (extra-wide version)
+ *
+ * Copyright 2022 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * This file is a "template" for instantiating PMULL-based crc32_arm functions.
+ * The "parameters" are:
+ *
+ * SUFFIX:
+ * Name suffix to append to all instantiated functions.
+ * ATTRIBUTES:
+ * Target function attributes to use.
+ * ENABLE_EOR3:
+ * Use the eor3 instruction (from the sha3 extension).
+ *
+ * This is the extra-wide version; it uses an unusually large stride length of
+ * 12, and it assumes that crc32 instructions are available too. It's intended
+ * for powerful CPUs that support both pmull and crc32 instructions, but where
+ * throughput of pmull and xor (given enough instructions issued in parallel) is
+ * significantly higher than that of crc32, thus making the crc32 instructions
+ * (counterintuitively) not actually the fastest way to compute the CRC-32. The
+ * Apple M1 processor is an example of such a CPU.
+ */
+
+#include "crc32_pmull_helpers.h"
+
+static ATTRIBUTES u32
+ADD_SUFFIX(crc32_arm)(u32 crc, const u8 *p, size_t len)
+{
+ uint8x16_t v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11;
+
+ if (len < 3 * 192) {
+ static const u64 _aligned_attribute(16) mults[3][2] = {
+ { CRC32_X543_MODG, CRC32_X479_MODG }, /* 4 vecs */
+ { CRC32_X287_MODG, CRC32_X223_MODG }, /* 2 vecs */
+ { CRC32_X159_MODG, CRC32_X95_MODG }, /* 1 vecs */
+ };
+ poly64x2_t multipliers_4, multipliers_2, multipliers_1;
+
+ if (len < 64)
+ goto tail;
+ multipliers_4 = load_multipliers(mults[0]);
+ multipliers_2 = load_multipliers(mults[1]);
+ multipliers_1 = load_multipliers(mults[2]);
+ /*
+ * Short length; don't bother aligning the pointer, and fold
+ * 64 bytes (4 vectors) at a time, at most.
+ */
+ v0 = veorq_u8(vld1q_u8(p + 0), u32_to_bytevec(crc));
+ v1 = vld1q_u8(p + 16);
+ v2 = vld1q_u8(p + 32);
+ v3 = vld1q_u8(p + 48);
+ p += 64;
+ len -= 64;
+ while (len >= 64) {
+ v0 = fold_vec(v0, vld1q_u8(p + 0), multipliers_4);
+ v1 = fold_vec(v1, vld1q_u8(p + 16), multipliers_4);
+ v2 = fold_vec(v2, vld1q_u8(p + 32), multipliers_4);
+ v3 = fold_vec(v3, vld1q_u8(p + 48), multipliers_4);
+ p += 64;
+ len -= 64;
+ }
+ v0 = fold_vec(v0, v2, multipliers_2);
+ v1 = fold_vec(v1, v3, multipliers_2);
+ if (len >= 32) {
+ v0 = fold_vec(v0, vld1q_u8(p + 0), multipliers_2);
+ v1 = fold_vec(v1, vld1q_u8(p + 16), multipliers_2);
+ p += 32;
+ len -= 32;
+ }
+ v0 = fold_vec(v0, v1, multipliers_1);
+ } else {
+ static const u64 _aligned_attribute(16) mults[4][2] = {
+ { CRC32_X1567_MODG, CRC32_X1503_MODG }, /* 12 vecs */
+ { CRC32_X799_MODG, CRC32_X735_MODG }, /* 6 vecs */
+ { CRC32_X415_MODG, CRC32_X351_MODG }, /* 3 vecs */
+ { CRC32_X159_MODG, CRC32_X95_MODG }, /* 1 vecs */
+ };
+ const poly64x2_t multipliers_12 = load_multipliers(mults[0]);
+ const poly64x2_t multipliers_6 = load_multipliers(mults[1]);
+ const poly64x2_t multipliers_3 = load_multipliers(mults[2]);
+ const poly64x2_t multipliers_1 = load_multipliers(mults[3]);
+ const size_t align = -(uintptr_t)p & 15;
+ const uint8x16_t *vp;
+
+ /* Align p to the next 16-byte boundary. */
+ if (align) {
+ if (align & 1)
+ crc = __crc32b(crc, *p++);
+ if (align & 2) {
+ crc = __crc32h(crc, le16_bswap(*(u16 *)p));
+ p += 2;
+ }
+ if (align & 4) {
+ crc = __crc32w(crc, le32_bswap(*(u32 *)p));
+ p += 4;
+ }
+ if (align & 8) {
+ crc = __crc32d(crc, le64_bswap(*(u64 *)p));
+ p += 8;
+ }
+ len -= align;
+ }
+ vp = (const uint8x16_t *)p;
+ v0 = veorq_u8(*vp++, u32_to_bytevec(crc));
+ v1 = *vp++;
+ v2 = *vp++;
+ v3 = *vp++;
+ v4 = *vp++;
+ v5 = *vp++;
+ v6 = *vp++;
+ v7 = *vp++;
+ v8 = *vp++;
+ v9 = *vp++;
+ v10 = *vp++;
+ v11 = *vp++;
+ len -= 192;
+ /* Fold 192 bytes (12 vectors) at a time. */
+ do {
+ v0 = fold_vec(v0, *vp++, multipliers_12);
+ v1 = fold_vec(v1, *vp++, multipliers_12);
+ v2 = fold_vec(v2, *vp++, multipliers_12);
+ v3 = fold_vec(v3, *vp++, multipliers_12);
+ v4 = fold_vec(v4, *vp++, multipliers_12);
+ v5 = fold_vec(v5, *vp++, multipliers_12);
+ v6 = fold_vec(v6, *vp++, multipliers_12);
+ v7 = fold_vec(v7, *vp++, multipliers_12);
+ v8 = fold_vec(v8, *vp++, multipliers_12);
+ v9 = fold_vec(v9, *vp++, multipliers_12);
+ v10 = fold_vec(v10, *vp++, multipliers_12);
+ v11 = fold_vec(v11, *vp++, multipliers_12);
+ len -= 192;
+ } while (len >= 192);
+
+ /*
+ * Fewer than 192 bytes left. Fold v0-v11 down to just v0,
+ * while processing up to 144 more bytes.
+ */
+ v0 = fold_vec(v0, v6, multipliers_6);
+ v1 = fold_vec(v1, v7, multipliers_6);
+ v2 = fold_vec(v2, v8, multipliers_6);
+ v3 = fold_vec(v3, v9, multipliers_6);
+ v4 = fold_vec(v4, v10, multipliers_6);
+ v5 = fold_vec(v5, v11, multipliers_6);
+ if (len >= 96) {
+ v0 = fold_vec(v0, *vp++, multipliers_6);
+ v1 = fold_vec(v1, *vp++, multipliers_6);
+ v2 = fold_vec(v2, *vp++, multipliers_6);
+ v3 = fold_vec(v3, *vp++, multipliers_6);
+ v4 = fold_vec(v4, *vp++, multipliers_6);
+ v5 = fold_vec(v5, *vp++, multipliers_6);
+ len -= 96;
+ }
+ v0 = fold_vec(v0, v3, multipliers_3);
+ v1 = fold_vec(v1, v4, multipliers_3);
+ v2 = fold_vec(v2, v5, multipliers_3);
+ if (len >= 48) {
+ v0 = fold_vec(v0, *vp++, multipliers_3);
+ v1 = fold_vec(v1, *vp++, multipliers_3);
+ v2 = fold_vec(v2, *vp++, multipliers_3);
+ len -= 48;
+ }
+ v0 = fold_vec(v0, v1, multipliers_1);
+ v0 = fold_vec(v0, v2, multipliers_1);
+ p = (const u8 *)vp;
+ }
+ /* Reduce 128 to 32 bits using crc32 instructions. */
+ crc = __crc32d(0, vgetq_lane_u64(vreinterpretq_u64_u8(v0), 0));
+ crc = __crc32d(crc, vgetq_lane_u64(vreinterpretq_u64_u8(v0), 1));
+tail:
+ /* Finish up the remainder using crc32 instructions. */
+ if (len & 32) {
+ crc = __crc32d(crc, get_unaligned_le64(p + 0));
+ crc = __crc32d(crc, get_unaligned_le64(p + 8));
+ crc = __crc32d(crc, get_unaligned_le64(p + 16));
+ crc = __crc32d(crc, get_unaligned_le64(p + 24));
+ p += 32;
+ }
+ if (len & 16) {
+ crc = __crc32d(crc, get_unaligned_le64(p + 0));
+ crc = __crc32d(crc, get_unaligned_le64(p + 8));
+ p += 16;
+ }
+ if (len & 8) {
+ crc = __crc32d(crc, get_unaligned_le64(p));
+ p += 8;
+ }
+ if (len & 4) {
+ crc = __crc32w(crc, get_unaligned_le32(p));
+ p += 4;
+ }
+ if (len & 2) {
+ crc = __crc32h(crc, get_unaligned_le16(p));
+ p += 2;
+ }
+ if (len & 1)
+ crc = __crc32b(crc, *p);
+ return crc;
+}
+
+#undef SUFFIX
+#undef ATTRIBUTES
+#undef ENABLE_EOR3
diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/matchfinder_impl.h b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/matchfinder_impl.h
new file mode 100644
index 0000000..79c1dbc
--- /dev/null
+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/arm/matchfinder_impl.h
@@ -0,0 +1,78 @@
+/*
+ * arm/matchfinder_impl.h - ARM implementations of matchfinder functions
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_ARM_MATCHFINDER_IMPL_H
+#define LIB_ARM_MATCHFINDER_IMPL_H
+
+#include "cpu_features.h"
+
+#if HAVE_NEON_NATIVE
+static forceinline void
+matchfinder_init_neon(mf_pos_t *data, size_t size)
+{
+ int16x8_t *p = (int16x8_t *)data;
+ int16x8_t v = vdupq_n_s16(MATCHFINDER_INITVAL);
+
+ STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+ STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
+ STATIC_ASSERT(sizeof(mf_pos_t) == 2);
+
+ do {
+ p[0] = v;
+ p[1] = v;
+ p[2] = v;
+ p[3] = v;
+ p += 4;
+ size -= 4 * sizeof(*p);
+ } while (size != 0);
+}
+#define matchfinder_init matchfinder_init_neon
+
+static forceinline void
+matchfinder_rebase_neon(mf_pos_t *data, size_t size)
+{
+ int16x8_t *p = (int16x8_t *)data;
+ int16x8_t v = vdupq_n_s16((u16)-MATCHFINDER_WINDOW_SIZE);
+
+ STATIC_ASSERT(MATCHFINDER_MEM_ALIGNMENT % sizeof(*p) == 0);
+ STATIC_ASSERT(MATCHFINDER_SIZE_ALIGNMENT % (4 * sizeof(*p)) == 0);
+ STATIC_ASSERT(sizeof(mf_pos_t) == 2);
+
+ do {
+ p[0] = vqaddq_s16(p[0], v);
+ p[1] = vqaddq_s16(p[1], v);
+ p[2] = vqaddq_s16(p[2], v);
+ p[3] = vqaddq_s16(p[3], v);
+ p += 4;
+ size -= 4 * sizeof(*p);
+ } while (size != 0);
+}
+#define matchfinder_rebase matchfinder_rebase_neon
+
+#endif /* HAVE_NEON_NATIVE */
+
+#endif /* LIB_ARM_MATCHFINDER_IMPL_H */
diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/cpu_features_common.h b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/cpu_features_common.h
new file mode 100644
index 0000000..d0531d5
--- /dev/null
+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/cpu_features_common.h
@@ -0,0 +1,93 @@
+/*
+ * cpu_features_common.h - code shared by all lib/$arch/cpu_features.c
+ *
+ * Copyright 2020 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef LIB_CPU_FEATURES_COMMON_H
+#define LIB_CPU_FEATURES_COMMON_H
+
+#if defined(TEST_SUPPORT__DO_NOT_USE) && !defined(FREESTANDING)
+ /* for strdup() and strtok_r() */
+# undef _ANSI_SOURCE
+# ifndef __APPLE__
+# undef _GNU_SOURCE
+# define _GNU_SOURCE
+# endif
+# include <stdio.h>
+# include <stdlib.h>
+# include <string.h>
+#endif
+
+#include "lib_common.h"
+
+struct cpu_feature {
+ u32 bit;
+ const char *name;
+};
+
+#if defined(TEST_SUPPORT__DO_NOT_USE) && !defined(FREESTANDING)
+/* Disable any features that are listed in $LIBDEFLATE_DISABLE_CPU_FEATURES. */
+static inline void
+disable_cpu_features_for_testing(u32 *features,
+ const struct cpu_feature *feature_table,
+ size_t feature_table_length)
+{
+ char *env_value, *strbuf, *p, *saveptr = NULL;
+ size_t i;
+
+ env_value = getenv("LIBDEFLATE_DISABLE_CPU_FEATURES");
+ if (!env_value)
+ return;
+ strbuf = strdup(env_value);
+ if (!strbuf)
+ abort();
+ p = strtok_r(strbuf, ",", &saveptr);
+ while (p) {
+ for (i = 0; i < feature_table_length; i++) {
+ if (strcmp(p, feature_table[i].name) == 0) {
+ *features &= ~feature_table[i].bit;
+ break;
+ }
+ }
+ if (i == feature_table_length) {
+ fprintf(stderr,
+ "unrecognized feature in LIBDEFLATE_DISABLE_CPU_FEATURES: \"%s\"\n",
+ p);
+ abort();
+ }
+ p = strtok_r(NULL, ",", &saveptr);
+ }
+ free(strbuf);
+}
+#else /* TEST_SUPPORT__DO_NOT_USE */
+static inline void
+disable_cpu_features_for_testing(u32 *features,
+ const struct cpu_feature *feature_table,
+ size_t feature_table_length)
+{
+}
+#endif /* !TEST_SUPPORT__DO_NOT_USE */
+
+#endif /* LIB_CPU_FEATURES_COMMON_H */
diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/decompress_template.h b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/decompress_template.h
new file mode 100644
index 0000000..8c874c3
--- /dev/null
+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/decompress_template.h
@@ -0,0 +1,777 @@
+/*
+ * decompress_template.h
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * This is the actual DEFLATE decompression routine, lifted out of
+ * deflate_decompress.c so that it can be compiled multiple times with different
+ * target instruction sets.
+ */
+
+#ifndef ATTRIBUTES
+# define ATTRIBUTES
+#endif
+#ifndef EXTRACT_VARBITS
+# define EXTRACT_VARBITS(word, count) ((word) & BITMASK(count))
+#endif
+#ifndef EXTRACT_VARBITS8
+# define EXTRACT_VARBITS8(word, count) ((word) & BITMASK((u8)(count)))
+#endif
+
+static ATTRIBUTES MAYBE_UNUSED enum libdeflate_result
+FUNCNAME(struct libdeflate_decompressor * restrict d,
+ const void * restrict in, size_t in_nbytes,
+ void * restrict out, size_t out_nbytes_avail,
+ size_t *actual_in_nbytes_ret, size_t *actual_out_nbytes_ret)
+{
+ u8 *out_next = out;
+ u8 * const out_end = out_next + out_nbytes_avail;
+ u8 * const out_fastloop_end =
+ out_end - MIN(out_nbytes_avail, FASTLOOP_MAX_BYTES_WRITTEN);
+
+ /* Input bitstream state; see deflate_decompress.c for documentation */
+ const u8 *in_next = in;
+ const u8 * const in_end = in_next + in_nbytes;
+ const u8 * const in_fastloop_end =
+ in_end - MIN(in_nbytes, FASTLOOP_MAX_BYTES_READ);
+ bitbuf_t bitbuf = 0;
+ bitbuf_t saved_bitbuf;
+ u32 bitsleft = 0;
+ size_t overread_count = 0;
+
+ bool is_final_block;
+ unsigned block_type;
+ unsigned num_litlen_syms;
+ unsigned num_offset_syms;
+ bitbuf_t litlen_tablemask;
+ u32 entry;
+
+next_block:
+ /* Starting to read the next block */
+ ;
+
+ STATIC_ASSERT(CAN_CONSUME(1 + 2 + 5 + 5 + 4 + 3));
+ REFILL_BITS();
+
+ /* BFINAL: 1 bit */
+ is_final_block = bitbuf & BITMASK(1);
+
+ /* BTYPE: 2 bits */
+ block_type = (bitbuf >> 1) & BITMASK(2);
+
+ if (block_type == DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN) {
+
+ /* Dynamic Huffman block */
+
+ /* The order in which precode lengths are stored */
+ static const u8 deflate_precode_lens_permutation[DEFLATE_NUM_PRECODE_SYMS] = {
+ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
+ };
+
+ unsigned num_explicit_precode_lens;
+ unsigned i;
+
+ /* Read the codeword length counts. */
+
+ STATIC_ASSERT(DEFLATE_NUM_LITLEN_SYMS == 257 + BITMASK(5));
+ num_litlen_syms = 257 + ((bitbuf >> 3) & BITMASK(5));
+
+ STATIC_ASSERT(DEFLATE_NUM_OFFSET_SYMS == 1 + BITMASK(5));
+ num_offset_syms = 1 + ((bitbuf >> 8) & BITMASK(5));
+
+ STATIC_ASSERT(DEFLATE_NUM_PRECODE_SYMS == 4 + BITMASK(4));
+ num_explicit_precode_lens = 4 + ((bitbuf >> 13) & BITMASK(4));
+
+ d->static_codes_loaded = false;
+
+ /*
+ * Read the precode codeword lengths.
+ *
+ * A 64-bit bitbuffer is just one bit too small to hold the
+ * maximum number of precode lens, so to minimize branches we
+ * merge one len with the previous fields.
+ */
+ STATIC_ASSERT(DEFLATE_MAX_PRE_CODEWORD_LEN == (1 << 3) - 1);
+ if (CAN_CONSUME(3 * (DEFLATE_NUM_PRECODE_SYMS - 1))) {
+ d->u.precode_lens[deflate_precode_lens_permutation[0]] =
+ (bitbuf >> 17) & BITMASK(3);
+ bitbuf >>= 20;
+ bitsleft -= 20;
+ REFILL_BITS();
+ i = 1;
+ do {
+ d->u.precode_lens[deflate_precode_lens_permutation[i]] =
+ bitbuf & BITMASK(3);
+ bitbuf >>= 3;
+ bitsleft -= 3;
+ } while (++i < num_explicit_precode_lens);
+ } else {
+ bitbuf >>= 17;
+ bitsleft -= 17;
+ i = 0;
+ do {
+ if ((u8)bitsleft < 3)
+ REFILL_BITS();
+ d->u.precode_lens[deflate_precode_lens_permutation[i]] =
+ bitbuf & BITMASK(3);
+ bitbuf >>= 3;
+ bitsleft -= 3;
+ } while (++i < num_explicit_precode_lens);
+ }
+ for (; i < DEFLATE_NUM_PRECODE_SYMS; i++)
+ d->u.precode_lens[deflate_precode_lens_permutation[i]] = 0;
+
+ /* Build the decode table for the precode. */
+ SAFETY_CHECK(build_precode_decode_table(d));
+
+ /* Decode the litlen and offset codeword lengths. */
+ i = 0;
+ do {
+ unsigned presym;
+ u8 rep_val;
+ unsigned rep_count;
+
+ if ((u8)bitsleft < DEFLATE_MAX_PRE_CODEWORD_LEN + 7)
+ REFILL_BITS();
+
+ /*
+ * The code below assumes that the precode decode table
+ * doesn't have any subtables.
+ */
+ STATIC_ASSERT(PRECODE_TABLEBITS == DEFLATE_MAX_PRE_CODEWORD_LEN);
+
+ /* Decode the next precode symbol. */
+ entry = d->u.l.precode_decode_table[
+ bitbuf & BITMASK(DEFLATE_MAX_PRE_CODEWORD_LEN)];
+ bitbuf >>= (u8)entry;
+ bitsleft -= entry; /* optimization: subtract full entry */
+ presym = entry >> 16;
+
+ if (presym < 16) {
+ /* Explicit codeword length */
+ d->u.l.lens[i++] = presym;
+ continue;
+ }
+
+ /* Run-length encoded codeword lengths */
+
+ /*
+ * Note: we don't need to immediately verify that the
+ * repeat count doesn't overflow the number of elements,
+ * since we've sized the lens array to have enough extra
+ * space to allow for the worst-case overrun (138 zeroes
+ * when only 1 length was remaining).
+ *
+ * In the case of the small repeat counts (presyms 16
+ * and 17), it is fastest to always write the maximum
+ * number of entries. That gets rid of branches that
+ * would otherwise be required.
+ *
+ * It is not just because of the numerical order that
+ * our checks go in the order 'presym < 16', 'presym ==
+ * 16', and 'presym == 17'. For typical data this is
+ * ordered from most frequent to least frequent case.
+ */
+ STATIC_ASSERT(DEFLATE_MAX_LENS_OVERRUN == 138 - 1);
+
+ if (presym == 16) {
+ /* Repeat the previous length 3 - 6 times. */
+ SAFETY_CHECK(i != 0);
+ rep_val = d->u.l.lens[i - 1];
+ STATIC_ASSERT(3 + BITMASK(2) == 6);
+ rep_count = 3 + (bitbuf & BITMASK(2));
+ bitbuf >>= 2;
+ bitsleft -= 2;
+ d->u.l.lens[i + 0] = rep_val;
+ d->u.l.lens[i + 1] = rep_val;
+ d->u.l.lens[i + 2] = rep_val;
+ d->u.l.lens[i + 3] = rep_val;
+ d->u.l.lens[i + 4] = rep_val;
+ d->u.l.lens[i + 5] = rep_val;
+ i += rep_count;
+ } else if (presym == 17) {
+ /* Repeat zero 3 - 10 times. */
+ STATIC_ASSERT(3 + BITMASK(3) == 10);
+ rep_count = 3 + (bitbuf & BITMASK(3));
+ bitbuf >>= 3;
+ bitsleft -= 3;
+ d->u.l.lens[i + 0] = 0;
+ d->u.l.lens[i + 1] = 0;
+ d->u.l.lens[i + 2] = 0;
+ d->u.l.lens[i + 3] = 0;
+ d->u.l.lens[i + 4] = 0;
+ d->u.l.lens[i + 5] = 0;
+ d->u.l.lens[i + 6] = 0;
+ d->u.l.lens[i + 7] = 0;
+ d->u.l.lens[i + 8] = 0;
+ d->u.l.lens[i + 9] = 0;
+ i += rep_count;
+ } else {
+ /* Repeat zero 11 - 138 times. */
+ STATIC_ASSERT(11 + BITMASK(7) == 138);
+ rep_count = 11 + (bitbuf & BITMASK(7));
+ bitbuf >>= 7;
+ bitsleft -= 7;
+ memset(&d->u.l.lens[i], 0,
+ rep_count * sizeof(d->u.l.lens[i]));
+ i += rep_count;
+ }
+ } while (i < num_litlen_syms + num_offset_syms);
+
+ /* Unnecessary, but check this for consistency with zlib. */
+ SAFETY_CHECK(i == num_litlen_syms + num_offset_syms);
+
+ } else if (block_type == DEFLATE_BLOCKTYPE_UNCOMPRESSED) {
+ u16 len, nlen;
+
+ /*
+ * Uncompressed block: copy 'len' bytes literally from the input
+ * buffer to the output buffer.
+ */
+
+ bitsleft -= 3; /* for BTYPE and BFINAL */
+
+ /*
+ * Align the bitstream to the next byte boundary. This means
+ * the next byte boundary as if we were reading a byte at a
+ * time. Therefore, we have to rewind 'in_next' by any bytes
+ * that have been refilled but not actually consumed yet (not
+ * counting overread bytes, which don't increment 'in_next').
+ */
+ bitsleft = (u8)bitsleft;
+ SAFETY_CHECK(overread_count <= (bitsleft >> 3));
+ in_next -= (bitsleft >> 3) - overread_count;
+ overread_count = 0;
+ bitbuf = 0;
+ bitsleft = 0;
+
+ SAFETY_CHECK(in_end - in_next >= 4);
+ len = get_unaligned_le16(in_next);
+ nlen = get_unaligned_le16(in_next + 2);
+ in_next += 4;
+
+ SAFETY_CHECK(len == (u16)~nlen);
+ if (unlikely(len > out_end - out_next))
+ return LIBDEFLATE_INSUFFICIENT_SPACE;
+ SAFETY_CHECK(len <= in_end - in_next);
+
+ memcpy(out_next, in_next, len);
+ in_next += len;
+ out_next += len;
+
+ goto block_done;
+
+ } else {
+ unsigned i;
+
+ SAFETY_CHECK(block_type == DEFLATE_BLOCKTYPE_STATIC_HUFFMAN);
+
+ /*
+ * Static Huffman block: build the decode tables for the static
+ * codes. Skip doing so if the tables are already set up from
+ * an earlier static block; this speeds up decompression of
+ * degenerate input of many empty or very short static blocks.
+ *
+ * Afterwards, the remainder is the same as decompressing a
+ * dynamic Huffman block.
+ */
+
+ bitbuf >>= 3; /* for BTYPE and BFINAL */
+ bitsleft -= 3;
+
+ if (d->static_codes_loaded)
+ goto have_decode_tables;
+
+ d->static_codes_loaded = true;
+
+ STATIC_ASSERT(DEFLATE_NUM_LITLEN_SYMS == 288);
+ STATIC_ASSERT(DEFLATE_NUM_OFFSET_SYMS == 32);
+
+ for (i = 0; i < 144; i++)
+ d->u.l.lens[i] = 8;
+ for (; i < 256; i++)
+ d->u.l.lens[i] = 9;
+ for (; i < 280; i++)
+ d->u.l.lens[i] = 7;
+ for (; i < 288; i++)
+ d->u.l.lens[i] = 8;
+
+ for (; i < 288 + 32; i++)
+ d->u.l.lens[i] = 5;
+
+ num_litlen_syms = 288;
+ num_offset_syms = 32;
+ }
+
+ /* Decompressing a Huffman block (either dynamic or static) */
+
+ SAFETY_CHECK(build_offset_decode_table(d, num_litlen_syms, num_offset_syms));
+ SAFETY_CHECK(build_litlen_decode_table(d, num_litlen_syms, num_offset_syms));
+have_decode_tables:
+ litlen_tablemask = BITMASK(d->litlen_tablebits);
+
+ /*
+ * This is the "fastloop" for decoding literals and matches. It does
+ * bounds checks on in_next and out_next in the loop conditions so that
+ * additional bounds checks aren't needed inside the loop body.
+ *
+ * To reduce latency, the bitbuffer is refilled and the next litlen
+ * decode table entry is preloaded before each loop iteration.
+ */
+ if (in_next >= in_fastloop_end || out_next >= out_fastloop_end)
+ goto generic_loop;
+ REFILL_BITS_IN_FASTLOOP();
+ entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
+ do {
+ u32 length, offset, lit;
+ const u8 *src;
+ u8 *dst;
+
+ /*
+ * Consume the bits for the litlen decode table entry. Save the
+ * original bitbuf for later, in case the extra match length
+ * bits need to be extracted from it.
+ */
+ saved_bitbuf = bitbuf;
+ bitbuf >>= (u8)entry;
+ bitsleft -= entry; /* optimization: subtract full entry */
+
+ /*
+ * Begin by checking for a "fast" literal, i.e. a literal that
+ * doesn't need a subtable.
+ */
+ if (entry & HUFFDEC_LITERAL) {
+ /*
+ * On 64-bit platforms, we decode up to 2 extra fast
+ * literals in addition to the primary item, as this
+ * increases performance and still leaves enough bits
+ * remaining for what follows. We could actually do 3,
+ * assuming LITLEN_TABLEBITS=11, but that actually
+ * decreases performance slightly (perhaps by messing
+ * with the branch prediction of the conditional refill
+ * that happens later while decoding the match offset).
+ *
+ * Note: the definitions of FASTLOOP_MAX_BYTES_WRITTEN
+ * and FASTLOOP_MAX_BYTES_READ need to be updated if the
+ * number of extra literals decoded here is changed.
+ */
+ if (/* enough bits for 2 fast literals + length + offset preload? */
+ CAN_CONSUME_AND_THEN_PRELOAD(2 * LITLEN_TABLEBITS +
+ LENGTH_MAXBITS,
+ OFFSET_TABLEBITS) &&
+ /* enough bits for 2 fast literals + slow literal + litlen preload? */
+ CAN_CONSUME_AND_THEN_PRELOAD(2 * LITLEN_TABLEBITS +
+ DEFLATE_MAX_LITLEN_CODEWORD_LEN,
+ LITLEN_TABLEBITS)) {
+ /* 1st extra fast literal */
+ lit = entry >> 16;
+ entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
+ saved_bitbuf = bitbuf;
+ bitbuf >>= (u8)entry;
+ bitsleft -= entry;
+ *out_next++ = lit;
+ if (entry & HUFFDEC_LITERAL) {
+ /* 2nd extra fast literal */
+ lit = entry >> 16;
+ entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
+ saved_bitbuf = bitbuf;
+ bitbuf >>= (u8)entry;
+ bitsleft -= entry;
+ *out_next++ = lit;
+ if (entry & HUFFDEC_LITERAL) {
+ /*
+ * Another fast literal, but
+ * this one is in lieu of the
+ * primary item, so it doesn't
+ * count as one of the extras.
+ */
+ lit = entry >> 16;
+ entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
+ REFILL_BITS_IN_FASTLOOP();
+ *out_next++ = lit;
+ continue;
+ }
+ }
+ } else {
+ /*
+ * Decode a literal. While doing so, preload
+ * the next litlen decode table entry and refill
+ * the bitbuffer. To reduce latency, we've
+ * arranged for there to be enough "preloadable"
+ * bits remaining to do the table preload
+ * independently of the refill.
+ */
+ STATIC_ASSERT(CAN_CONSUME_AND_THEN_PRELOAD(
+ LITLEN_TABLEBITS, LITLEN_TABLEBITS));
+ lit = entry >> 16;
+ entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
+ REFILL_BITS_IN_FASTLOOP();
+ *out_next++ = lit;
+ continue;
+ }
+ }
+
+ /*
+ * It's not a literal entry, so it can be a length entry, a
+ * subtable pointer entry, or an end-of-block entry. Detect the
+ * two unlikely cases by testing the HUFFDEC_EXCEPTIONAL flag.
+ */
+ if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
+ /* Subtable pointer or end-of-block entry */
+
+ if (unlikely(entry & HUFFDEC_END_OF_BLOCK))
+ goto block_done;
+
+ /*
+ * A subtable is required. Load and consume the
+ * subtable entry. The subtable entry can be of any
+ * type: literal, length, or end-of-block.
+ */
+ entry = d->u.litlen_decode_table[(entry >> 16) +
+ EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
+ saved_bitbuf = bitbuf;
+ bitbuf >>= (u8)entry;
+ bitsleft -= entry;
+
+ /*
+ * 32-bit platforms that use the byte-at-a-time refill
+ * method have to do a refill here for there to always
+ * be enough bits to decode a literal that requires a
+ * subtable, then preload the next litlen decode table
+ * entry; or to decode a match length that requires a
+ * subtable, then preload the offset decode table entry.
+ */
+ if (!CAN_CONSUME_AND_THEN_PRELOAD(DEFLATE_MAX_LITLEN_CODEWORD_LEN,
+ LITLEN_TABLEBITS) ||
+ !CAN_CONSUME_AND_THEN_PRELOAD(LENGTH_MAXBITS,
+ OFFSET_TABLEBITS))
+ REFILL_BITS_IN_FASTLOOP();
+ if (entry & HUFFDEC_LITERAL) {
+ /* Decode a literal that required a subtable. */
+ lit = entry >> 16;
+ entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
+ REFILL_BITS_IN_FASTLOOP();
+ *out_next++ = lit;
+ continue;
+ }
+ if (unlikely(entry & HUFFDEC_END_OF_BLOCK))
+ goto block_done;
+ /* Else, it's a length that required a subtable. */
+ }
+
+ /*
+ * Decode the match length: the length base value associated
+ * with the litlen symbol (which we extract from the decode
+ * table entry), plus the extra length bits. We don't need to
+ * consume the extra length bits here, as they were included in
+ * the bits consumed by the entry earlier. We also don't need
+ * to check for too-long matches here, as this is inside the
+ * fastloop where it's already been verified that the output
+ * buffer has enough space remaining to copy a max-length match.
+ */
+ length = entry >> 16;
+ length += EXTRACT_VARBITS8(saved_bitbuf, entry) >> (u8)(entry >> 8);
+
+ /*
+ * Decode the match offset. There are enough "preloadable" bits
+ * remaining to preload the offset decode table entry, but a
+ * refill might be needed before consuming it.
+ */
+ STATIC_ASSERT(CAN_CONSUME_AND_THEN_PRELOAD(LENGTH_MAXFASTBITS,
+ OFFSET_TABLEBITS));
+ entry = d->offset_decode_table[bitbuf & BITMASK(OFFSET_TABLEBITS)];
+ if (CAN_CONSUME_AND_THEN_PRELOAD(OFFSET_MAXBITS,
+ LITLEN_TABLEBITS)) {
+ /*
+ * Decoding a match offset on a 64-bit platform. We may
+ * need to refill once, but then we can decode the whole
+ * offset and preload the next litlen table entry.
+ */
+ if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
+ /* Offset codeword requires a subtable */
+ if (unlikely((u8)bitsleft < OFFSET_MAXBITS +
+ LITLEN_TABLEBITS - PRELOAD_SLACK))
+ REFILL_BITS_IN_FASTLOOP();
+ bitbuf >>= OFFSET_TABLEBITS;
+ bitsleft -= OFFSET_TABLEBITS;
+ entry = d->offset_decode_table[(entry >> 16) +
+ EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
+ } else if (unlikely((u8)bitsleft < OFFSET_MAXFASTBITS +
+ LITLEN_TABLEBITS - PRELOAD_SLACK))
+ REFILL_BITS_IN_FASTLOOP();
+ } else {
+ /* Decoding a match offset on a 32-bit platform */
+ REFILL_BITS_IN_FASTLOOP();
+ if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
+ /* Offset codeword requires a subtable */
+ bitbuf >>= OFFSET_TABLEBITS;
+ bitsleft -= OFFSET_TABLEBITS;
+ entry = d->offset_decode_table[(entry >> 16) +
+ EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
+ REFILL_BITS_IN_FASTLOOP();
+ /* No further refill needed before extra bits */
+ STATIC_ASSERT(CAN_CONSUME(
+ OFFSET_MAXBITS - OFFSET_TABLEBITS));
+ } else {
+ /* No refill needed before extra bits */
+ STATIC_ASSERT(CAN_CONSUME(OFFSET_MAXFASTBITS));
+ }
+ }
+ saved_bitbuf = bitbuf;
+ bitbuf >>= (u8)entry;
+ bitsleft -= entry; /* optimization: subtract full entry */
+ offset = entry >> 16;
+ offset += EXTRACT_VARBITS8(saved_bitbuf, entry) >> (u8)(entry >> 8);
+
+ /* Validate the match offset; needed even in the fastloop. */
+ SAFETY_CHECK(offset <= out_next - (const u8 *)out);
+ src = out_next - offset;
+ dst = out_next;
+ out_next += length;
+
+ /*
+ * Before starting to issue the instructions to copy the match,
+ * refill the bitbuffer and preload the litlen decode table
+ * entry for the next loop iteration. This can increase
+ * performance by allowing the latency of the match copy to
+ * overlap with these other operations. To further reduce
+ * latency, we've arranged for there to be enough bits remaining
+ * to do the table preload independently of the refill, except
+ * on 32-bit platforms using the byte-at-a-time refill method.
+ */
+ if (!CAN_CONSUME_AND_THEN_PRELOAD(
+ MAX(OFFSET_MAXBITS - OFFSET_TABLEBITS,
+ OFFSET_MAXFASTBITS),
+ LITLEN_TABLEBITS) &&
+ unlikely((u8)bitsleft < LITLEN_TABLEBITS - PRELOAD_SLACK))
+ REFILL_BITS_IN_FASTLOOP();
+ entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
+ REFILL_BITS_IN_FASTLOOP();
+
+ /*
+ * Copy the match. On most CPUs the fastest method is a
+ * word-at-a-time copy, unconditionally copying about 5 words
+ * since this is enough for most matches without being too much.
+ *
+ * The normal word-at-a-time copy works for offset >= WORDBYTES,
+ * which is most cases. The case of offset == 1 is also common
+ * and is worth optimizing for, since it is just RLE encoding of
+ * the previous byte, which is the result of compressing long
+ * runs of the same byte.
+ *
+ * Writing past the match 'length' is allowed here, since it's
+ * been ensured there is enough output space left for a slight
+ * overrun. FASTLOOP_MAX_BYTES_WRITTEN needs to be updated if
+ * the maximum possible overrun here is changed.
+ */
+ if (UNALIGNED_ACCESS_IS_FAST && offset >= WORDBYTES) {
+ store_word_unaligned(load_word_unaligned(src), dst);
+ src += WORDBYTES;
+ dst += WORDBYTES;
+ store_word_unaligned(load_word_unaligned(src), dst);
+ src += WORDBYTES;
+ dst += WORDBYTES;
+ store_word_unaligned(load_word_unaligned(src), dst);
+ src += WORDBYTES;
+ dst += WORDBYTES;
+ store_word_unaligned(load_word_unaligned(src), dst);
+ src += WORDBYTES;
+ dst += WORDBYTES;
+ store_word_unaligned(load_word_unaligned(src), dst);
+ src += WORDBYTES;
+ dst += WORDBYTES;
+ while (dst < out_next) {
+ store_word_unaligned(load_word_unaligned(src), dst);
+ src += WORDBYTES;
+ dst += WORDBYTES;
+ store_word_unaligned(load_word_unaligned(src), dst);
+ src += WORDBYTES;
+ dst += WORDBYTES;
+ store_word_unaligned(load_word_unaligned(src), dst);
+ src += WORDBYTES;
+ dst += WORDBYTES;
+ store_word_unaligned(load_word_unaligned(src), dst);
+ src += WORDBYTES;
+ dst += WORDBYTES;
+ store_word_unaligned(load_word_unaligned(src), dst);
+ src += WORDBYTES;
+ dst += WORDBYTES;
+ }
+ } else if (UNALIGNED_ACCESS_IS_FAST && offset == 1) {
+ machine_word_t v;
+
+ /*
+ * This part tends to get auto-vectorized, so keep it
+ * copying a multiple of 16 bytes at a time.
+ */
+ v = (machine_word_t)0x0101010101010101 * src[0];
+ store_word_unaligned(v, dst);
+ dst += WORDBYTES;
+ store_word_unaligned(v, dst);
+ dst += WORDBYTES;
+ store_word_unaligned(v, dst);
+ dst += WORDBYTES;
+ store_word_unaligned(v, dst);
+ dst += WORDBYTES;
+ while (dst < out_next) {
+ store_word_unaligned(v, dst);
+ dst += WORDBYTES;
+ store_word_unaligned(v, dst);
+ dst += WORDBYTES;
+ store_word_unaligned(v, dst);
+ dst += WORDBYTES;
+ store_word_unaligned(v, dst);
+ dst += WORDBYTES;
+ }
+ } else if (UNALIGNED_ACCESS_IS_FAST) {
+ store_word_unaligned(load_word_unaligned(src), dst);
+ src += offset;
+ dst += offset;
+ store_word_unaligned(load_word_unaligned(src), dst);
+ src += offset;
+ dst += offset;
+ do {
+ store_word_unaligned(load_word_unaligned(src), dst);
+ src += offset;
+ dst += offset;
+ store_word_unaligned(load_word_unaligned(src), dst);
+ src += offset;
+ dst += offset;
+ } while (dst < out_next);
+ } else {
+ *dst++ = *src++;
+ *dst++ = *src++;
+ do {
+ *dst++ = *src++;
+ } while (dst < out_next);
+ }
+ } while (in_next < in_fastloop_end && out_next < out_fastloop_end);
+
+ /*
+ * This is the generic loop for decoding literals and matches. This
+ * handles cases where in_next and out_next are close to the end of
+ * their respective buffers. Usually this loop isn't performance-
+ * critical, as most time is spent in the fastloop above instead. We
+ * therefore omit some optimizations here in favor of smaller code.
+ */
+generic_loop:
+ for (;;) {
+ u32 length, offset;
+ const u8 *src;
+ u8 *dst;
+
+ REFILL_BITS();
+ entry = d->u.litlen_decode_table[bitbuf & litlen_tablemask];
+ saved_bitbuf = bitbuf;
+ bitbuf >>= (u8)entry;
+ bitsleft -= entry;
+ if (unlikely(entry & HUFFDEC_SUBTABLE_POINTER)) {
+ entry = d->u.litlen_decode_table[(entry >> 16) +
+ EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
+ saved_bitbuf = bitbuf;
+ bitbuf >>= (u8)entry;
+ bitsleft -= entry;
+ }
+ length = entry >> 16;
+ if (entry & HUFFDEC_LITERAL) {
+ if (unlikely(out_next == out_end))
+ return LIBDEFLATE_INSUFFICIENT_SPACE;
+ *out_next++ = length;
+ continue;
+ }
+ if (unlikely(entry & HUFFDEC_END_OF_BLOCK))
+ goto block_done;
+ length += EXTRACT_VARBITS8(saved_bitbuf, entry) >> (u8)(entry >> 8);
+ if (unlikely(length > out_end - out_next))
+ return LIBDEFLATE_INSUFFICIENT_SPACE;
+
+ if (!CAN_CONSUME(LENGTH_MAXBITS + OFFSET_MAXBITS))
+ REFILL_BITS();
+ entry = d->offset_decode_table[bitbuf & BITMASK(OFFSET_TABLEBITS)];
+ if (unlikely(entry & HUFFDEC_EXCEPTIONAL)) {
+ bitbuf >>= OFFSET_TABLEBITS;
+ bitsleft -= OFFSET_TABLEBITS;
+ entry = d->offset_decode_table[(entry >> 16) +
+ EXTRACT_VARBITS(bitbuf, (entry >> 8) & 0x3F)];
+ if (!CAN_CONSUME(OFFSET_MAXBITS))
+ REFILL_BITS();
+ }
+ offset = entry >> 16;
+ offset += EXTRACT_VARBITS8(bitbuf, entry) >> (u8)(entry >> 8);
+ bitbuf >>= (u8)entry;
+ bitsleft -= entry;
+
+ SAFETY_CHECK(offset <= out_next - (const u8 *)out);
+ src = out_next - offset;
+ dst = out_next;
+ out_next += length;
+
+ STATIC_ASSERT(DEFLATE_MIN_MATCH_LEN == 3);
+ *dst++ = *src++;
+ *dst++ = *src++;
+ do {
+ *dst++ = *src++;
+ } while (dst < out_next);
+ }
+
+block_done:
+ /* Finished decoding a block */
+
+ if (!is_final_block)
+ goto next_block;
+
+ /* That was the last block. */
+
+ bitsleft = (u8)bitsleft;
+
+ /*
+ * If any of the implicit appended zero bytes were consumed (not just
+ * refilled) before hitting end of stream, then the data is bad.
+ */
+ SAFETY_CHECK(overread_count <= (bitsleft >> 3));
+
+ /* Optionally return the actual number of bytes consumed. */
+ if (actual_in_nbytes_ret) {
+ /* Don't count bytes that were refilled but not consumed. */
+ in_next -= (bitsleft >> 3) - overread_count;
+
+ *actual_in_nbytes_ret = in_next - (u8 *)in;
+ }
+
+ /* Optionally return the actual number of bytes written. */
+ if (actual_out_nbytes_ret) {
+ *actual_out_nbytes_ret = out_next - (u8 *)out;
+ } else {
+ if (out_next != out_end)
+ return LIBDEFLATE_SHORT_OUTPUT;
+ }
+ return LIBDEFLATE_SUCCESS;
+}
+
+#undef FUNCNAME
+#undef ATTRIBUTES
+#undef EXTRACT_VARBITS
+#undef EXTRACT_VARBITS8
diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/deflate_constants.h b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/deflate_constants.h
new file mode 100644
index 0000000..95c9e0a
--- /dev/null
+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/deflate_constants.h
@@ -0,0 +1,56 @@
+/*
+ * deflate_constants.h - constants for the DEFLATE compression format
+ */
+
+#ifndef LIB_DEFLATE_CONSTANTS_H
+#define LIB_DEFLATE_CONSTANTS_H
+
+/* Valid block types */
+#define DEFLATE_BLOCKTYPE_UNCOMPRESSED 0
+#define DEFLATE_BLOCKTYPE_STATIC_HUFFMAN 1
+#define DEFLATE_BLOCKTYPE_DYNAMIC_HUFFMAN 2
+
+/* Minimum and maximum supported match lengths (in bytes) */
+#define DEFLATE_MIN_MATCH_LEN 3
+#define DEFLATE_MAX_MATCH_LEN 258
+
+/* Maximum supported match offset (in bytes) */
+#define DEFLATE_MAX_MATCH_OFFSET 32768
+
+/* log2 of DEFLATE_MAX_MATCH_OFFSET */
+#define DEFLATE_WINDOW_ORDER 15
+
+/* Number of symbols in each Huffman code. Note: for the literal/length
+ * and offset codes, these are actually the maximum values; a given block
+ * might use fewer symbols. */
+#define DEFLATE_NUM_PRECODE_SYMS 19
+#define DEFLATE_NUM_LITLEN_SYMS 288
+#define DEFLATE_NUM_OFFSET_SYMS 32
+
+/* The maximum number of symbols across all codes */
+#define DEFLATE_MAX_NUM_SYMS 288
+
+/* Division of symbols in the literal/length code */
+#define DEFLATE_NUM_LITERALS 256
+#define DEFLATE_END_OF_BLOCK 256
+#define DEFLATE_FIRST_LEN_SYM 257
+
+/* Maximum codeword length, in bits, within each Huffman code */
+#define DEFLATE_MAX_PRE_CODEWORD_LEN 7
+#define DEFLATE_MAX_LITLEN_CODEWORD_LEN 15
+#define DEFLATE_MAX_OFFSET_CODEWORD_LEN 15
+
+/* The maximum codeword length across all codes */
+#define DEFLATE_MAX_CODEWORD_LEN 15
+
+/* Maximum possible overrun when decoding codeword lengths */
+#define DEFLATE_MAX_LENS_OVERRUN 137
+
+/*
+ * Maximum number of extra bits that may be required to represent a match
+ * length or offset.
+ */
+#define DEFLATE_MAX_EXTRA_LENGTH_BITS 5
+#define DEFLATE_MAX_EXTRA_OFFSET_BITS 13
+
+#endif /* LIB_DEFLATE_CONSTANTS_H */
diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/deflate_decompress.c b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/deflate_decompress.c
new file mode 100644
index 0000000..63726c7
--- /dev/null
+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/deflate_decompress.c
@@ -0,0 +1,1208 @@
+/*
+ * deflate_decompress.c - a decompressor for DEFLATE
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * ---------------------------------------------------------------------------
+ *
+ * This is a highly optimized DEFLATE decompressor. It is much faster than
+ * vanilla zlib, typically well over twice as fast, though results vary by CPU.
+ *
+ * Why this is faster than vanilla zlib:
+ *
+ * - Word accesses rather than byte accesses when reading input
+ * - Word accesses rather than byte accesses when copying matches
+ * - Faster Huffman decoding combined with various DEFLATE-specific tricks
+ * - Larger bitbuffer variable that doesn't need to be refilled as often
+ * - Other optimizations to remove unnecessary branches
+ * - Only full-buffer decompression is supported, so the code doesn't need to
+ * support stopping and resuming decompression.
+ * - On x86_64, a version of the decompression routine is compiled with BMI2
+ * instructions enabled and is used automatically at runtime when supported.
+ */
+
+#include "lib_common.h"
+#include "deflate_constants.h"
+
+/*
+ * If the expression passed to SAFETY_CHECK() evaluates to false, then the
+ * decompression routine immediately returns LIBDEFLATE_BAD_DATA, indicating the
+ * compressed data is invalid.
+ *
+ * Theoretically, these checks could be disabled for specialized applications
+ * where all input to the decompressor will be trusted.
+ */
+#if 0
+# pragma message("UNSAFE DECOMPRESSION IS ENABLED. THIS MUST ONLY BE USED IF THE DECOMPRESSOR INPUT WILL ALWAYS BE TRUSTED!")
+# define SAFETY_CHECK(expr) (void)(expr)
+#else
+# define SAFETY_CHECK(expr) if (unlikely(!(expr))) return LIBDEFLATE_BAD_DATA
+#endif
+
+/*****************************************************************************
+ * Input bitstream *
+ *****************************************************************************/
+
+/*
+ * The state of the "input bitstream" consists of the following variables:
+ *
+ * - in_next: a pointer to the next unread byte in the input buffer
+ *
+ * - in_end: a pointer to just past the end of the input buffer
+ *
+ * - bitbuf: a word-sized variable containing bits that have been read from
+ * the input buffer or from the implicit appended zero bytes
+ *
+ * - bitsleft: the number of bits in 'bitbuf' available to be consumed.
+ * After REFILL_BITS_BRANCHLESS(), 'bitbuf' can actually
+ * contain more bits than this. However, only the bits counted
+ * by 'bitsleft' can actually be consumed; the rest can only be
+ * used for preloading.
+ *
+ * As a micro-optimization, we allow bits 8 and higher of
+ * 'bitsleft' to contain garbage. When consuming the bits
+ * associated with a decode table entry, this allows us to do
+ * 'bitsleft -= entry' instead of 'bitsleft -= (u8)entry'.
+ * On some CPUs, this helps reduce instruction dependencies.
+ * This does have the disadvantage that 'bitsleft' sometimes
+ * needs to be cast to 'u8', such as when it's used as a shift
+ * amount in REFILL_BITS_BRANCHLESS(). But that one happens
+ * for free since most CPUs ignore high bits in shift amounts.
+ *
+ * - overread_count: the total number of implicit appended zero bytes that
+ * have been loaded into the bitbuffer, including any
+ * counted by 'bitsleft' and any already consumed
+ */
+
+/*
+ * The type for the bitbuffer variable ('bitbuf' described above). For best
+ * performance, this should have size equal to a machine word.
+ *
+ * 64-bit platforms have a significant advantage: they get a bigger bitbuffer
+ * which they don't have to refill as often.
+ */
+typedef machine_word_t bitbuf_t;
+#define BITBUF_NBITS (8 * (int)sizeof(bitbuf_t))
+
+/* BITMASK(n) returns a bitmask of length 'n'. */
+#define BITMASK(n) (((bitbuf_t)1 << (n)) - 1)
+
+/*
+ * MAX_BITSLEFT is the maximum number of consumable bits, i.e. the maximum value
+ * of '(u8)bitsleft'. This is the size of the bitbuffer variable, minus 1 if
+ * the branchless refill method is being used (see REFILL_BITS_BRANCHLESS()).
+ */
+#define MAX_BITSLEFT \
+ (UNALIGNED_ACCESS_IS_FAST ? BITBUF_NBITS - 1 : BITBUF_NBITS)
+
+/*
+ * CONSUMABLE_NBITS is the minimum number of bits that are guaranteed to be
+ * consumable (counted in 'bitsleft') immediately after refilling the bitbuffer.
+ * Since only whole bytes can be added to 'bitsleft', the worst case is
+ * 'MAX_BITSLEFT - 7': the smallest amount where another byte doesn't fit.
+ */
+#define CONSUMABLE_NBITS (MAX_BITSLEFT - 7)
+
+/*
+ * FASTLOOP_PRELOADABLE_NBITS is the minimum number of bits that are guaranteed
+ * to be preloadable immediately after REFILL_BITS_IN_FASTLOOP(). (It is *not*
+ * guaranteed after REFILL_BITS(), since REFILL_BITS() falls back to a
+ * byte-at-a-time refill method near the end of input.) This may exceed the
+ * number of consumable bits (counted by 'bitsleft'). Any bits not counted in
+ * 'bitsleft' can only be used for precomputation and cannot be consumed.
+ */
+#define FASTLOOP_PRELOADABLE_NBITS \
+ (UNALIGNED_ACCESS_IS_FAST ? BITBUF_NBITS : CONSUMABLE_NBITS)
+
+/*
+ * PRELOAD_SLACK is the minimum number of bits that are guaranteed to be
+ * preloadable but not consumable, following REFILL_BITS_IN_FASTLOOP() and any
+ * subsequent consumptions. This is 1 bit if the branchless refill method is
+ * being used, and 0 bits otherwise.
+ */
+#define PRELOAD_SLACK MAX(0, FASTLOOP_PRELOADABLE_NBITS - MAX_BITSLEFT)
+
+/*
+ * CAN_CONSUME(n) is true if it's guaranteed that if the bitbuffer has just been
+ * refilled, then it's always possible to consume 'n' bits from it. 'n' should
+ * be a compile-time constant, to enable compile-time evaluation.
+ */
+#define CAN_CONSUME(n) (CONSUMABLE_NBITS >= (n))
+
+/*
+ * CAN_CONSUME_AND_THEN_PRELOAD(consume_nbits, preload_nbits) is true if it's
+ * guaranteed that after REFILL_BITS_IN_FASTLOOP(), it's always possible to
+ * consume 'consume_nbits' bits, then preload 'preload_nbits' bits. The
+ * arguments should be compile-time constants to enable compile-time evaluation.
+ */
+#define CAN_CONSUME_AND_THEN_PRELOAD(consume_nbits, preload_nbits) \
+ (CONSUMABLE_NBITS >= (consume_nbits) && \
+ FASTLOOP_PRELOADABLE_NBITS >= (consume_nbits) + (preload_nbits))
+
+/*
+ * REFILL_BITS_BRANCHLESS() branchlessly refills the bitbuffer variable by
+ * reading the next word from the input buffer and updating 'in_next' and
+ * 'bitsleft' based on how many bits were refilled -- counting whole bytes only.
+ * This is much faster than reading a byte at a time, at least if the CPU is
+ * little endian and supports fast unaligned memory accesses.
+ *
+ * The simplest way of branchlessly updating 'bitsleft' would be:
+ *
+ * bitsleft += (MAX_BITSLEFT - bitsleft) & ~7;
+ *
+ * To make it faster, we define MAX_BITSLEFT to be 'WORDBITS - 1' rather than
+ * WORDBITS, so that in binary it looks like 111111 or 11111. Then, we update
+ * 'bitsleft' by just setting the bits above the low 3 bits:
+ *
+ * bitsleft |= MAX_BITSLEFT & ~7;
+ *
+ * That compiles down to a single instruction like 'or $0x38, %rbp'. Using
+ * 'MAX_BITSLEFT == WORDBITS - 1' also has the advantage that refills can be
+ * done when 'bitsleft == MAX_BITSLEFT' without invoking undefined behavior.
+ *
+ * The simplest way of branchlessly updating 'in_next' would be:
+ *
+ * in_next += (MAX_BITSLEFT - bitsleft) >> 3;
+ *
+ * With 'MAX_BITSLEFT == WORDBITS - 1' we could use an XOR instead, though this
+ * isn't really better:
+ *
+ * in_next += (MAX_BITSLEFT ^ bitsleft) >> 3;
+ *
+ * An alternative which can be marginally better is the following:
+ *
+ * in_next += sizeof(bitbuf_t) - 1;
+ * in_next -= (bitsleft >> 3) & 0x7;
+ *
+ * It seems this would increase the number of CPU instructions from 3 (sub, shr,
+ * add) to 4 (add, shr, and, sub). However, if the CPU has a bitfield
+ * extraction instruction (e.g. arm's ubfx), it stays at 3, and is potentially
+ * more efficient because the length of the longest dependency chain decreases
+ * from 3 to 2. This alternative also has the advantage that it ignores the
+ * high bits in 'bitsleft', so it is compatible with the micro-optimization we
+ * use where we let the high bits of 'bitsleft' contain garbage.
+ */
+#define REFILL_BITS_BRANCHLESS() \
+do { \
+ bitbuf |= get_unaligned_leword(in_next) << (u8)bitsleft; \
+ in_next += sizeof(bitbuf_t) - 1; \
+ in_next -= (bitsleft >> 3) & 0x7; \
+ bitsleft |= MAX_BITSLEFT & ~7; \
+} while (0)
+
+/*
+ * REFILL_BITS() loads bits from the input buffer until the bitbuffer variable
+ * contains at least CONSUMABLE_NBITS consumable bits.
+ *
+ * This checks for the end of input, and it doesn't guarantee
+ * FASTLOOP_PRELOADABLE_NBITS, so it can't be used in the fastloop.
+ *
+ * If we would overread the input buffer, we just don't read anything, leaving
+ * the bits zeroed but marking them filled. This simplifies the decompressor
+ * because it removes the need to always be able to distinguish between real
+ * overreads and overreads caused only by the decompressor's own lookahead.
+ *
+ * We do still keep track of the number of bytes that have been overread, for
+ * two reasons. First, it allows us to determine the exact number of bytes that
+ * were consumed once the stream ends or an uncompressed block is reached.
+ * Second, it allows us to stop early if the overread amount gets so large (more
+ * than sizeof bitbuf) that it can only be caused by a real overread. (The
+ * second part is arguably unneeded, since libdeflate is buffer-based; given
+ * infinite zeroes, it will eventually either completely fill the output buffer
+ * or return an error. However, we do it to be slightly more friendly to the
+ * not-recommended use case of decompressing with an unknown output size.)
+ */
+#define REFILL_BITS() \
+do { \
+ if (UNALIGNED_ACCESS_IS_FAST && \
+ likely(in_end - in_next >= sizeof(bitbuf_t))) { \
+ REFILL_BITS_BRANCHLESS(); \
+ } else { \
+ while ((u8)bitsleft < CONSUMABLE_NBITS) { \
+ if (likely(in_next != in_end)) { \
+ bitbuf |= (bitbuf_t)*in_next++ << \
+ (u8)bitsleft; \
+ } else { \
+ overread_count++; \
+ SAFETY_CHECK(overread_count <= \
+ sizeof(bitbuf_t)); \
+ } \
+ bitsleft += 8; \
+ } \
+ } \
+} while (0)
+
+/*
+ * REFILL_BITS_IN_FASTLOOP() is like REFILL_BITS(), but it doesn't check for the
+ * end of the input. It can only be used in the fastloop.
+ */
+#define REFILL_BITS_IN_FASTLOOP() \
+do { \
+ STATIC_ASSERT(UNALIGNED_ACCESS_IS_FAST || \
+ FASTLOOP_PRELOADABLE_NBITS == CONSUMABLE_NBITS); \
+ if (UNALIGNED_ACCESS_IS_FAST) { \
+ REFILL_BITS_BRANCHLESS(); \
+ } else { \
+ while ((u8)bitsleft < CONSUMABLE_NBITS) { \
+ bitbuf |= (bitbuf_t)*in_next++ << (u8)bitsleft; \
+ bitsleft += 8; \
+ } \
+ } \
+} while (0)
+
+/*
+ * This is the worst-case maximum number of output bytes that are written to
+ * during each iteration of the fastloop. The worst case is 2 literals, then a
+ * match of length DEFLATE_MAX_MATCH_LEN. Additionally, some slack space must
+ * be included for the intentional overrun in the match copy implementation.
+ */
+#define FASTLOOP_MAX_BYTES_WRITTEN \
+ (2 + DEFLATE_MAX_MATCH_LEN + (5 * WORDBYTES) - 1)
+
+/*
+ * This is the worst-case maximum number of input bytes that are read during
+ * each iteration of the fastloop. To get this value, we first compute the
+ * greatest number of bits that can be refilled during a loop iteration. The
+ * refill at the beginning can add at most MAX_BITSLEFT, and the amount that can
+ * be refilled later is no more than the maximum amount that can be consumed by
+ * 2 literals that don't need a subtable, then a match. We convert this value
+ * to bytes, rounding up; this gives the maximum number of bytes that 'in_next'
+ * can be advanced. Finally, we add sizeof(bitbuf_t) to account for
+ * REFILL_BITS_BRANCHLESS() reading a word past 'in_next'.
+ */
+#define FASTLOOP_MAX_BYTES_READ \
+ (DIV_ROUND_UP(MAX_BITSLEFT + (2 * LITLEN_TABLEBITS) + \
+ LENGTH_MAXBITS + OFFSET_MAXBITS, 8) + \
+ sizeof(bitbuf_t))
+
+/*****************************************************************************
+ * Huffman decoding *
+ *****************************************************************************/
+
+/*
+ * The fastest way to decode Huffman-encoded data is basically to use a decode
+ * table that maps the next TABLEBITS bits of data to their symbol. Each entry
+ * decode_table[i] maps to the symbol whose codeword is a prefix of 'i'. A
+ * symbol with codeword length 'n' has '2**(TABLEBITS-n)' entries in the table.
+ *
+ * Ideally, TABLEBITS and the maximum codeword length would be the same; some
+ * compression formats are designed with this goal in mind. Unfortunately, in
+ * DEFLATE, the maximum litlen and offset codeword lengths are 15 bits, which is
+ * too large for a practical TABLEBITS. It's not *that* much larger, though, so
+ * the workaround is to use a single level of subtables. In the main table,
+ * entries for prefixes of codewords longer than TABLEBITS contain a "pointer"
+ * to the appropriate subtable along with the number of bits it is indexed with.
+ *
+ * The most efficient way to allocate subtables is to allocate them dynamically
+ * after the main table. The worst-case number of table entries needed,
+ * including subtables, is precomputable; see the ENOUGH constants below.
+ *
+ * A useful optimization is to store the codeword lengths in the decode table so
+ * that they don't have to be looked up by indexing a separate table that maps
+ * symbols to their codeword lengths. We basically do this; however, for the
+ * litlen and offset codes we also implement some DEFLATE-specific optimizations
+ * that build in the consideration of the "extra bits" and the
+ * literal/length/end-of-block division. For the exact decode table entry
+ * format we use, see the definitions of the *_decode_results[] arrays below.
+ */
+
+
+/*
+ * These are the TABLEBITS values we use for each of the DEFLATE Huffman codes,
+ * along with their corresponding ENOUGH values.
+ *
+ * For the precode, we use PRECODE_TABLEBITS == 7 since this is the maximum
+ * precode codeword length. This avoids ever needing subtables.
+ *
+ * For the litlen and offset codes, we cannot realistically avoid ever needing
+ * subtables, since litlen and offset codewords can be up to 15 bits. A higher
+ * TABLEBITS reduces the number of lookups that need a subtable, which increases
+ * performance; however, it increases memory usage and makes building the table
+ * take longer, which decreases performance. We choose values that work well in
+ * practice, making subtables rarely needed without making the tables too large.
+ *
+ * Our choice of OFFSET_TABLEBITS == 8 is a bit low; without any special
+ * considerations, 9 would fit the trade-off curve better. However, there is a
+ * performance benefit to using exactly 8 bits when it is a compile-time
+ * constant, as many CPUs can take the low byte more easily than the low 9 bits.
+ *
+ * zlib treats its equivalents of TABLEBITS as maximum values; whenever it
+ * builds a table, it caps the actual table_bits to the longest codeword. This
+ * makes sense in theory, as there's no need for the table to be any larger than
+ * needed to support the longest codeword. However, having the table bits be a
+ * compile-time constant is beneficial to the performance of the decode loop, so
+ * there is a trade-off. libdeflate currently uses the dynamic table_bits
+ * strategy for the litlen table only, due to its larger maximum size.
+ * PRECODE_TABLEBITS and OFFSET_TABLEBITS are smaller, so going dynamic there
+ * isn't as useful, and OFFSET_TABLEBITS=8 is useful as mentioned above.
+ *
+ * Each TABLEBITS value has a corresponding ENOUGH value that gives the
+ * worst-case maximum number of decode table entries, including the main table
+ * and all subtables. The ENOUGH value depends on three parameters:
+ *
+ * (1) the maximum number of symbols in the code (DEFLATE_NUM_*_SYMS)
+ * (2) the maximum number of main table bits (*_TABLEBITS)
+ * (3) the maximum allowed codeword length (DEFLATE_MAX_*_CODEWORD_LEN)
+ *
+ * The ENOUGH values were computed using the utility program 'enough' from zlib.
+ */
+#define PRECODE_TABLEBITS 7
+#define PRECODE_ENOUGH 128 /* enough 19 7 7 */
+#define LITLEN_TABLEBITS 11
+#define LITLEN_ENOUGH 2342 /* enough 288 11 15 */
+#define OFFSET_TABLEBITS 8
+#define OFFSET_ENOUGH 402 /* enough 32 8 15 */
+
+/*
+ * make_decode_table_entry() creates a decode table entry for the given symbol
+ * by combining the static part 'decode_results[sym]' with the dynamic part
+ * 'len', which is the remaining codeword length (the codeword length for main
+ * table entries, or the codeword length minus TABLEBITS for subtable entries).
+ *
+ * In all cases, we add 'len' to each of the two low-order bytes to create the
+ * appropriately-formatted decode table entry. See the definitions of the
+ * *_decode_results[] arrays below, where the entry format is described.
+ */
+static forceinline u32
+make_decode_table_entry(const u32 decode_results[], u32 sym, u32 len)
+{
+ return decode_results[sym] + (len << 8) + len;
+}
+
+/*
+ * Here is the format of our precode decode table entries. Bits not explicitly
+ * described contain zeroes:
+ *
+ * Bit 20-16: presym
+ * Bit 10-8: codeword length [not used]
+ * Bit 2-0: codeword length
+ *
+ * The precode decode table never has subtables, since we use
+ * PRECODE_TABLEBITS == DEFLATE_MAX_PRE_CODEWORD_LEN.
+ *
+ * precode_decode_results[] contains the static part of the entry for each
+ * symbol. make_decode_table_entry() produces the final entries.
+ */
+static const u32 precode_decode_results[] = {
+#define ENTRY(presym) ((u32)presym << 16)
+ ENTRY(0) , ENTRY(1) , ENTRY(2) , ENTRY(3) ,
+ ENTRY(4) , ENTRY(5) , ENTRY(6) , ENTRY(7) ,
+ ENTRY(8) , ENTRY(9) , ENTRY(10) , ENTRY(11) ,
+ ENTRY(12) , ENTRY(13) , ENTRY(14) , ENTRY(15) ,
+ ENTRY(16) , ENTRY(17) , ENTRY(18) ,
+#undef ENTRY
+};
+
+/* Litlen and offset decode table entry flags */
+
+/* Indicates a literal entry in the litlen decode table */
+#define HUFFDEC_LITERAL 0x80000000
+
+/* Indicates that HUFFDEC_SUBTABLE_POINTER or HUFFDEC_END_OF_BLOCK is set */
+#define HUFFDEC_EXCEPTIONAL 0x00008000
+
+/* Indicates a subtable pointer entry in the litlen or offset decode table */
+#define HUFFDEC_SUBTABLE_POINTER 0x00004000
+
+/* Indicates an end-of-block entry in the litlen decode table */
+#define HUFFDEC_END_OF_BLOCK 0x00002000
+
+/* Maximum number of bits that can be consumed by decoding a match length */
+#define LENGTH_MAXBITS (DEFLATE_MAX_LITLEN_CODEWORD_LEN + \
+ DEFLATE_MAX_EXTRA_LENGTH_BITS)
+#define LENGTH_MAXFASTBITS (LITLEN_TABLEBITS /* no subtable needed */ + \
+ DEFLATE_MAX_EXTRA_LENGTH_BITS)
+
+/*
+ * Here is the format of our litlen decode table entries. Bits not explicitly
+ * described contain zeroes:
+ *
+ * Literals:
+ * Bit 31: 1 (HUFFDEC_LITERAL)
+ * Bit 23-16: literal value
+ * Bit 15: 0 (!HUFFDEC_EXCEPTIONAL)
+ * Bit 14: 0 (!HUFFDEC_SUBTABLE_POINTER)
+ * Bit 13: 0 (!HUFFDEC_END_OF_BLOCK)
+ * Bit 11-8: remaining codeword length [not used]
+ * Bit 3-0: remaining codeword length
+ * Lengths:
+ * Bit 31: 0 (!HUFFDEC_LITERAL)
+ * Bit 24-16: length base value
+ * Bit 15: 0 (!HUFFDEC_EXCEPTIONAL)
+ * Bit 14: 0 (!HUFFDEC_SUBTABLE_POINTER)
+ * Bit 13: 0 (!HUFFDEC_END_OF_BLOCK)
+ * Bit 11-8: remaining codeword length
+ * Bit 4-0: remaining codeword length + number of extra bits
+ * End of block:
+ * Bit 31: 0 (!HUFFDEC_LITERAL)
+ * Bit 15: 1 (HUFFDEC_EXCEPTIONAL)
+ * Bit 14: 0 (!HUFFDEC_SUBTABLE_POINTER)
+ * Bit 13: 1 (HUFFDEC_END_OF_BLOCK)
+ * Bit 11-8: remaining codeword length [not used]
+ * Bit 3-0: remaining codeword length
+ * Subtable pointer:
+ * Bit 31: 0 (!HUFFDEC_LITERAL)
+ * Bit 30-16: index of start of subtable
+ * Bit 15: 1 (HUFFDEC_EXCEPTIONAL)
+ * Bit 14: 1 (HUFFDEC_SUBTABLE_POINTER)
+ * Bit 13: 0 (!HUFFDEC_END_OF_BLOCK)
+ * Bit 11-8: number of subtable bits
+ * Bit 3-0: number of main table bits
+ *
+ * This format has several desirable properties:
+ *
+ * - The codeword length, length slot base, and number of extra length bits
+ * are all built in. This eliminates the need to separately look up this
+ * information by indexing separate arrays by symbol or length slot.
+ *
+ * - The HUFFDEC_* flags enable easily distinguishing between the different
+ * types of entries. The HUFFDEC_LITERAL flag enables a fast path for
+ * literals; the high bit is used for this, as some CPUs can test the
+ * high bit more easily than other bits. The HUFFDEC_EXCEPTIONAL flag
+ * makes it possible to detect the two unlikely cases (subtable pointer
+ * and end of block) in a single bit flag test.
+ *
+ * - The low byte is the number of bits that need to be removed from the
+ * bitstream; this makes this value easily accessible, and it enables the
+ * micro-optimization of doing 'bitsleft -= entry' instead of
+ * 'bitsleft -= (u8)entry'. It also includes the number of extra bits,
+ * so they don't need to be removed separately.
+ *
+ * - The flags in bits 15-13 are arranged to be 0 when the
+ * "remaining codeword length" in bits 11-8 is needed, making this value
+ * fairly easily accessible as well via a shift and downcast.
+ *
+ * - Similarly, bits 13-12 are 0 when the "subtable bits" in bits 11-8 are
+ * needed, making it possible to extract this value with '& 0x3F' rather
+ * than '& 0xF'. This value is only used as a shift amount, so this can
+ * save an 'and' instruction as the masking by 0x3F happens implicitly.
+ *
+ * litlen_decode_results[] contains the static part of the entry for each
+ * symbol. make_decode_table_entry() produces the final entries.
+ */
+static const u32 litlen_decode_results[] = {
+
+ /* Literals */
+#define ENTRY(literal) (HUFFDEC_LITERAL | ((u32)literal << 16))
+ ENTRY(0) , ENTRY(1) , ENTRY(2) , ENTRY(3) ,
+ ENTRY(4) , ENTRY(5) , ENTRY(6) , ENTRY(7) ,
+ ENTRY(8) , ENTRY(9) , ENTRY(10) , ENTRY(11) ,
+ ENTRY(12) , ENTRY(13) , ENTRY(14) , ENTRY(15) ,
+ ENTRY(16) , ENTRY(17) , ENTRY(18) , ENTRY(19) ,
+ ENTRY(20) , ENTRY(21) , ENTRY(22) , ENTRY(23) ,
+ ENTRY(24) , ENTRY(25) , ENTRY(26) , ENTRY(27) ,
+ ENTRY(28) , ENTRY(29) , ENTRY(30) , ENTRY(31) ,
+ ENTRY(32) , ENTRY(33) , ENTRY(34) , ENTRY(35) ,
+ ENTRY(36) , ENTRY(37) , ENTRY(38) , ENTRY(39) ,
+ ENTRY(40) , ENTRY(41) , ENTRY(42) , ENTRY(43) ,
+ ENTRY(44) , ENTRY(45) , ENTRY(46) , ENTRY(47) ,
+ ENTRY(48) , ENTRY(49) , ENTRY(50) , ENTRY(51) ,
+ ENTRY(52) , ENTRY(53) , ENTRY(54) , ENTRY(55) ,
+ ENTRY(56) , ENTRY(57) , ENTRY(58) , ENTRY(59) ,
+ ENTRY(60) , ENTRY(61) , ENTRY(62) , ENTRY(63) ,
+ ENTRY(64) , ENTRY(65) , ENTRY(66) , ENTRY(67) ,
+ ENTRY(68) , ENTRY(69) , ENTRY(70) , ENTRY(71) ,
+ ENTRY(72) , ENTRY(73) , ENTRY(74) , ENTRY(75) ,
+ ENTRY(76) , ENTRY(77) , ENTRY(78) , ENTRY(79) ,
+ ENTRY(80) , ENTRY(81) , ENTRY(82) , ENTRY(83) ,
+ ENTRY(84) , ENTRY(85) , ENTRY(86) , ENTRY(87) ,
+ ENTRY(88) , ENTRY(89) , ENTRY(90) , ENTRY(91) ,
+ ENTRY(92) , ENTRY(93) , ENTRY(94) , ENTRY(95) ,
+ ENTRY(96) , ENTRY(97) , ENTRY(98) , ENTRY(99) ,
+ ENTRY(100) , ENTRY(101) , ENTRY(102) , ENTRY(103) ,
+ ENTRY(104) , ENTRY(105) , ENTRY(106) , ENTRY(107) ,
+ ENTRY(108) , ENTRY(109) , ENTRY(110) , ENTRY(111) ,
+ ENTRY(112) , ENTRY(113) , ENTRY(114) , ENTRY(115) ,
+ ENTRY(116) , ENTRY(117) , ENTRY(118) , ENTRY(119) ,
+ ENTRY(120) , ENTRY(121) , ENTRY(122) , ENTRY(123) ,
+ ENTRY(124) , ENTRY(125) , ENTRY(126) , ENTRY(127) ,
+ ENTRY(128) , ENTRY(129) , ENTRY(130) , ENTRY(131) ,
+ ENTRY(132) , ENTRY(133) , ENTRY(134) , ENTRY(135) ,
+ ENTRY(136) , ENTRY(137) , ENTRY(138) , ENTRY(139) ,
+ ENTRY(140) , ENTRY(141) , ENTRY(142) , ENTRY(143) ,
+ ENTRY(144) , ENTRY(145) , ENTRY(146) , ENTRY(147) ,
+ ENTRY(148) , ENTRY(149) , ENTRY(150) , ENTRY(151) ,
+ ENTRY(152) , ENTRY(153) , ENTRY(154) , ENTRY(155) ,
+ ENTRY(156) , ENTRY(157) , ENTRY(158) , ENTRY(159) ,
+ ENTRY(160) , ENTRY(161) , ENTRY(162) , ENTRY(163) ,
+ ENTRY(164) , ENTRY(165) , ENTRY(166) , ENTRY(167) ,
+ ENTRY(168) , ENTRY(169) , ENTRY(170) , ENTRY(171) ,
+ ENTRY(172) , ENTRY(173) , ENTRY(174) , ENTRY(175) ,
+ ENTRY(176) , ENTRY(177) , ENTRY(178) , ENTRY(179) ,
+ ENTRY(180) , ENTRY(181) , ENTRY(182) , ENTRY(183) ,
+ ENTRY(184) , ENTRY(185) , ENTRY(186) , ENTRY(187) ,
+ ENTRY(188) , ENTRY(189) , ENTRY(190) , ENTRY(191) ,
+ ENTRY(192) , ENTRY(193) , ENTRY(194) , ENTRY(195) ,
+ ENTRY(196) , ENTRY(197) , ENTRY(198) , ENTRY(199) ,
+ ENTRY(200) , ENTRY(201) , ENTRY(202) , ENTRY(203) ,
+ ENTRY(204) , ENTRY(205) , ENTRY(206) , ENTRY(207) ,
+ ENTRY(208) , ENTRY(209) , ENTRY(210) , ENTRY(211) ,
+ ENTRY(212) , ENTRY(213) , ENTRY(214) , ENTRY(215) ,
+ ENTRY(216) , ENTRY(217) , ENTRY(218) , ENTRY(219) ,
+ ENTRY(220) , ENTRY(221) , ENTRY(222) , ENTRY(223) ,
+ ENTRY(224) , ENTRY(225) , ENTRY(226) , ENTRY(227) ,
+ ENTRY(228) , ENTRY(229) , ENTRY(230) , ENTRY(231) ,
+ ENTRY(232) , ENTRY(233) , ENTRY(234) , ENTRY(235) ,
+ ENTRY(236) , ENTRY(237) , ENTRY(238) , ENTRY(239) ,
+ ENTRY(240) , ENTRY(241) , ENTRY(242) , ENTRY(243) ,
+ ENTRY(244) , ENTRY(245) , ENTRY(246) , ENTRY(247) ,
+ ENTRY(248) , ENTRY(249) , ENTRY(250) , ENTRY(251) ,
+ ENTRY(252) , ENTRY(253) , ENTRY(254) , ENTRY(255) ,
+#undef ENTRY
+
+ /* End of block */
+ HUFFDEC_EXCEPTIONAL | HUFFDEC_END_OF_BLOCK,
+
+ /* Lengths */
+#define ENTRY(length_base, num_extra_bits) \
+ (((u32)(length_base) << 16) | (num_extra_bits))
+ ENTRY(3 , 0) , ENTRY(4 , 0) , ENTRY(5 , 0) , ENTRY(6 , 0),
+ ENTRY(7 , 0) , ENTRY(8 , 0) , ENTRY(9 , 0) , ENTRY(10 , 0),
+ ENTRY(11 , 1) , ENTRY(13 , 1) , ENTRY(15 , 1) , ENTRY(17 , 1),
+ ENTRY(19 , 2) , ENTRY(23 , 2) , ENTRY(27 , 2) , ENTRY(31 , 2),
+ ENTRY(35 , 3) , ENTRY(43 , 3) , ENTRY(51 , 3) , ENTRY(59 , 3),
+ ENTRY(67 , 4) , ENTRY(83 , 4) , ENTRY(99 , 4) , ENTRY(115, 4),
+ ENTRY(131, 5) , ENTRY(163, 5) , ENTRY(195, 5) , ENTRY(227, 5),
+ ENTRY(258, 0) , ENTRY(258, 0) , ENTRY(258, 0) ,
+#undef ENTRY
+};
+
+/* Maximum number of bits that can be consumed by decoding a match offset */
+#define OFFSET_MAXBITS (DEFLATE_MAX_OFFSET_CODEWORD_LEN + \
+ DEFLATE_MAX_EXTRA_OFFSET_BITS)
+#define OFFSET_MAXFASTBITS (OFFSET_TABLEBITS /* no subtable needed */ + \
+ DEFLATE_MAX_EXTRA_OFFSET_BITS)
+
+/*
+ * Here is the format of our offset decode table entries. Bits not explicitly
+ * described contain zeroes:
+ *
+ * Offsets:
+ * Bit 31-16: offset base value
+ * Bit 15: 0 (!HUFFDEC_EXCEPTIONAL)
+ * Bit 14: 0 (!HUFFDEC_SUBTABLE_POINTER)
+ * Bit 11-8: remaining codeword length
+ * Bit 4-0: remaining codeword length + number of extra bits
+ * Subtable pointer:
+ * Bit 31-16: index of start of subtable
+ * Bit 15: 1 (HUFFDEC_EXCEPTIONAL)
+ * Bit 14: 1 (HUFFDEC_SUBTABLE_POINTER)
+ * Bit 11-8: number of subtable bits
+ * Bit 3-0: number of main table bits
+ *
+ * These work the same way as the length entries and subtable pointer entries in
+ * the litlen decode table; see litlen_decode_results[] above.
+ */
+static const u32 offset_decode_results[] = {
+#define ENTRY(offset_base, num_extra_bits) \
+ (((u32)(offset_base) << 16) | (num_extra_bits))
+ ENTRY(1 , 0) , ENTRY(2 , 0) , ENTRY(3 , 0) , ENTRY(4 , 0) ,
+ ENTRY(5 , 1) , ENTRY(7 , 1) , ENTRY(9 , 2) , ENTRY(13 , 2) ,
+ ENTRY(17 , 3) , ENTRY(25 , 3) , ENTRY(33 , 4) , ENTRY(49 , 4) ,
+ ENTRY(65 , 5) , ENTRY(97 , 5) , ENTRY(129 , 6) , ENTRY(193 , 6) ,
+ ENTRY(257 , 7) , ENTRY(385 , 7) , ENTRY(513 , 8) , ENTRY(769 , 8) ,
+ ENTRY(1025 , 9) , ENTRY(1537 , 9) , ENTRY(2049 , 10) , ENTRY(3073 , 10) ,
+ ENTRY(4097 , 11) , ENTRY(6145 , 11) , ENTRY(8193 , 12) , ENTRY(12289 , 12) ,
+ ENTRY(16385 , 13) , ENTRY(24577 , 13) , ENTRY(24577 , 13) , ENTRY(24577 , 13) ,
+#undef ENTRY
+};
+
+/*
+ * The main DEFLATE decompressor structure. Since libdeflate only supports
+ * full-buffer decompression, this structure doesn't store the entire
+ * decompression state, most of which is in stack variables. Instead, this
+ * struct just contains the decode tables and some temporary arrays used for
+ * building them, as these are too large to comfortably allocate on the stack.
+ *
+ * Storing the decode tables in the decompressor struct also allows the decode
+ * tables for the static codes to be reused whenever two static Huffman blocks
+ * are decoded without an intervening dynamic block, even across streams.
+ */
+struct libdeflate_decompressor {
+
+ /*
+ * The arrays aren't all needed at the same time. 'precode_lens' and
+ * 'precode_decode_table' are unneeded after 'lens' has been filled.
+ * Furthermore, 'lens' need not be retained after building the litlen
+ * and offset decode tables. In fact, 'lens' can be in union with
+ * 'litlen_decode_table' provided that 'offset_decode_table' is separate
+ * and is built first.
+ */
+
+ union {
+ u8 precode_lens[DEFLATE_NUM_PRECODE_SYMS];
+
+ struct {
+ u8 lens[DEFLATE_NUM_LITLEN_SYMS +
+ DEFLATE_NUM_OFFSET_SYMS +
+ DEFLATE_MAX_LENS_OVERRUN];
+
+ u32 precode_decode_table[PRECODE_ENOUGH];
+ } l;
+
+ u32 litlen_decode_table[LITLEN_ENOUGH];
+ } u;
+
+ u32 offset_decode_table[OFFSET_ENOUGH];
+
+ /* used only during build_decode_table() */
+ u16 sorted_syms[DEFLATE_MAX_NUM_SYMS];
+
+ bool static_codes_loaded;
+ unsigned litlen_tablebits;
+
+ /* The free() function for this struct, chosen at allocation time */
+ free_func_t free_func;
+};
+
+/*
+ * Build a table for fast decoding of symbols from a Huffman code. As input,
+ * this function takes the codeword length of each symbol which may be used in
+ * the code. As output, it produces a decode table for the canonical Huffman
+ * code described by the codeword lengths. The decode table is built with the
+ * assumption that it will be indexed with "bit-reversed" codewords, where the
+ * low-order bit is the first bit of the codeword. This format is used for all
+ * Huffman codes in DEFLATE.
+ *
+ * @decode_table
+ * The array in which the decode table will be generated. This array must
+ * have sufficient length; see the definition of the ENOUGH numbers.
+ * @lens
+ * An array which provides, for each symbol, the length of the
+ * corresponding codeword in bits, or 0 if the symbol is unused. This may
+ * alias @decode_table, since nothing is written to @decode_table until all
+ * @lens have been consumed. All codeword lengths are assumed to be <=
+ * @max_codeword_len but are otherwise considered untrusted. If they do
+ * not form a valid Huffman code, then the decode table is not built and
+ * %false is returned.
+ * @num_syms
+ * The number of symbols in the code, including all unused symbols.
+ * @decode_results
+ * An array which gives the incomplete decode result for each symbol. The
+ * needed values in this array will be combined with codeword lengths to
+ * make the final decode table entries using make_decode_table_entry().
+ * @table_bits
+ * The log base-2 of the number of main table entries to use.
+ * If @table_bits_ret != NULL, then @table_bits is treated as a maximum
+ * value and it will be decreased if a smaller table would be sufficient.
+ * @max_codeword_len
+ * The maximum allowed codeword length for this Huffman code.
+ * Must be <= DEFLATE_MAX_CODEWORD_LEN.
+ * @sorted_syms
+ * A temporary array of length @num_syms.
+ * @table_bits_ret
+ * If non-NULL, then the dynamic table_bits is enabled, and the actual
+ * table_bits value will be returned here.
+ *
+ * Returns %true if successful; %false if the codeword lengths do not form a
+ * valid Huffman code.
+ */
+static bool
+build_decode_table(u32 decode_table[],
+ const u8 lens[],
+ const unsigned num_syms,
+ const u32 decode_results[],
+ unsigned table_bits,
+ unsigned max_codeword_len,
+ u16 *sorted_syms,
+ unsigned *table_bits_ret)
+{
+ unsigned len_counts[DEFLATE_MAX_CODEWORD_LEN + 1];
+ unsigned offsets[DEFLATE_MAX_CODEWORD_LEN + 1];
+ unsigned sym; /* current symbol */
+ unsigned codeword; /* current codeword, bit-reversed */
+ unsigned len; /* current codeword length in bits */
+ unsigned count; /* num codewords remaining with this length */
+ u32 codespace_used; /* codespace used out of '2^max_codeword_len' */
+ unsigned cur_table_end; /* end index of current table */
+ unsigned subtable_prefix; /* codeword prefix of current subtable */
+ unsigned subtable_start; /* start index of current subtable */
+ unsigned subtable_bits; /* log2 of current subtable length */
+
+ /* Count how many codewords have each length, including 0. */
+ for (len = 0; len <= max_codeword_len; len++)
+ len_counts[len] = 0;
+ for (sym = 0; sym < num_syms; sym++)
+ len_counts[lens[sym]]++;
+
+ /*
+ * Determine the actual maximum codeword length that was used, and
+ * decrease table_bits to it if allowed.
+ */
+ while (max_codeword_len > 1 && len_counts[max_codeword_len] == 0)
+ max_codeword_len--;
+ if (table_bits_ret != NULL) {
+ table_bits = MIN(table_bits, max_codeword_len);
+ *table_bits_ret = table_bits;
+ }
+
+ /*
+ * Sort the symbols primarily by increasing codeword length and
+ * secondarily by increasing symbol value; or equivalently by their
+ * codewords in lexicographic order, since a canonical code is assumed.
+ *
+ * For efficiency, also compute 'codespace_used' in the same pass over
+ * 'len_counts[]' used to build 'offsets[]' for sorting.
+ */
+
+ /* Ensure that 'codespace_used' cannot overflow. */
+ STATIC_ASSERT(sizeof(codespace_used) == 4);
+ STATIC_ASSERT(UINT32_MAX / (1U << (DEFLATE_MAX_CODEWORD_LEN - 1)) >=
+ DEFLATE_MAX_NUM_SYMS);
+
+ offsets[0] = 0;
+ offsets[1] = len_counts[0];
+ codespace_used = 0;
+ for (len = 1; len < max_codeword_len; len++) {
+ offsets[len + 1] = offsets[len] + len_counts[len];
+ codespace_used = (codespace_used << 1) + len_counts[len];
+ }
+ codespace_used = (codespace_used << 1) + len_counts[len];
+
+ for (sym = 0; sym < num_syms; sym++)
+ sorted_syms[offsets[lens[sym]]++] = sym;
+
+ sorted_syms += offsets[0]; /* Skip unused symbols */
+
+ /* lens[] is done being used, so we can write to decode_table[] now. */
+
+ /*
+ * Check whether the lengths form a complete code (exactly fills the
+ * codespace), an incomplete code (doesn't fill the codespace), or an
+ * overfull code (overflows the codespace). A codeword of length 'n'
+ * uses proportion '1/(2^n)' of the codespace. An overfull code is
+ * nonsensical, so is considered invalid. An incomplete code is
+ * considered valid only in two specific cases; see below.
+ */
+
+ /* overfull code? */
+ if (unlikely(codespace_used > (1U << max_codeword_len)))
+ return false;
+
+ /* incomplete code? */
+ if (unlikely(codespace_used < (1U << max_codeword_len))) {
+ u32 entry;
+ unsigned i;
+
+ /*
+ * The DEFLATE RFC explicitly allows the offset code to be
+ * incomplete in two cases: a code containing just 1 codeword,
+ * if that codeword has length 1; and a code containing no
+ * codewords. Note: the list of offset codeword lengths is
+ * always nonempty, but lengths of 0 don't count as codewords.
+ *
+ * The RFC doesn't say whether the same cases are allowed for
+ * the litlen and pre codes. It's actually impossible for no
+ * symbols to be used from these codes; however, it's
+ * technically possible for only one symbol to be used. zlib
+ * allows 1 codeword for the litlen code, but not the precode.
+ * The RFC also doesn't say whether, when there is 1 codeword,
+ * that codeword is '0' or '1'. zlib uses '0'.
+ *
+ * We accept what zlib accepts, plus a bit more. First, we
+ * don't treat the precode more strictly than the litlen and
+ * offset codes. There's no convincing reason to add a special
+ * case for the precode here.
+ *
+ * Second, we just map each allowed incompete code to a complete
+ * code with only real symbols. To do this, we choose a symbol,
+ * either the used symbol (for codes with 1 codeword) or an
+ * arbitrary symbol (for empty codes), and give it both
+ * codewords '0' and '1'. zlib instead uses a special ERROR
+ * symbol in the part of the codespace the code doesn't use.
+ * However, having an ERROR symbol reduces the performance of
+ * the Huffman decoder, for no real benefit. Our approach also
+ * avoids having to decide whether '0' or '1' is correct.
+ *
+ * Like zlib, we still reject all incomplete codes that contain
+ * more than 1 codeword or a codeword length greater than 1.
+ */
+ if (codespace_used == 0) {
+ sym = 0; /* arbitrary */
+ } else {
+ if (codespace_used != (1U << (max_codeword_len - 1)) ||
+ len_counts[1] != 1)
+ return false;
+ sym = sorted_syms[0];
+ }
+ entry = make_decode_table_entry(decode_results, sym, 1);
+ for (i = 0; i < (1U << table_bits); i++)
+ decode_table[i] = entry;
+ return true;
+ }
+
+ /*
+ * The lengths form a complete code. Now, enumerate the codewords in
+ * lexicographic order and fill the decode table entries for each one.
+ *
+ * First, process all codewords with len <= table_bits. Each one gets
+ * '2^(table_bits-len)' direct entries in the table.
+ *
+ * Since DEFLATE uses bit-reversed codewords, these entries aren't
+ * consecutive but rather are spaced '2^len' entries apart. This makes
+ * filling them naively somewhat awkward and inefficient, since strided
+ * stores are less cache-friendly and preclude the use of word or
+ * vector-at-a-time stores to fill multiple entries per instruction.
+ *
+ * To optimize this, we incrementally double the table size. When
+ * processing codewords with length 'len', the table is treated as
+ * having only '2^len' entries, so each codeword uses just one entry.
+ * Then, each time 'len' is incremented, the table size is doubled and
+ * the first half is copied to the second half. This significantly
+ * improves performance over naively doing strided stores.
+ *
+ * Note that some entries copied for each table doubling may not have
+ * been initialized yet, but it doesn't matter since they're guaranteed
+ * to be initialized later (because the Huffman code is complete).
+ */
+ codeword = 0;
+ len = 1;
+ while ((count = len_counts[len]) == 0)
+ len++;
+ cur_table_end = 1U << len;
+ while (len <= table_bits) {
+ /* Process all 'count' codewords with length 'len' bits. */
+ do {
+ unsigned bit;
+
+ /* Fill the first entry for the current codeword. */
+ decode_table[codeword] =
+ make_decode_table_entry(decode_results,
+ *sorted_syms++, len);
+
+ if (codeword == cur_table_end - 1) {
+ /* Last codeword (all 1's) */
+ for (; len < table_bits; len++) {
+ memcpy(&decode_table[cur_table_end],
+ decode_table,
+ cur_table_end *
+ sizeof(decode_table[0]));
+ cur_table_end <<= 1;
+ }
+ return true;
+ }
+ /*
+ * To advance to the lexicographically next codeword in
+ * the canonical code, the codeword must be incremented,
+ * then 0's must be appended to the codeword as needed
+ * to match the next codeword's length.
+ *
+ * Since the codeword is bit-reversed, appending 0's is
+ * a no-op. However, incrementing it is nontrivial. To
+ * do so efficiently, use the 'bsr' instruction to find
+ * the last (highest order) 0 bit in the codeword, set
+ * it, and clear any later (higher order) 1 bits. But
+ * 'bsr' actually finds the highest order 1 bit, so to
+ * use it first flip all bits in the codeword by XOR'ing
+ * it with (1U << len) - 1 == cur_table_end - 1.
+ */
+ bit = 1U << bsr32(codeword ^ (cur_table_end - 1));
+ codeword &= bit - 1;
+ codeword |= bit;
+ } while (--count);
+
+ /* Advance to the next codeword length. */
+ do {
+ if (++len <= table_bits) {
+ memcpy(&decode_table[cur_table_end],
+ decode_table,
+ cur_table_end * sizeof(decode_table[0]));
+ cur_table_end <<= 1;
+ }
+ } while ((count = len_counts[len]) == 0);
+ }
+
+ /* Process codewords with len > table_bits. These require subtables. */
+ cur_table_end = 1U << table_bits;
+ subtable_prefix = -1;
+ subtable_start = 0;
+ for (;;) {
+ u32 entry;
+ unsigned i;
+ unsigned stride;
+ unsigned bit;
+
+ /*
+ * Start a new subtable if the first 'table_bits' bits of the
+ * codeword don't match the prefix of the current subtable.
+ */
+ if ((codeword & ((1U << table_bits) - 1)) != subtable_prefix) {
+ subtable_prefix = (codeword & ((1U << table_bits) - 1));
+ subtable_start = cur_table_end;
+ /*
+ * Calculate the subtable length. If the codeword has
+ * length 'table_bits + n', then the subtable needs
+ * '2^n' entries. But it may need more; if fewer than
+ * '2^n' codewords of length 'table_bits + n' remain,
+ * then the length will need to be incremented to bring
+ * in longer codewords until the subtable can be
+ * completely filled. Note that because the Huffman
+ * code is complete, it will always be possible to fill
+ * the subtable eventually.
+ */
+ subtable_bits = len - table_bits;
+ codespace_used = count;
+ while (codespace_used < (1U << subtable_bits)) {
+ subtable_bits++;
+ codespace_used = (codespace_used << 1) +
+ len_counts[table_bits + subtable_bits];
+ }
+ cur_table_end = subtable_start + (1U << subtable_bits);
+
+ /*
+ * Create the entry that points from the main table to
+ * the subtable.
+ */
+ decode_table[subtable_prefix] =
+ ((u32)subtable_start << 16) |
+ HUFFDEC_EXCEPTIONAL |
+ HUFFDEC_SUBTABLE_POINTER |
+ (subtable_bits << 8) | table_bits;
+ }
+
+ /* Fill the subtable entries for the current codeword. */
+ entry = make_decode_table_entry(decode_results, *sorted_syms++,
+ len - table_bits);
+ i = subtable_start + (codeword >> table_bits);
+ stride = 1U << (len - table_bits);
+ do {
+ decode_table[i] = entry;
+ i += stride;
+ } while (i < cur_table_end);
+
+ /* Advance to the next codeword. */
+ if (codeword == (1U << len) - 1) /* last codeword (all 1's)? */
+ return true;
+ bit = 1U << bsr32(codeword ^ ((1U << len) - 1));
+ codeword &= bit - 1;
+ codeword |= bit;
+ count--;
+ while (count == 0)
+ count = len_counts[++len];
+ }
+}
+
+/* Build the decode table for the precode. */
+static bool
+build_precode_decode_table(struct libdeflate_decompressor *d)
+{
+ /* When you change TABLEBITS, you must change ENOUGH, and vice versa! */
+ STATIC_ASSERT(PRECODE_TABLEBITS == 7 && PRECODE_ENOUGH == 128);
+
+ STATIC_ASSERT(ARRAY_LEN(precode_decode_results) ==
+ DEFLATE_NUM_PRECODE_SYMS);
+
+ return build_decode_table(d->u.l.precode_decode_table,
+ d->u.precode_lens,
+ DEFLATE_NUM_PRECODE_SYMS,
+ precode_decode_results,
+ PRECODE_TABLEBITS,
+ DEFLATE_MAX_PRE_CODEWORD_LEN,
+ d->sorted_syms,
+ NULL);
+}
+
+/* Build the decode table for the literal/length code. */
+static bool
+build_litlen_decode_table(struct libdeflate_decompressor *d,
+ unsigned num_litlen_syms, unsigned num_offset_syms)
+{
+ /* When you change TABLEBITS, you must change ENOUGH, and vice versa! */
+ STATIC_ASSERT(LITLEN_TABLEBITS == 11 && LITLEN_ENOUGH == 2342);
+
+ STATIC_ASSERT(ARRAY_LEN(litlen_decode_results) ==
+ DEFLATE_NUM_LITLEN_SYMS);
+
+ return build_decode_table(d->u.litlen_decode_table,
+ d->u.l.lens,
+ num_litlen_syms,
+ litlen_decode_results,
+ LITLEN_TABLEBITS,
+ DEFLATE_MAX_LITLEN_CODEWORD_LEN,
+ d->sorted_syms,
+ &d->litlen_tablebits);
+}
+
+/* Build the decode table for the offset code. */
+static bool
+build_offset_decode_table(struct libdeflate_decompressor *d,
+ unsigned num_litlen_syms, unsigned num_offset_syms)
+{
+ /* When you change TABLEBITS, you must change ENOUGH, and vice versa! */
+ STATIC_ASSERT(OFFSET_TABLEBITS == 8 && OFFSET_ENOUGH == 402);
+
+ STATIC_ASSERT(ARRAY_LEN(offset_decode_results) ==
+ DEFLATE_NUM_OFFSET_SYMS);
+
+ return build_decode_table(d->offset_decode_table,
+ d->u.l.lens + num_litlen_syms,
+ num_offset_syms,
+ offset_decode_results,
+ OFFSET_TABLEBITS,
+ DEFLATE_MAX_OFFSET_CODEWORD_LEN,
+ d->sorted_syms,
+ NULL);
+}
+
+/*****************************************************************************
+ * Main decompression routine
+ *****************************************************************************/
+
+typedef enum libdeflate_result (*decompress_func_t)
+ (struct libdeflate_decompressor * restrict d,
+ const void * restrict in, size_t in_nbytes,
+ void * restrict out, size_t out_nbytes_avail,
+ size_t *actual_in_nbytes_ret, size_t *actual_out_nbytes_ret);
+
+#define FUNCNAME deflate_decompress_default
+#undef ATTRIBUTES
+#undef EXTRACT_VARBITS
+#undef EXTRACT_VARBITS8
+#include "decompress_template.h"
+
+/* Include architecture-specific implementation(s) if available. */
+#undef DEFAULT_IMPL
+#undef arch_select_decompress_func
+#if defined(ARCH_X86_32) || defined(ARCH_X86_64)
+# include "x86/decompress_impl.h"
+#endif
+
+#ifndef DEFAULT_IMPL
+# define DEFAULT_IMPL deflate_decompress_default
+#endif
+
+#ifdef arch_select_decompress_func
+static enum libdeflate_result
+dispatch_decomp(struct libdeflate_decompressor *d,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail,
+ size_t *actual_in_nbytes_ret, size_t *actual_out_nbytes_ret);
+
+static volatile decompress_func_t decompress_impl = dispatch_decomp;
+
+/* Choose the best implementation at runtime. */
+static enum libdeflate_result
+dispatch_decomp(struct libdeflate_decompressor *d,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail,
+ size_t *actual_in_nbytes_ret, size_t *actual_out_nbytes_ret)
+{
+ decompress_func_t f = arch_select_decompress_func();
+
+ if (f == NULL)
+ f = DEFAULT_IMPL;
+
+ decompress_impl = f;
+ return f(d, in, in_nbytes, out, out_nbytes_avail,
+ actual_in_nbytes_ret, actual_out_nbytes_ret);
+}
+#else
+/* The best implementation is statically known, so call it directly. */
+# define decompress_impl DEFAULT_IMPL
+#endif
+
+/*
+ * This is the main DEFLATE decompression routine. See libdeflate.h for the
+ * documentation.
+ *
+ * Note that the real code is in decompress_template.h. The part here just
+ * handles calling the appropriate implementation depending on the CPU features
+ * at runtime.
+ */
+LIBDEFLATEAPI enum libdeflate_result
+libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *d,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail,
+ size_t *actual_in_nbytes_ret,
+ size_t *actual_out_nbytes_ret)
+{
+ return decompress_impl(d, in, in_nbytes, out, out_nbytes_avail,
+ actual_in_nbytes_ret, actual_out_nbytes_ret);
+}
+
+LIBDEFLATEAPI enum libdeflate_result
+libdeflate_deflate_decompress(struct libdeflate_decompressor *d,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail,
+ size_t *actual_out_nbytes_ret)
+{
+ return libdeflate_deflate_decompress_ex(d, in, in_nbytes,
+ out, out_nbytes_avail,
+ NULL, actual_out_nbytes_ret);
+}
+
+LIBDEFLATEAPI struct libdeflate_decompressor *
+libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options)
+{
+ struct libdeflate_decompressor *d;
+
+ /*
+ * Note: if more fields are added to libdeflate_options, this code will
+ * need to be updated to support both the old and new structs.
+ */
+ if (options->sizeof_options != sizeof(*options))
+ return NULL;
+
+ d = (options->malloc_func ? options->malloc_func :
+ libdeflate_default_malloc_func)(sizeof(*d));
+ if (d == NULL)
+ return NULL;
+ /*
+ * Note that only certain parts of the decompressor actually must be
+ * initialized here:
+ *
+ * - 'static_codes_loaded' must be initialized to false.
+ *
+ * - The first half of the main portion of each decode table must be
+ * initialized to any value, to avoid reading from uninitialized
+ * memory during table expansion in build_decode_table(). (Although,
+ * this is really just to avoid warnings with dynamic tools like
+ * valgrind, since build_decode_table() is guaranteed to initialize
+ * all entries eventually anyway.)
+ *
+ * - 'free_func' must be set.
+ *
+ * But for simplicity, we currently just zero the whole decompressor.
+ */
+ memset(d, 0, sizeof(*d));
+ d->free_func = options->free_func ?
+ options->free_func : libdeflate_default_free_func;
+ return d;
+}
+
+LIBDEFLATEAPI struct libdeflate_decompressor *
+libdeflate_alloc_decompressor(void)
+{
+ static const struct libdeflate_options defaults = {
+ .sizeof_options = sizeof(defaults),
+ };
+ return libdeflate_alloc_decompressor_ex(&defaults);
+}
+
+LIBDEFLATEAPI void
+libdeflate_free_decompressor(struct libdeflate_decompressor *d)
+{
+ if (d)
+ d->free_func(d);
+}
diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/lib_common.h b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/lib_common.h
new file mode 100644
index 0000000..faedfcb
--- /dev/null
+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/lib_common.h
@@ -0,0 +1,106 @@
+/*
+ * lib_common.h - internal header included by all library code
+ */
+
+#ifndef LIB_LIB_COMMON_H
+#define LIB_LIB_COMMON_H
+
+#ifdef LIBDEFLATE_H
+ /*
+ * When building the library, LIBDEFLATEAPI needs to be defined properly before
+ * including libdeflate.h.
+ */
+# error "lib_common.h must always be included before libdeflate.h"
+#endif
+
+#if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
+# define LIBDEFLATE_EXPORT_SYM __declspec(dllexport)
+#elif defined(__GNUC__)
+# define LIBDEFLATE_EXPORT_SYM __attribute__((visibility("default")))
+#else
+# define LIBDEFLATE_EXPORT_SYM
+#endif
+
+/*
+ * On i386, gcc assumes that the stack is 16-byte aligned at function entry.
+ * However, some compilers (e.g. MSVC) and programming languages (e.g. Delphi)
+ * only guarantee 4-byte alignment when calling functions. This is mainly an
+ * issue on Windows, but it has been seen on Linux too. Work around this ABI
+ * incompatibility by realigning the stack pointer when entering libdeflate.
+ * This prevents crashes in SSE/AVX code.
+ */
+#if defined(__GNUC__) && defined(__i386__)
+# define LIBDEFLATE_ALIGN_STACK __attribute__((force_align_arg_pointer))
+#else
+# define LIBDEFLATE_ALIGN_STACK
+#endif
+
+#define LIBDEFLATEAPI LIBDEFLATE_EXPORT_SYM LIBDEFLATE_ALIGN_STACK
+
+#include "../common_defs.h"
+
+typedef void *(*malloc_func_t)(size_t);
+typedef void (*free_func_t)(void *);
+
+extern malloc_func_t libdeflate_default_malloc_func;
+extern free_func_t libdeflate_default_free_func;
+
+void *libdeflate_aligned_malloc(malloc_func_t malloc_func,
+ size_t alignment, size_t size);
+void libdeflate_aligned_free(free_func_t free_func, void *ptr);
+
+#ifdef FREESTANDING
+/*
+ * With -ffreestanding, <string.h> may be missing, and we must provide
+ * implementations of memset(), memcpy(), memmove(), and memcmp().
+ * See https://gcc.gnu.org/onlinedocs/gcc/Standards.html
+ *
+ * Also, -ffreestanding disables interpreting calls to these functions as
+ * built-ins. E.g., calling memcpy(&v, p, WORDBYTES) will make a function call,
+ * not be optimized to a single load instruction. For performance reasons we
+ * don't want that. So, declare these functions as macros that expand to the
+ * corresponding built-ins. This approach is recommended in the gcc man page.
+ * We still need the actual function definitions in case gcc calls them.
+ */
+void *memset(void *s, int c, size_t n);
+#define memset(s, c, n) __builtin_memset((s), (c), (n))
+
+void *memcpy(void *dest, const void *src, size_t n);
+#define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
+
+void *memmove(void *dest, const void *src, size_t n);
+#define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
+
+int memcmp(const void *s1, const void *s2, size_t n);
+#define memcmp(s1, s2, n) __builtin_memcmp((s1), (s2), (n))
+
+#undef LIBDEFLATE_ENABLE_ASSERTIONS
+#else
+# include <string.h>
+ /*
+ * To prevent false positive static analyzer warnings, ensure that assertions
+ * are visible to the static analyzer.
+ */
+# ifdef __clang_analyzer__
+# define LIBDEFLATE_ENABLE_ASSERTIONS
+# endif
+#endif
+
+/*
+ * Runtime assertion support. Don't enable this in production builds; it may
+ * hurt performance significantly.
+ */
+#ifdef LIBDEFLATE_ENABLE_ASSERTIONS
+NORETURN void
+libdeflate_assertion_failed(const char *expr, const char *file, int line);
+#define ASSERT(expr) { if (unlikely(!(expr))) \
+ libdeflate_assertion_failed(#expr, __FILE__, __LINE__); }
+#else
+#define ASSERT(expr) (void)(expr)
+#endif
+
+#define CONCAT_IMPL(a, b) a##b
+#define CONCAT(a, b) CONCAT_IMPL(a, b)
+#define ADD_SUFFIX(name) CONCAT(name, SUFFIX)
+
+#endif /* LIB_LIB_COMMON_H */
diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/zlib_constants.h b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/zlib_constants.h
new file mode 100644
index 0000000..f304310
--- /dev/null
+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/zlib_constants.h
@@ -0,0 +1,21 @@
+/*
+ * zlib_constants.h - constants for the zlib wrapper format
+ */
+
+#ifndef LIB_ZLIB_CONSTANTS_H
+#define LIB_ZLIB_CONSTANTS_H
+
+#define ZLIB_MIN_HEADER_SIZE 2
+#define ZLIB_FOOTER_SIZE 4
+#define ZLIB_MIN_OVERHEAD (ZLIB_MIN_HEADER_SIZE + ZLIB_FOOTER_SIZE)
+
+#define ZLIB_CM_DEFLATE 8
+
+#define ZLIB_CINFO_32K_WINDOW 7
+
+#define ZLIB_FASTEST_COMPRESSION 0
+#define ZLIB_FAST_COMPRESSION 1
+#define ZLIB_DEFAULT_COMPRESSION 2
+#define ZLIB_SLOWEST_COMPRESSION 3
+
+#endif /* LIB_ZLIB_CONSTANTS_H */
diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/lib/zlib_decompress.c b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/zlib_decompress.c
new file mode 100644
index 0000000..f5e43ea
--- /dev/null
+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/lib/zlib_decompress.c
@@ -0,0 +1,104 @@
+/*
+ * zlib_decompress.c - decompress with a zlib wrapper
+ *
+ * Copyright 2016 Eric Biggers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "lib_common.h"
+#include "zlib_constants.h"
+
+LIBDEFLATEAPI enum libdeflate_result
+libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *d,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail,
+ size_t *actual_in_nbytes_ret,
+ size_t *actual_out_nbytes_ret)
+{
+ const u8 *in_next = in;
+ const u8 * const in_end = in_next + in_nbytes;
+ u16 hdr;
+ size_t actual_in_nbytes;
+ size_t actual_out_nbytes;
+ enum libdeflate_result result;
+
+ if (in_nbytes < ZLIB_MIN_OVERHEAD)
+ return LIBDEFLATE_BAD_DATA;
+
+ /* 2 byte header: CMF and FLG */
+ hdr = get_unaligned_be16(in_next);
+ in_next += 2;
+
+ /* FCHECK */
+ if ((hdr % 31) != 0)
+ return LIBDEFLATE_BAD_DATA;
+
+ /* CM */
+ if (((hdr >> 8) & 0xF) != ZLIB_CM_DEFLATE)
+ return LIBDEFLATE_BAD_DATA;
+
+ /* CINFO */
+ if ((hdr >> 12) > ZLIB_CINFO_32K_WINDOW)
+ return LIBDEFLATE_BAD_DATA;
+
+ /* FDICT */
+ if ((hdr >> 5) & 1)
+ return LIBDEFLATE_BAD_DATA;
+
+ /* Compressed data */
+ result = libdeflate_deflate_decompress_ex(d, in_next,
+ in_end - ZLIB_FOOTER_SIZE - in_next,
+ out, out_nbytes_avail,
+ &actual_in_nbytes, actual_out_nbytes_ret);
+ if (result != LIBDEFLATE_SUCCESS)
+ return result;
+
+ if (actual_out_nbytes_ret)
+ actual_out_nbytes = *actual_out_nbytes_ret;
+ else
+ actual_out_nbytes = out_nbytes_avail;
+
+ in_next += actual_in_nbytes;
+
+ /* ADLER32 */
+ if (libdeflate_adler32(1, out, actual_out_nbytes) !=
+ get_unaligned_be32(in_next))
+ return LIBDEFLATE_BAD_DATA;
+ in_next += 4;
+
+ if (actual_in_nbytes_ret)
+ *actual_in_nbytes_ret = in_next - (u8 *)in;
+
+ return LIBDEFLATE_SUCCESS;
+}
+
+LIBDEFLATEAPI enum libdeflate_result
+libdeflate_zlib_decompress(struct libdeflate_decompressor *d,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail,
+ size_t *actual_out_nbytes_ret)
+{
+ return libdeflate_zlib_decompress_ex(d, in, in_nbytes,
+ out, out_nbytes_avail,
+ NULL, actual_out_nbytes_ret);
+}
diff --git a/xnu/files/bsd/miscfs/apfs/libdeflate/libdeflate.h b/xnu/files/bsd/miscfs/apfs/libdeflate/libdeflate.h
new file mode 100644
index 0000000..52236da
--- /dev/null
+++ b/xnu/files/bsd/miscfs/apfs/libdeflate/libdeflate.h
@@ -0,0 +1,411 @@
+/*
+ * libdeflate.h - public header for libdeflate
+ */
+
+#ifndef LIBDEFLATE_H
+#define LIBDEFLATE_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define LIBDEFLATE_VERSION_MAJOR 1
+#define LIBDEFLATE_VERSION_MINOR 24
+#define LIBDEFLATE_VERSION_STRING "1.24"
+
+/*
+ * Users of libdeflate.dll on Windows can define LIBDEFLATE_DLL to cause
+ * __declspec(dllimport) to be used. This should be done when it's easy to do.
+ * Otherwise it's fine to skip it, since it is a very minor performance
+ * optimization that is irrelevant for most use cases of libdeflate.
+ */
+#ifndef LIBDEFLATEAPI
+# if defined(LIBDEFLATE_DLL) && (defined(_WIN32) || defined(__CYGWIN__))
+# define LIBDEFLATEAPI __declspec(dllimport)
+# else
+# define LIBDEFLATEAPI
+# endif
+#endif
+
+/* ========================================================================== */
+/* Compression */
+/* ========================================================================== */
+
+struct libdeflate_compressor;
+struct libdeflate_options;
+
+/*
+ * libdeflate_alloc_compressor() allocates a new compressor that supports
+ * DEFLATE, zlib, and gzip compression. 'compression_level' is the compression
+ * level on a zlib-like scale but with a higher maximum value (1 = fastest, 6 =
+ * medium/default, 9 = slow, 12 = slowest). Level 0 is also supported and means
+ * "no compression", specifically "create a valid stream, but only emit
+ * uncompressed blocks" (this will expand the data slightly).
+ *
+ * The return value is a pointer to the new compressor, or NULL if out of memory
+ * or if the compression level is invalid (i.e. outside the range [0, 12]).
+ *
+ * Note: for compression, the sliding window size is defined at compilation time
+ * to 32768, the largest size permissible in the DEFLATE format. It cannot be
+ * changed at runtime.
+ *
+ * A single compressor is not safe to use by multiple threads concurrently.
+ * However, different threads may use different compressors concurrently.
+ */
+LIBDEFLATEAPI struct libdeflate_compressor *
+libdeflate_alloc_compressor(int compression_level);
+
+/*
+ * Like libdeflate_alloc_compressor(), but adds the 'options' argument.
+ */
+LIBDEFLATEAPI struct libdeflate_compressor *
+libdeflate_alloc_compressor_ex(int compression_level,
+ const struct libdeflate_options *options);
+
+/*
+ * libdeflate_deflate_compress() performs raw DEFLATE compression on a buffer of
+ * data. It attempts to compress 'in_nbytes' bytes of data located at 'in' and
+ * write the result to 'out', which has space for 'out_nbytes_avail' bytes. The
+ * return value is the compressed size in bytes, or 0 if the data could not be
+ * compressed to 'out_nbytes_avail' bytes or fewer.
+ *
+ * If compression is successful, then the output data is guaranteed to be a
+ * valid DEFLATE stream that decompresses to the input data. No other
+ * guarantees are made about the output data. Notably, different versions of
+ * libdeflate can produce different compressed data for the same uncompressed
+ * data, even at the same compression level. Do ***NOT*** do things like
+ * writing tests that compare compressed data to a golden output, as this can
+ * break when libdeflate is updated. (This property isn't specific to
+ * libdeflate; the same is true for zlib and other compression libraries too.)
+ */
+LIBDEFLATEAPI size_t
+libdeflate_deflate_compress(struct libdeflate_compressor *compressor,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail);
+
+/*
+ * libdeflate_deflate_compress_bound() returns a worst-case upper bound on the
+ * number of bytes of compressed data that may be produced by compressing any
+ * buffer of length less than or equal to 'in_nbytes' using
+ * libdeflate_deflate_compress() with the specified compressor. This bound will
+ * necessarily be a number greater than or equal to 'in_nbytes'. It may be an
+ * overestimate of the true upper bound. The return value is guaranteed to be
+ * the same for all invocations with the same compressor and same 'in_nbytes'.
+ *
+ * As a special case, 'compressor' may be NULL. This causes the bound to be
+ * taken across *any* libdeflate_compressor that could ever be allocated with
+ * this build of the library, with any options.
+ *
+ * Note that this function is not necessary in many applications. With
+ * block-based compression, it is usually preferable to separately store the
+ * uncompressed size of each block and to store any blocks that did not compress
+ * to less than their original size uncompressed. In that scenario, there is no
+ * need to know the worst-case compressed size, since the maximum number of
+ * bytes of compressed data that may be used would always be one less than the
+ * input length. You can just pass a buffer of that size to
+ * libdeflate_deflate_compress() and store the data uncompressed if
+ * libdeflate_deflate_compress() returns 0, indicating that the compressed data
+ * did not fit into the provided output buffer.
+ */
+LIBDEFLATEAPI size_t
+libdeflate_deflate_compress_bound(struct libdeflate_compressor *compressor,
+ size_t in_nbytes);
+
+/*
+ * Like libdeflate_deflate_compress(), but uses the zlib wrapper format instead
+ * of raw DEFLATE.
+ */
+LIBDEFLATEAPI size_t
+libdeflate_zlib_compress(struct libdeflate_compressor *compressor,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail);
+
+/*
+ * Like libdeflate_deflate_compress_bound(), but assumes the data will be
+ * compressed with libdeflate_zlib_compress() rather than with
+ * libdeflate_deflate_compress().
+ */
+LIBDEFLATEAPI size_t
+libdeflate_zlib_compress_bound(struct libdeflate_compressor *compressor,
+ size_t in_nbytes);
+
+/*
+ * Like libdeflate_deflate_compress(), but uses the gzip wrapper format instead
+ * of raw DEFLATE.
+ */
+LIBDEFLATEAPI size_t
+libdeflate_gzip_compress(struct libdeflate_compressor *compressor,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail);
+
+/*
+ * Like libdeflate_deflate_compress_bound(), but assumes the data will be
+ * compressed with libdeflate_gzip_compress() rather than with
+ * libdeflate_deflate_compress().
+ */
+LIBDEFLATEAPI size_t
+libdeflate_gzip_compress_bound(struct libdeflate_compressor *compressor,
+ size_t in_nbytes);
+
+/*
+ * libdeflate_free_compressor() frees a compressor that was allocated with
+ * libdeflate_alloc_compressor(). If a NULL pointer is passed in, no action is
+ * taken.
+ */
+LIBDEFLATEAPI void
+libdeflate_free_compressor(struct libdeflate_compressor *compressor);
+
+/* ========================================================================== */
+/* Decompression */
+/* ========================================================================== */
+
+struct libdeflate_decompressor;
+struct libdeflate_options;
+
+/*
+ * libdeflate_alloc_decompressor() allocates a new decompressor that can be used
+ * for DEFLATE, zlib, and gzip decompression. The return value is a pointer to
+ * the new decompressor, or NULL if out of memory.
+ *
+ * This function takes no parameters, and the returned decompressor is valid for
+ * decompressing data that was compressed at any compression level and with any
+ * sliding window size.
+ *
+ * A single decompressor is not safe to use by multiple threads concurrently.
+ * However, different threads may use different decompressors concurrently.
+ */
+LIBDEFLATEAPI struct libdeflate_decompressor *
+libdeflate_alloc_decompressor(void);
+
+/*
+ * Like libdeflate_alloc_decompressor(), but adds the 'options' argument.
+ */
+LIBDEFLATEAPI struct libdeflate_decompressor *
+libdeflate_alloc_decompressor_ex(const struct libdeflate_options *options);
+
+/*
+ * Result of a call to libdeflate_deflate_decompress(),
+ * libdeflate_zlib_decompress(), or libdeflate_gzip_decompress().
+ */
+enum libdeflate_result {
+ /* Decompression was successful. */
+ LIBDEFLATE_SUCCESS = 0,
+
+ /* Decompression failed because the compressed data was invalid,
+ * corrupt, or otherwise unsupported. */
+ LIBDEFLATE_BAD_DATA = 1,
+
+ /* A NULL 'actual_out_nbytes_ret' was provided, but the data would have
+ * decompressed to fewer than 'out_nbytes_avail' bytes. */
+ LIBDEFLATE_SHORT_OUTPUT = 2,
+
+ /* The data would have decompressed to more than 'out_nbytes_avail'
+ * bytes. */
+ LIBDEFLATE_INSUFFICIENT_SPACE = 3,
+};
+
+/*
+ * libdeflate_deflate_decompress() decompresses a DEFLATE stream from the buffer
+ * 'in' with compressed size up to 'in_nbytes' bytes. The uncompressed data is
+ * written to 'out', a buffer with size 'out_nbytes_avail' bytes. If
+ * decompression succeeds, then 0 (LIBDEFLATE_SUCCESS) is returned. Otherwise,
+ * a nonzero result code such as LIBDEFLATE_BAD_DATA is returned, and the
+ * contents of the output buffer are undefined.
+ *
+ * Decompression stops at the end of the DEFLATE stream (as indicated by the
+ * BFINAL flag), even if it is actually shorter than 'in_nbytes' bytes.
+ *
+ * libdeflate_deflate_decompress() can be used in cases where the actual
+ * uncompressed size is known (recommended) or unknown (not recommended):
+ *
+ * - If the actual uncompressed size is known, then pass the actual
+ * uncompressed size as 'out_nbytes_avail' and pass NULL for
+ * 'actual_out_nbytes_ret'. This makes libdeflate_deflate_decompress() fail
+ * with LIBDEFLATE_SHORT_OUTPUT if the data decompressed to fewer than the
+ * specified number of bytes.
+ *
+ * - If the actual uncompressed size is unknown, then provide a non-NULL
+ * 'actual_out_nbytes_ret' and provide a buffer with some size
+ * 'out_nbytes_avail' that you think is large enough to hold all the
+ * uncompressed data. In this case, if the data decompresses to less than
+ * or equal to 'out_nbytes_avail' bytes, then
+ * libdeflate_deflate_decompress() will write the actual uncompressed size
+ * to *actual_out_nbytes_ret and return 0 (LIBDEFLATE_SUCCESS). Otherwise,
+ * it will return LIBDEFLATE_INSUFFICIENT_SPACE if the provided buffer was
+ * not large enough but no other problems were encountered, or another
+ * nonzero result code if decompression failed for another reason.
+ */
+LIBDEFLATEAPI enum libdeflate_result
+libdeflate_deflate_decompress(struct libdeflate_decompressor *decompressor,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail,
+ size_t *actual_out_nbytes_ret);
+
+/*
+ * Like libdeflate_deflate_decompress(), but adds the 'actual_in_nbytes_ret'
+ * argument. If decompression succeeds and 'actual_in_nbytes_ret' is not NULL,
+ * then the actual compressed size of the DEFLATE stream (aligned to the next
+ * byte boundary) is written to *actual_in_nbytes_ret.
+ */
+LIBDEFLATEAPI enum libdeflate_result
+libdeflate_deflate_decompress_ex(struct libdeflate_decompressor *decompressor,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail,
+ size_t *actual_in_nbytes_ret,
+ size_t *actual_out_nbytes_ret);
+
+/*
+ * Like libdeflate_deflate_decompress(), but assumes the zlib wrapper format
+ * instead of raw DEFLATE.
+ *
+ * Decompression will stop at the end of the zlib stream, even if it is shorter
+ * than 'in_nbytes'. If you need to know exactly where the zlib stream ended,
+ * use libdeflate_zlib_decompress_ex().
+ */
+LIBDEFLATEAPI enum libdeflate_result
+libdeflate_zlib_decompress(struct libdeflate_decompressor *decompressor,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail,
+ size_t *actual_out_nbytes_ret);
+
+/*
+ * Like libdeflate_zlib_decompress(), but adds the 'actual_in_nbytes_ret'
+ * argument. If 'actual_in_nbytes_ret' is not NULL and the decompression
+ * succeeds (indicating that the first zlib-compressed stream in the input
+ * buffer was decompressed), then the actual number of input bytes consumed is
+ * written to *actual_in_nbytes_ret.
+ */
+LIBDEFLATEAPI enum libdeflate_result
+libdeflate_zlib_decompress_ex(struct libdeflate_decompressor *decompressor,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail,
+ size_t *actual_in_nbytes_ret,
+ size_t *actual_out_nbytes_ret);
+
+/*
+ * Like libdeflate_deflate_decompress(), but assumes the gzip wrapper format
+ * instead of raw DEFLATE.
+ *
+ * If multiple gzip-compressed members are concatenated, then only the first
+ * will be decompressed. Use libdeflate_gzip_decompress_ex() if you need
+ * multi-member support.
+ */
+LIBDEFLATEAPI enum libdeflate_result
+libdeflate_gzip_decompress(struct libdeflate_decompressor *decompressor,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail,
+ size_t *actual_out_nbytes_ret);
+
+/*
+ * Like libdeflate_gzip_decompress(), but adds the 'actual_in_nbytes_ret'
+ * argument. If 'actual_in_nbytes_ret' is not NULL and the decompression
+ * succeeds (indicating that the first gzip-compressed member in the input
+ * buffer was decompressed), then the actual number of input bytes consumed is
+ * written to *actual_in_nbytes_ret.
+ */
+LIBDEFLATEAPI enum libdeflate_result
+libdeflate_gzip_decompress_ex(struct libdeflate_decompressor *decompressor,
+ const void *in, size_t in_nbytes,
+ void *out, size_t out_nbytes_avail,
+ size_t *actual_in_nbytes_ret,
+ size_t *actual_out_nbytes_ret);
+
+/*
+ * libdeflate_free_decompressor() frees a decompressor that was allocated with
+ * libdeflate_alloc_decompressor(). If a NULL pointer is passed in, no action
+ * is taken.
+ */
+LIBDEFLATEAPI void
+libdeflate_free_decompressor(struct libdeflate_decompressor *decompressor);
+
+/* ========================================================================== */
+/* Checksums */
+/* ========================================================================== */
+
+/*
+ * libdeflate_adler32() updates a running Adler-32 checksum with 'len' bytes of
+ * data and returns the updated checksum. When starting a new checksum, the
+ * required initial value for 'adler' is 1. This value is also returned when
+ * 'buffer' is specified as NULL.
+ */
+LIBDEFLATEAPI uint32_t
+libdeflate_adler32(uint32_t adler, const void *buffer, size_t len);
+
+
+/*
+ * libdeflate_crc32() updates a running CRC-32 checksum with 'len' bytes of data
+ * and returns the updated checksum. When starting a new checksum, the required
+ * initial value for 'crc' is 0. This value is also returned when 'buffer' is
+ * specified as NULL.
+ */
+LIBDEFLATEAPI uint32_t
+libdeflate_crc32(uint32_t crc, const void *buffer, size_t len);
+
+/* ========================================================================== */
+/* Custom memory allocator */
+/* ========================================================================== */
+
+/*
+ * Install a custom memory allocator which libdeflate will use for all memory
+ * allocations by default. 'malloc_func' is a function that must behave like
+ * malloc(), and 'free_func' is a function that must behave like free().
+ *
+ * The per-(de)compressor custom memory allocator that can be specified in
+ * 'struct libdeflate_options' takes priority over this.
+ *
+ * This doesn't affect the free() function that will be used to free
+ * (de)compressors that were already in existence when this is called.
+ */
+LIBDEFLATEAPI void
+libdeflate_set_memory_allocator(void *(*malloc_func)(size_t),
+ void (*free_func)(void *));
+
+/*
+ * Advanced options. This is the options structure that
+ * libdeflate_alloc_compressor_ex() and libdeflate_alloc_decompressor_ex()
+ * require. Most users won't need this and should just use the non-"_ex"
+ * functions instead. If you do need this, it should be initialized like this:
+ *
+ * struct libdeflate_options options;
+ *
+ * memset(&options, 0, sizeof(options));
+ * options.sizeof_options = sizeof(options);
+ * // Then set the fields that you need to override the defaults for.
+ */
+struct libdeflate_options {
+
+ /*
+ * This field must be set to the struct size. This field exists for
+ * extensibility, so that fields can be appended to this struct in
+ * future versions of libdeflate while still supporting old binaries.
+ */
+ size_t sizeof_options;
+
+ /*
+ * An optional custom memory allocator to use for this (de)compressor.
+ * 'malloc_func' must be a function that behaves like malloc(), and
+ * 'free_func' must be a function that behaves like free().
+ *
+ * This is useful in cases where a process might have multiple users of
+ * libdeflate who want to use different memory allocators. For example,
+ * a library might want to use libdeflate with a custom memory allocator
+ * without interfering with user code that might use libdeflate too.
+ *
+ * This takes priority over the "global" memory allocator (which by
+ * default is malloc() and free(), but can be changed by
+ * libdeflate_set_memory_allocator()). Moreover, libdeflate will never
+ * call the "global" memory allocator if a per-(de)compressor custom
+ * allocator is always given.
+ */
+ void *(*malloc_func)(size_t);
+ void (*free_func)(void *);
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* LIBDEFLATE_H */