Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

support muse_glimmer int8 attention (#971)

CCYeh597e5e5013cb
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/attention/prepare_int8.rs b/crates/uzu-engine/src/backends/cpu/kernel/attention/prepare_int8.rs
index ad7150121..180a1c1ac 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/attention/prepare_int8.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/attention/prepare_int8.rs
@@ -14,7 +14,7 @@ use crate::{
#[kernel(AttentionPrepareInt8)]
#[variants(ElementT, bf16)]
#[variants(RopeT, f32)]
-#[variants(HEAD_DIM, 256)]
+#[variants(HEAD_DIM, 128, 256)]
pub fn attention_prepare_int8<ElementT: DataTypeRepr + Float, RopeT: DataTypeRepr + Float, const HEAD_DIM: u32>(
qkvg: *const ElementT,
#[optional(!quantize_queries)] queries: Option<*mut ElementT>,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention_int8.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention_int8.metal
index 8abe1be58..8fe859f61 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention_int8.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/attention_int8.metal
@@ -19,7 +19,7 @@ struct Int8Scratch {
UZU_CONST int STATS_OFFSET = Layout::SIMDGROUPS * Layout::SCORE_TILE_ELEMENTS / int(sizeof(float));
UZU_CONST int SUMS_OFFSET = STATS_OFFSET + Layout::SIMDGROUPS * Layout::TILE_ROWS * 2;
UZU_CONST int SLOT_FLOATS = SUMS_OFFSET + Layout::SIMDGROUPS * Layout::TILE_ROWS;
- UZU_CONST int TRIE_MASKS = Layout::BLOCK_ROWS;
+ UZU_CONST int ROW_MASKS = Layout::BLOCK_ROWS;
UZU_CONST int QUERY_VECTORS_PER_BAND = int(HEAD_DIM) / (2 * uzu::kv_layout::TILE_DIMS) * METAL_SIMD_SIZE;
UZU_CONST int QUERY_VECTORS = Layout::BANDS * QUERY_VECTORS_PER_BAND;
@@ -114,7 +114,7 @@ struct Int8Softmax {
template <typename ScaleType, uint HEAD_DIM>
VARIANTS(ScaleType, bfloat)
-VARIANTS(HEAD_DIM, 256)
+VARIANTS(HEAD_DIM, 128, 256)
KERNEL(AttentionGemmGroupedInt8)(
const device int8_t* query_codes,
const device ScaleType* query_scales,
@@ -136,7 +136,7 @@ KERNEL(AttentionGemmGroupedInt8)(
const bool is_trie SPECIALIZE,
const uint32_t num_kv_heads SPECIALIZE,
threadgroup float probability_exchange[2 * Int8Scratch<HEAD_DIM>::SLOT_FLOATS],
- threadgroup ulong trie_masks[Int8Scratch<HEAD_DIM>::TRIE_MASKS],
+ threadgroup ulong row_masks[Int8Scratch<HEAD_DIM>::ROW_MASKS],
threadgroup uint4 query_stage[Int8Scratch<HEAD_DIM>::QUERY_VECTORS],
const ThreadContext thread_context,
const uint tile_split_index GROUPS(query_tiles * split_count),
@@ -199,30 +199,30 @@ KERNEL(AttentionGemmGroupedInt8)(
uint(score_position.y) + uint(Ops::THREAD_ELEMENT_ROW_STRIDE)
};
const uint query_band_offset = uint(query_band) * uint(Layout::TILE_ROWS);
- uint query_positions[ROWS_PER_LANE];
if (!mask_free) {
+ ulong row_mask[ROWS_PER_LANE];
+ uint query_positions[ROWS_PER_LANE];
METAL_PRAGMA_UNROLL
for (ushort row = 0; row < ROWS_PER_LANE; ++row) {
query_positions[row] = (query_row_base + lane_rows[row]) % params.q_len;
+ row_mask[row] = is_trie ? 0ul : ulong(prefix_length + query_positions[row]);
}
if (is_trie) {
- ulong row_trie_mask[ROWS_PER_LANE] = {};
for (uint node_index = 0; node_index < trie_nodes; ++node_index) {
const uzu::trie::TrieNode node = trie[node_index];
METAL_PRAGMA_UNROLL
for (ushort row = 0; row < ROWS_PER_LANE; ++row) {
if (query_positions[row] >= node.trie_start && query_positions[row] <= node.trie_end) {
- row_trie_mask[row] |= (1ul << node_index);
+ row_mask[row] |= (1ul << node_index);
}
}
}
- if (score_position.x == 0) {
- METAL_PRAGMA_UNROLL
- for (ushort row = 0; row < ROWS_PER_LANE; ++row) {
- trie_masks[query_band_offset + lane_rows[row]] = row_trie_mask[row];
- }
+ }
+ if (score_position.x == 0) {
+ METAL_PRAGMA_UNROLL
+ for (ushort row = 0; row < ROWS_PER_LANE; ++row) {
+ row_masks[query_band_offset + lane_rows[row]] = row_mask[row];
}
- threadgroup_barrier(mem_flags::mem_threadgroup);
}
}
@@ -348,21 +348,19 @@ KERNEL(AttentionGemmGroupedInt8)(
);
if (!mask_free && (is_tail_block || (key_base + uint(uzu::kv_layout::TILE_KEYS) > prefix_length))) {
- ulong trie_row_masks[ROWS_PER_LANE] = {0ul, 0ul};
- if (is_trie) {
- METAL_PRAGMA_UNROLL
- for (ushort row = 0; row < ROWS_PER_LANE; ++row) {
- trie_row_masks[row] = trie_masks[query_band_offset + lane_rows[row]];
- }
+ ulong row_mask[ROWS_PER_LANE];
+ METAL_PRAGMA_UNROLL
+ for (ushort row = 0; row < ROWS_PER_LANE; ++row) {
+ row_mask[row] = row_masks[query_band_offset + lane_rows[row]];
}
map_lane_rows(scores, score_position, [&](ushort lane_row, short column, AccumType value) UZU_ALWAYS_INLINE {
const uint key = key_base + uint(column);
bool visible = key < key_length;
if (is_trie) {
- visible = visible &&
- (key < prefix_length || ((trie_row_masks[lane_row] >> ulong(key - prefix_length)) & 1ul) != 0ul);
+ visible =
+ visible && (key < prefix_length || ((row_mask[lane_row] >> ulong(key - prefix_length)) & 1ul) != 0ul);
} else {
- visible = visible && key <= prefix_length + query_positions[lane_row];
+ visible = visible && key <= uint(row_mask[lane_row]);
}
return visible ? value : -metal::numeric_limits<AccumType>::infinity();
});
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/combine_int8.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/combine_int8.metal
index 7259b82d6..a97562335 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/combine_int8.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/combine_int8.metal
@@ -9,7 +9,7 @@ using namespace metal;
template <typename T, uint BD>
VARIANTS(T, bfloat)
-VARIANTS(BD, 256)
+VARIANTS(BD, 128, 256)
KERNEL(AttentionGemmGroupedInt8Combine)(
const device bfloat* partials,
const device float* partial_maxs,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs
index 25ccfad65..c922132e4 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/mod.rs
@@ -29,6 +29,10 @@ const SLICE_COLS: u32 = 64;
const TILE_ROWS: u32 = 16;
const BLOCK_K: u32 = 32;
+pub fn query_rows_per_threadgroup(head_dim: u32) -> u32 {
+ SIMDGROUPS_PER_THREADGROUP / (head_dim / SLICE_COLS) * TILE_ROWS
+}
+
struct AttentionGemmGroupedMetal {
kernel: AttentionGemmGroupedMetalKernel,
split_kernel: AttentionGemmGroupedMetalKernel,
@@ -38,7 +42,7 @@ struct AttentionGemmGroupedMetal {
num_q_heads: u32,
scale: Option<f32>,
gate_row_stride: Option<u32>,
- block_rows: u32,
+ query_rows_per_threadgroup: u32,
mask: MaskKind,
}
@@ -55,7 +59,6 @@ impl AttentionGemmGroupedMetal {
assert!(matches!(head_dim, 128 | 256), "head_dim must be 128 or 256");
assert_eq!(num_q_heads % num_groups, 0, "num_q_heads must be divisible by num_groups");
let slices = head_dim / SLICE_COLS;
- let block_rows = SIMDGROUPS_PER_THREADGROUP / slices * TILE_ROWS;
let new_kernel = |split| {
AttentionGemmGroupedMetalKernel::new(
context,
@@ -79,7 +82,7 @@ impl AttentionGemmGroupedMetal {
num_q_heads,
scale,
gate_row_stride,
- block_rows,
+ query_rows_per_threadgroup: query_rows_per_threadgroup(head_dim),
mask,
})
}
@@ -99,7 +102,7 @@ impl AttentionGemmGroupedMetal {
) -> Result<bool, MetalError> {
let params = self.params(suffix_length, kv_length);
let grouped_rows = self.num_q_heads / self.num_groups * suffix_length;
- let query_tiles = grouped_rows.div_ceil(self.block_rows);
+ let query_tiles = grouped_rows.div_ceil(self.query_rows_per_threadgroup);
assert_eq!(trie.is_some(), self.mask.is_trie(), "trie presence must match mask");
if num_splits == 1 {
@@ -121,7 +124,7 @@ impl AttentionGemmGroupedMetal {
return Ok(false);
}
- let partial_rows = self.num_groups * query_tiles * num_splits * self.block_rows;
+ let partial_rows = self.num_groups * query_tiles * num_splits * self.query_rows_per_threadgroup;
let mut partials = command_buffer.allocate_scratch_for_shape(&[partial_rows, self.head_dim], DataType::F32)?;
let mut partial_maxs = command_buffer.allocate_scratch_for_shape(&[partial_rows], DataType::F32)?;
let mut partial_sums = command_buffer.allocate_scratch_for_shape(&[partial_rows], DataType::F32)?;
@@ -150,7 +153,7 @@ impl AttentionGemmGroupedMetal {
gate,
params,
grouped_rows,
- query_tiles * self.block_rows,
+ query_tiles * self.query_rows_per_threadgroup,
self.num_groups,
num_splits,
self.gate_row_stride,
@@ -277,7 +280,7 @@ impl AttentionGemmGrouped {
head_dim: core.head_dim,
num_q_heads: core.num_q_heads,
num_groups: core.num_groups,
- block_rows: core.block_rows,
+ query_rows_per_threadgroup: core.query_rows_per_threadgroup,
block_k: BLOCK_K,
},
suffix_length,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs
index 58b76a6a8..4762d687f 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/gemm_grouped/policy.rs
@@ -78,7 +78,7 @@ pub struct SplitGeometry {
pub head_dim: u32,
pub num_q_heads: u32,
pub num_groups: u32,
- pub block_rows: u32,
+ pub query_rows_per_threadgroup: u32,
pub block_k: u32,
}
@@ -123,14 +123,14 @@ pub fn choose_splits(
gpu_core_count: u32,
) -> u32 {
let threadgroups_per_kv_split = (geometry.num_q_heads / geometry.num_groups * suffix_length)
- .div_ceil(geometry.block_rows)
+ .div_ceil(geometry.query_rows_per_threadgroup)
* geometry.num_groups;
let max_splits = kv_length.div_ceil(geometry.block_k).max(1);
if suffix_length > GEMM_GROUPED_DECODE_SUFFIX_MAX && kv_length >= PREFILL_SPLIT_MIN_KV {
// Each padded row has head_dim partial values, one max, and one sum, all F32.
let bytes_per_split = threadgroups_per_kv_split as u64
- * geometry.block_rows as u64
+ * geometry.query_rows_per_threadgroup as u64
* (geometry.head_dim as u64 + PARTIAL_STATS_PER_ROW)
* DataType::F32.size_in_bytes() as u64;
let requested_splits = kv_length.div_ceil(PREFILL_SPLIT_KEYS).min(max_splits);
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/int8/mod.rs b/crates/uzu-engine/src/backends/metal/kernel/attention/int8/mod.rs
index e3f731421..47427026d 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/int8/mod.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/int8/mod.rs
@@ -1,4 +1,7 @@
-use super::gemm_grouped::policy::{MAX_TRIE_SUFFIX, int8_splits};
+use super::gemm_grouped::{
+ policy::{self, MAX_TRIE_SUFFIX, int8_splits},
+ query_rows_per_threadgroup,
+};
use crate::{
backends::{
common::{
@@ -18,8 +21,6 @@ use crate::{
data_type::DataType,
};
-const QUERIES_PER_BLOCK: u32 = 2 * kv_layout::TILE_QUERIES;
-
pub struct Int8Attention {
config: AttentionKernelConfig,
mask_free: AttentionGemmGroupedInt8MetalKernel,
@@ -33,12 +34,7 @@ impl Int8Attention {
config: &AttentionKernelConfig,
context: &MetalContext,
) -> bool {
- context.supports_mxu
- && config.head_dim == 256
- && config.is_causal
- && config.data_type == DataType::BF16
- && !config.has_sinks
- && config.sliding_window_size.is_none()
+ policy::is_supported(config, context) && config.is_causal
}
pub fn new(
@@ -94,8 +90,9 @@ impl Int8Attention {
let gqa_factor = num_q_heads / num_groups;
let grouped_rows = gqa_factor * suffix_length;
- let query_tiles = grouped_rows.div_ceil(QUERIES_PER_BLOCK);
- let split_rows = num_groups * query_tiles * QUERIES_PER_BLOCK;
+ let query_rows_per_threadgroup = query_rows_per_threadgroup(head_dim);
+ let query_tiles = grouped_rows.div_ceil(query_rows_per_threadgroup);
+ let split_rows = num_groups * query_tiles * query_rows_per_threadgroup;
let num_splits = int8_splits(split_rows, head_dim, suffix_length, kv_length);
let rows = split_rows * num_splits;
@@ -160,7 +157,7 @@ impl Int8Attention {
gate,
params,
grouped_rows,
- query_tiles * QUERIES_PER_BLOCK,
+ query_tiles * query_rows_per_threadgroup,
num_groups,
num_splits,
config.gate_row_stride,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/attention/prepare_int8.metal b/crates/uzu-engine/src/backends/metal/kernel/attention/prepare_int8.metal
index 26724143f..f26252997 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/attention/prepare_int8.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/attention/prepare_int8.metal
@@ -29,7 +29,7 @@ METAL_FUNC ElementT kv_quantize_row(const thread float* values, thread int* code
template <typename ElementT, typename RopeT, uint HEAD_DIM>
VARIANTS(ElementT, bfloat)
VARIANTS(RopeT, float)
-VARIANTS(HEAD_DIM, 256)
+VARIANTS(HEAD_DIM, 128, 256)
PUBLIC KERNEL(AttentionPrepareInt8)(
const device ElementT* qkvg,
device ElementT* queries OPTIONAL(!quantize_queries),
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs
index 1ca7955e6..a66352667 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/gemm_grouped_policy_test.rs
@@ -3,7 +3,7 @@ use uzu_engine_macros::uzu_test;
use super::{MaskKind, SplitGeometry, choose_splits, int8_splits, should_encode};
fn geometry(head_dim: u32) -> SplitGeometry {
- let (num_q_heads, block_rows, num_groups) = if head_dim == 128 {
+ let (num_q_heads, query_rows_per_threadgroup, num_groups) = if head_dim == 128 {
(32, 64, 8)
} else {
(24, 32, 4)
@@ -12,7 +12,7 @@ fn geometry(head_dim: u32) -> SplitGeometry {
head_dim,
num_q_heads,
num_groups,
- block_rows,
+ query_rows_per_threadgroup,
block_k: 32,
}
}
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
index 627a1efdd..b736e5e54 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/attention/kernel_test.rs
@@ -272,8 +272,6 @@ fn attention_gate_matches_standalone_gate_on_metal() {
#[uzu_test]
fn int8_attention_matches_cpu() {
- const DIM: usize = 256;
- const KV_HEADS: usize = 4;
let (metal, cpu) = (<Metal as Backend>::Context::new().unwrap(), <Cpu as Backend>::Context::new().unwrap());
type Prepare = <<Metal as Backend>::Kernels as Kernels>::AttentionPrepareInt8Kernel;
let trie: [GpuTrieNode; 3] = bytemuck::cast([0u32, 2, 0, 1, 1, 1, 2, 2, 1]); // (start, end, height) per token
@@ -287,28 +285,43 @@ fn int8_attention_matches_cpu() {
(24, 3, 16, true, true, NAN),
(24, 8, 32, false, false, DENORMAL),
(16, 3, 37, true, false, DENORMAL),
+ (32, 1, 1024, false, false, NAN),
+ (32, 1024, 24, false, false, bf16::MAX.to_bits()),
+ (32, 3, 1024, true, false, bf16::INFINITY.to_bits()),
] {
- let (shape, nodes) = ((DIM, q_heads, KV_HEADS, suffix, prefix, true), use_trie.then_some(trie.as_slice()));
- let config = super::default_attention_config::<bf16>(DIM, q_heads, KV_HEADS, true);
+ let (head_dim, kv_heads) = if q_heads == 32 {
+ (128, 2)
+ } else {
+ (256, 4)
+ };
+ let (shape, nodes) = ((head_dim, q_heads, kv_heads, suffix, prefix, true), use_trie.then_some(trie.as_slice()));
+ let config = super::default_attention_config::<bf16>(head_dim, q_heads, kv_heads, true);
let kernel = <<Metal as Backend>::Kernels as Kernels>::AttentionKernel::new(&metal, config).unwrap();
if !kernel.supports_int8() {
return;
}
// Fused QKV rows over the data `run_attention` uses; its queries are head-major.
- let ((keys, values), stride) = (attention_data(shape), (q_heads + 2 * KV_HEADS) * DIM);
+ let ((keys, values), stride) = (attention_data(shape), (q_heads + 2 * kv_heads) * head_dim);
let mut rows = vec![bf16::ZERO; (prefix + suffix) * stride];
- for (row, (k, v)) in rows.chunks_mut(stride).zip(keys.chunks(KV_HEADS * DIM).zip(values.chunks(KV_HEADS * DIM)))
+ for (row, (k, v)) in
+ rows.chunks_mut(stride).zip(keys.chunks(kv_heads * head_dim).zip(values.chunks(kv_heads * head_dim)))
{
- let (key_row, value_row) = row[q_heads * DIM..].split_at_mut(KV_HEADS * DIM);
+ let (key_row, value_row) = row[q_heads * head_dim..].split_at_mut(kv_heads * head_dim);
key_row.copy_from_slice(k);
value_row.copy_from_slice(v);
}
- for (index, query) in fill_attention(q_heads * suffix * DIM, 0.5).chunks(DIM).enumerate() {
- rows[(prefix + index % suffix) * stride + index / suffix * DIM..][..DIM].copy_from_slice(query);
+ for (index, query) in fill_attention(q_heads * suffix * head_dim, 0.5).chunks(head_dim).enumerate() {
+ rows[(prefix + index % suffix) * stride + index / suffix * head_dim..][..head_dim].copy_from_slice(query);
}
- let mut cache =
- KVStorage::<Metal>::create_empty(KVFormat::INT8, KV_HEADS as u32, DIM as u32, 2048, DataType::BF16, &metal)
- .unwrap();
+ let mut cache = KVStorage::<Metal>::create_empty(
+ KVFormat::INT8,
+ kv_heads as u32,
+ head_dim as u32,
+ 2048,
+ DataType::BF16,
+ &metal,
+ )
+ .unwrap();
cache.prepare(2048).unwrap();
let KVStorage::Int8(kv, _) = &mut cache else {
unreachable!()
@@ -325,20 +338,20 @@ fn int8_attention_matches_cpu() {
submit_command_buffer(fill);
let mut step = |rows: &[bf16], prefix: usize, trie: Option<&[GpuTrieNode]>, append_kv: bool| {
let suffix = rows.len() / stride;
- let query_rows = q_heads / KV_HEADS * suffix;
+ let query_rows = q_heads / kv_heads * suffix;
let prepare = Prepare::new(
&metal,
DataType::BF16,
DataType::F32,
- DIM as u32,
+ head_dim as u32,
false,
true,
- KV_HEADS as u32,
+ kv_heads as u32,
append_kv,
)
.unwrap();
- let mut query_codes = create_buffer::<Metal, i8>(&metal, KV_HEADS * query_rows * DIM);
- let mut query_scales = create_buffer::<Metal, bf16>(&metal, KV_HEADS * query_rows);
+ let mut query_codes = create_buffer::<Metal, i8>(&metal, kv_heads * query_rows * head_dim);
+ let mut query_scales = create_buffer::<Metal, bf16>(&metal, kv_heads * query_rows);
let mut command_buffer = metal.create_command_buffer(None, None).unwrap();
let trie = trie.map(|nodes| command_buffer.allocate_constant_from_slice(nodes).unwrap());
prepare.encode(
@@ -376,7 +389,7 @@ fn int8_attention_matches_cpu() {
gate: None::<&<Metal as Backend>::GlobalBuffer>,
};
let pooled = kernel.encode(input, arguments, &mut command_buffer).unwrap();
- let mut output = create_buffer::<Metal, bf16>(&metal, suffix * q_heads * DIM);
+ let mut output = create_buffer::<Metal, bf16>(&metal, suffix * q_heads * head_dim);
command_buffer.encode_copy(&pooled, &mut output);
drop((pooled, trie));
submit_command_buffer(command_buffer);
@@ -386,7 +399,7 @@ fn int8_attention_matches_cpu() {
let mut actual = step(&rows[prefix * stride..], prefix, nodes, true);
if reader {
// The reader's own K/V rows are junk: appending them would change the output.
- rows[prefix * stride..].chunks_mut(stride).for_each(|row| row[q_heads * DIM..].fill(bf16::ONE));
+ rows[prefix * stride..].chunks_mut(stride).for_each(|row| row[q_heads * head_dim..].fill(bf16::ONE));
actual = step(&rows[prefix * stride..], prefix, nodes, false);
}
let label = format!("int8 attention q{q_heads} S{suffix} L{prefix} trie={use_trie} reader={reader}");