Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

docs(examples): align recipes with recommended W2 and batch size settings (#2499) (#2511)

Vishva Ra0134a4ca807
diff --git a/examples/Cloudflare-clef/clef-flash_W4A16.ipynb b/examples/Cloudflare-clef/clef-flash_W4A16.ipynb
index 8b9705543..2a8905bdb 100644
--- a/examples/Cloudflare-clef/clef-flash_W4A16.ipynb
+++ b/examples/Cloudflare-clef/clef-flash_W4A16.ipynb
@@ -163,6 +163,7 @@
" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
" nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
" batch_size=2, # Batch size for calibration; kept < 8 to avoid slow execution/high memory overhead\n",
+ " gradient_accumulate_steps=4, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n",
" seqlen=2048, # Calibration context length; captures long-range dependency activations\n",
" low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
" quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",
diff --git a/examples/Cloudflare-clef/clef_W4A16.ipynb b/examples/Cloudflare-clef/clef_W4A16.ipynb
index fcfa9e6a4..e355413ed 100644
--- a/examples/Cloudflare-clef/clef_W4A16.ipynb
+++ b/examples/Cloudflare-clef/clef_W4A16.ipynb
@@ -163,6 +163,7 @@
" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
" nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
" batch_size=2, # Batch size for calibration; kept < 8 to avoid slow execution/high memory overhead\n",
+ " gradient_accumulate_steps=4, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n",
" seqlen=2048, # Calibration context length; captures long-range dependency activations\n",
" low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
" quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",
diff --git a/examples/DotsOCR/DotsmOCR_W4A16.ipynb b/examples/DotsOCR/DotsmOCR_W4A16.ipynb
index 275b0cd60..dcc7ba9e8 100644
--- a/examples/DotsOCR/DotsmOCR_W4A16.ipynb
+++ b/examples/DotsOCR/DotsmOCR_W4A16.ipynb
@@ -143,6 +143,7 @@
" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
" nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
" batch_size=4, # Batch size for calibration; kept < 8 to avoid slow execution/high memory overhead\n",
+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n",
" seqlen=4096, # Calibration context length; captures long-range dependency activations\n",
" low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
" enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",
diff --git a/examples/LightOnOCR/LightOnOCR-3-0.8B_W4A16.ipynb b/examples/LightOnOCR/LightOnOCR-3-0.8B_W4A16.ipynb
index 70c786d8b..9f944f6b9 100644
--- a/examples/LightOnOCR/LightOnOCR-3-0.8B_W4A16.ipynb
+++ b/examples/LightOnOCR/LightOnOCR-3-0.8B_W4A16.ipynb
@@ -163,6 +163,7 @@
" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
" nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
" batch_size=4, # Batch size for calibration; kept < 8 to avoid slow execution/high memory overhead\n",
+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n",
" seqlen=4096, # Calibration context length; captures long-range dependency activations\n",
" low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
" enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",
diff --git a/examples/LightOnOCR/LightOnOCR-3-4B_W4A16.ipynb b/examples/LightOnOCR/LightOnOCR-3-4B_W4A16.ipynb
index 5ab1885a8..b5863c09e 100644
--- a/examples/LightOnOCR/LightOnOCR-3-4B_W4A16.ipynb
+++ b/examples/LightOnOCR/LightOnOCR-3-4B_W4A16.ipynb
@@ -163,6 +163,7 @@
" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
" nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
" batch_size=4, # Batch size for calibration; kept < 8 to avoid slow execution/high memory overhead\n",
+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n",
" seqlen=4096, # Calibration context length; captures long-range dependency activations\n",
" low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
" enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",
diff --git a/examples/LiquidAI/LiquidAId3B_W4A16.ipynb b/examples/LiquidAI/LiquidAId3B_W4A16.ipynb
index cabfa2e15..85e44c4b2 100644
--- a/examples/LiquidAI/LiquidAId3B_W4A16.ipynb
+++ b/examples/LiquidAI/LiquidAId3B_W4A16.ipynb
@@ -144,6 +144,7 @@
" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
" nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
" batch_size=2, # Batch size for calibration; kept < 8 to avoid slow execution/high memory overhead\n",
+ " gradient_accumulate_steps=4, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n",
" seqlen=2048, # Calibration context length; captures long-range dependency activations\n",
" low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
" enable_torch_compile=False, # Compiles forward passes using torch.compile to accelerate calibration\n",
diff --git a/examples/Muse-Glimmer/Muse-Glimmer_W2A16.ipynb b/examples/Muse-Glimmer/Muse-Glimmer_W2A16.ipynb
index 5600fb801..d389ec8c9 100644
--- a/examples/Muse-Glimmer/Muse-Glimmer_W2A16.ipynb
+++ b/examples/Muse-Glimmer/Muse-Glimmer_W2A16.ipynb
@@ -118,9 +118,11 @@
"source": [
"# Configure optimization parameters for the AutoRound / SignRound algorithm\n",
"alg_config = SignRoundConfig(\n",
- " iters=800, # Optimization iterations per block; high accuracy for production grade\n",
+ " iters=1000, # Optimization iterations per block (steps ~= 1000 recommended for W2)\n",
" group_size=64, # Number of weights sharing one scale. Cannot go below 32 (vLLM's GPTQ kernel won't run it). Only use 16 if targeting NVFP4.\n",
" sym=True, # Symmetric quantization; enables faster compute and broader engine compatibility\n",
+ " enable_alg_ext=True, # Algorithm extension for extreme low-bit (W2A16) precision boost\n",
+ " lr=2 / 1000, # lr = 2 / steps (~0.002); prevents gradient divergence during 2-bit optimization\n",
")\n",
"\n",
"# Initialize the AutoRound quantization pipeline\n",
@@ -132,6 +134,7 @@
" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
" nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
" batch_size=4, # Batch size for calibration; reduced to manage GPU memory usage on 48GB VRAM\n",
+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n",
" seqlen=2048, # Calibration context length; captures long-range dependency activations\n",
" low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
" enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",
diff --git a/examples/Muse-Glimmer/Muse-Glimmer_W4A16.ipynb b/examples/Muse-Glimmer/Muse-Glimmer_W4A16.ipynb
index fd28d1176..d9dd3e1fd 100644
--- a/examples/Muse-Glimmer/Muse-Glimmer_W4A16.ipynb
+++ b/examples/Muse-Glimmer/Muse-Glimmer_W4A16.ipynb
@@ -132,6 +132,7 @@
" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
" nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
" batch_size=4, # Batch size for calibration; reduced to manage GPU memory usage on 48GB VRAM\n",
+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n",
" seqlen=2048, # Calibration context length; captures long-range dependency activations\n",
" low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
" enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",
diff --git a/examples/NuExtract3/NuExtract3_W4A16.ipynb b/examples/NuExtract3/NuExtract3_W4A16.ipynb
index 032a43106..3c4d112af 100644
--- a/examples/NuExtract3/NuExtract3_W4A16.ipynb
+++ b/examples/NuExtract3/NuExtract3_W4A16.ipynb
@@ -162,6 +162,7 @@
" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
" nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
" batch_size=4, # Batch size for calibration; reduced to manage GPU memory usage on 48GB VRAM\n",
+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n",
" seqlen=4096, # Calibration context length; captures long-range dependency activations\n",
" low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
" enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",
diff --git a/examples/Qwen3.5/Qwen3.5-0.8B_NVFP4.ipynb b/examples/Qwen3.5/Qwen3.5-0.8B_NVFP4.ipynb
index 12baac191..c5107793a 100644
--- a/examples/Qwen3.5/Qwen3.5-0.8B_NVFP4.ipynb
+++ b/examples/Qwen3.5/Qwen3.5-0.8B_NVFP4.ipynb
@@ -164,6 +164,7 @@
" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
" nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
" batch_size=4, # Calibration batch size; balances processing speed and GPU memory usage\n",
+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n",
" seqlen=2048, # Calibration context length; captures long-range dependency activations\n",
" low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
" quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",
diff --git a/examples/Qwen3.5/Qwen3.5-2B_NVFP4.ipynb b/examples/Qwen3.5/Qwen3.5-2B_NVFP4.ipynb
index 84b796c8c..4886741d5 100644
--- a/examples/Qwen3.5/Qwen3.5-2B_NVFP4.ipynb
+++ b/examples/Qwen3.5/Qwen3.5-2B_NVFP4.ipynb
@@ -164,6 +164,7 @@
" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
" nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
" batch_size=4, # Calibration batch size; balances processing speed and GPU memory usage\n",
+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n",
" seqlen=2048, # Calibration context length; captures long-range dependency activations\n",
" low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
" quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",
diff --git a/examples/Qwen3.5/Qwen3.5-4B_NVFP4.ipynb b/examples/Qwen3.5/Qwen3.5-4B_NVFP4.ipynb
index ca2123e9b..aa3c4870d 100644
--- a/examples/Qwen3.5/Qwen3.5-4B_NVFP4.ipynb
+++ b/examples/Qwen3.5/Qwen3.5-4B_NVFP4.ipynb
@@ -164,6 +164,7 @@
" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
" nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
" batch_size=4, # Calibration batch size; balances processing speed and GPU memory usage\n",
+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n",
" seqlen=2048, # Calibration context length; captures long-range dependency activations\n",
" low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
" quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",
diff --git a/examples/Qwen3.5/Qwen3.5-9B_NVFP4.ipynb b/examples/Qwen3.5/Qwen3.5-9B_NVFP4.ipynb
index ef9dcdd1b..a5241cdaf 100644
--- a/examples/Qwen3.5/Qwen3.5-9B_NVFP4.ipynb
+++ b/examples/Qwen3.5/Qwen3.5-9B_NVFP4.ipynb
@@ -164,6 +164,7 @@
" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
" nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
" batch_size=4, # Calibration batch size; balances processing speed and GPU memory usage\n",
+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n",
" seqlen=2048, # Calibration context length; captures long-range dependency activations\n",
" low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
" quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",
diff --git a/examples/Qwen3.5/Qwen3_5_0.8B_W4A16.ipynb b/examples/Qwen3.5/Qwen3_5_0.8B_W4A16.ipynb
index c7d54d82c..9cf29a703 100644
--- a/examples/Qwen3.5/Qwen3_5_0.8B_W4A16.ipynb
+++ b/examples/Qwen3.5/Qwen3_5_0.8B_W4A16.ipynb
@@ -153,6 +153,7 @@
" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
" nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
" batch_size=2, # Batch size for calibration; reduced to manage GPU memory usage on 48GB VRAM\n",
+ " gradient_accumulate_steps=4, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n",
" seqlen=2048, # Calibration context length; captures long-range dependency activations\n",
" low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
" enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",
diff --git a/examples/Qwen3.5/Qwen3_5_4B_W2A16.ipynb b/examples/Qwen3.5/Qwen3_5_4B_W2A16.ipynb
index f22b28de0..c179b8714 100644
--- a/examples/Qwen3.5/Qwen3_5_4B_W2A16.ipynb
+++ b/examples/Qwen3.5/Qwen3_5_4B_W2A16.ipynb
@@ -138,9 +138,11 @@
"\n",
"# Configure optimization parameters for the AutoRound / SignRound algorithm\n",
"alg_config = SignRoundConfig(\n",
- " iters=1000, # Optimization iterations per block; balances calibration time and accuracy\n",
- " group_size=32, # Number of weights sharing one scale. Cannot go below 32 (vLLM's GPTQ kernel won't run it). Only use 16 if targeting NVFP4.\n",
- " sym=True, # Symmetric quantization; enables faster compute and broader engine compatibility\n",
+ " iters=1000, # Optimization iterations per block (steps ~= 1000 recommended for W2)\n",
+ " group_size=64, # Number of weights sharing one scale. Cannot go below 32 (vLLM's GPTQ kernel won't run it). Only use 16 if targeting NVFP4.\n",
+ " sym=True, # Symmetric quantization; enables faster compute and broader engine compatibility\n",
+ " enable_alg_ext=True, # Algorithm extension for extreme low-bit (W2A16) precision boost\n",
+ " lr=2 / 1000, # lr = 2 / steps (~0.002); prevents gradient divergence during 2-bit optimization\n",
")\n",
"\n",
"# Initialize the AutoRound quantization pipeline\n",
@@ -152,6 +154,7 @@
" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
" nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
" batch_size=2, # Batch size for calibration; balances speed and GPU memory usage\n",
+ " gradient_accumulate_steps=4, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n",
" seqlen=2048, # Calibration context length; captures long-range dependency activations\n",
" low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
" enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",
diff --git a/examples/Qwen3.5/Qwen3_5_4B_W4A16.ipynb b/examples/Qwen3.5/Qwen3_5_4B_W4A16.ipynb
index 7358fee3e..e51e9e85f 100644
--- a/examples/Qwen3.5/Qwen3_5_4B_W4A16.ipynb
+++ b/examples/Qwen3.5/Qwen3_5_4B_W4A16.ipynb
@@ -162,6 +162,7 @@
" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
" nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
" batch_size=2, # Batch size for calibration; reduced to manage GPU memory usage on 48GB VRAM\n",
+ " gradient_accumulate_steps=4, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n",
" seqlen=2048, # Calibration context length; captures long-range dependency activations\n",
" low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
" enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",
diff --git a/examples/Qwen3.5/Qwen3_5_9B_W4A16.ipynb b/examples/Qwen3.5/Qwen3_5_9B_W4A16.ipynb
index 6f0f33765..6b9ca991c 100644
--- a/examples/Qwen3.5/Qwen3_5_9B_W4A16.ipynb
+++ b/examples/Qwen3.5/Qwen3_5_9B_W4A16.ipynb
@@ -139,30 +139,6 @@
"tokenizer = processor.tokenizer\n"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "7c1063ff",
- "metadata": {},
- "outputs": [],
- "source": [
- "TUNING_CONFIG = {\n",
- " \"group_size\": 32,\n",
- " \"sym\": True,\n",
- " \"iters\": 1200, # High accuracy (Production grade)\n",
- " \"nsamples\": 512, # More calibration data\n",
- " \"batch_size\": 4,\n",
- " \"seqlen\": 4096,\n",
- " \"low_gpu_mem_usage\": False, # Keep on GPU for speed\n",
- " \"enable_torch_compile\": True, # JIT acceleration\n",
- " \"quant_nontext_module\": False, # Keep Vision Tower in FP16 (Crucial for VLM accuracy)\n",
- " \"layer_config\": {\n",
- " \"mtp\": {\"data_type\": \"bfloat16\"},\n",
- " \"mtp.fc\": {\"data_type\": \"bfloat16\"},\n",
- " },\n",
- "}"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -186,6 +162,7 @@
" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
" nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
" batch_size=4, # Batch size for calibration; keep < 8 to avoid slow execution/high memory overhead\n",
+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n",
" seqlen=4096, # Calibration context length; captures long-range dependency activations\n",
" low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
" enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration (JIT acceleration)\n",
diff --git a/examples/Qwen3.8/Qwen3_8_27B_W4A16.ipynb b/examples/Qwen3.8/Qwen3_8_27B_W4A16.ipynb
index a53d6e7c8..202ef646c 100644
--- a/examples/Qwen3.8/Qwen3_8_27B_W4A16.ipynb
+++ b/examples/Qwen3.8/Qwen3_8_27B_W4A16.ipynb
@@ -163,6 +163,7 @@
" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
" nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
" batch_size=2, # Batch size for calibration; reduced to manage GPU memory usage on 48GB VRAM\n",
+ " gradient_accumulate_steps=4, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n",
" seqlen=2048, # Calibration context length; captures long-range dependency activations\n",
" low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
" enable_torch_compile=True, # Compiles forward passes using torch.compile to accelerate calibration\n",
diff --git a/examples/vLLM-sr/d3-edge_W4A16.ipynb b/examples/vLLM-sr/d3-edge_W4A16.ipynb
index ff51b7364..f573b8581 100644
--- a/examples/vLLM-sr/d3-edge_W4A16.ipynb
+++ b/examples/vLLM-sr/d3-edge_W4A16.ipynb
@@ -163,6 +163,7 @@
" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
" nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
" batch_size=4, # Batch size for calibration; kept < 8 to avoid slow execution/high memory overhead\n",
+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n",
" seqlen=4096, # Calibration context length; captures long-range dependency activations\n",
" low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
" quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",
diff --git a/examples/vLLM-sr/d3-lite_W4A16.ipynb b/examples/vLLM-sr/d3-lite_W4A16.ipynb
index 399cebc9b..7fcae6a27 100644
--- a/examples/vLLM-sr/d3-lite_W4A16.ipynb
+++ b/examples/vLLM-sr/d3-lite_W4A16.ipynb
@@ -163,6 +163,7 @@
" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
" nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
" batch_size=4, # Batch size for calibration; kept < 8 to avoid slow execution/high memory overhead\n",
+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n",
" seqlen=4096, # Calibration context length; captures long-range dependency activations\n",
" low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
" quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",
diff --git a/examples/vLLM-sr/d3-mini_W4A16.ipynb b/examples/vLLM-sr/d3-mini_W4A16.ipynb
index 245426f7f..ed61d720d 100644
--- a/examples/vLLM-sr/d3-mini_W4A16.ipynb
+++ b/examples/vLLM-sr/d3-mini_W4A16.ipynb
@@ -163,6 +163,7 @@
" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
" nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
" batch_size=4, # Batch size for calibration; kept < 8 to avoid slow execution/high memory overhead\n",
+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n",
" seqlen=4096, # Calibration context length; captures long-range dependency activations\n",
" low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
" quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",
diff --git a/examples/vLLM-sr/d3-nano_W4A16.ipynb b/examples/vLLM-sr/d3-nano_W4A16.ipynb
index f02594aa9..1a511c48a 100644
--- a/examples/vLLM-sr/d3-nano_W4A16.ipynb
+++ b/examples/vLLM-sr/d3-nano_W4A16.ipynb
@@ -163,6 +163,7 @@
" alg_configs=alg_config, # Applies the algorithm-specific optimization settings defined above\n",
" nsamples=512, # Total calibration dataset samples to maintain quality and avoid overfitting\n",
" batch_size=4, # Batch size for calibration; kept < 8 to avoid slow execution/high memory overhead\n",
+ " gradient_accumulate_steps=2, # Set to 8 / batch_size to maintain an effective global batch size of 8; smaller values risk accuracy drop \n",
" seqlen=4096, # Calibration context length; captures long-range dependency activations\n",
" low_gpu_mem_usage=False, # Keeps modules on GPU to maximize speed if sufficient VRAM is available\n",
" quant_nontext_module=False, # Leaves non-text modules (e.g., vision encoders) unquantized to protect precision\n",