Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix(diffusion): use real images for I2V calibration (#2272)

Wang, Chang1df1cef5fc0f
diff --git a/README.md b/README.md
index 1f75c07c9..cdeaef4cb 100644
--- a/README.md
+++ b/README.md
@@ -260,6 +260,7 @@ ar.quantize_and_save(output_dir="./qmodel", format="auto_round")
##### Calibration Dataset
- **`dataset` (str|list|tuple|torch.utils.data.DataLoader)**: The dataset for tuning (default is `"NeelNanda/pile-10k"`). Supports local JSON files and dataset combinations, e.g. `"./tmp.json,NeelNanda/pile-10k:train,mbpp:train+validation+test"`.
+- I2V calibration uses real images paired with the default COCO2014 captions. Before sampling, it joins the official COCO metadata and retains only Creative Commons Attribution 2.0 images (license ID 4); the filtered manifest preserves the license and Flickr source URL. Its manifests and selected images are cached under `~/.cache/auto_round/datasets/coco2014`, or under `$AUTO_ROUND_CACHE/datasets/coco2014` when `AUTO_ROUND_CACHE` is set. T2V keeps parsing the caption manifest in memory and does not use this cache. A local TSV may instead provide `id`, `caption`, and `image` columns; `image` accepts absolute paths or paths relative to the TSV.
- **`nsamples` (int)**: Number of samples for tuning (default is `128`).
- **`seqlen` (int)**: Data length of the sequence for tuning (default is `2048`).
diff --git a/README_CN.md b/README_CN.md
index 73494030c..f149bec6c 100644
--- a/README_CN.md
+++ b/README_CN.md
@@ -255,6 +255,7 @@ ar.quantize_and_save(output_dir="./qmodel", format="auto_round")
##### 标定数据集
- ​**​`dataset`​**​(str | list | tuple | DataLoader):量化中用于校准的数据集(默认 `"NeelNanda/pile-10k"`​)。支持本地 JSON 文件和数据集组合使用,如 `"./tmp.json,NeelNanda/pile-10k:train,mbpp:train+validation+test"`。
+- I2V 标定默认使用与 COCO2014 caption 配对的真实图片。采样前会关联 COCO 官方元数据,仅保留知识共享署名 2.0(CC BY 2.0,license ID 4)图片;筛选后的清单会保留许可证和 Flickr 来源 URL。清单和选中的图片均缓存在 `~/.cache/auto_round/datasets/coco2014`;设置 `AUTO_ROUND_CACHE` 后则缓存在 `$AUTO_ROUND_CACHE/datasets/coco2014`。T2V 仍只在内存中解析 caption 清单,不使用该缓存。也可使用包含 `id`、`caption` 和 `image` 列的本地 TSV;`image` 支持绝对路径或相对于 TSV 的路径。
- ​**​`nsamples`​**​(int):校准时使用的样本数(默认 `128`)。
- ​**​`seqlen`​**​(int):每条样本在校准时使用 token 的序列长度(默认 `2048`)。
diff --git a/auto_round/calibration/diffusion.py b/auto_round/calibration/diffusion.py
index 8f5bcbe7e..c60a08279 100644
--- a/auto_round/calibration/diffusion.py
+++ b/auto_round/calibration/diffusion.py
@@ -22,7 +22,7 @@ and customises:
"""
import inspect
-from typing import Optional
+import os
import torch
from tqdm import tqdm
@@ -68,33 +68,29 @@ class DiffusionCalibrator(LLMCalibrator):
"""Diffusion calibration never early-stops: all denoising steps execute."""
return False
- def _get_calibration_image(self, batch_size: int):
- """Return a synthetic PIL Image for I2V pipeline calibration."""
- params = inspect.signature(self.pipe.__call__).parameters
- width_param = params.get("width")
- height_param = params.get("height")
- width = (
- 832
- if width_param is None or width_param.default in (inspect.Parameter.empty, None)
- else width_param.default
- )
- height = (
- 480
- if height_param is None or height_param.default in (inspect.Parameter.empty, None)
- else height_param.default
- )
- from PIL import Image # pylint: disable=E0401
-
- image = Image.new("RGB", (int(width), int(height)), color=(127, 127, 127))
- if batch_size == 1:
- return image
- return [image.copy() for _ in range(batch_size)]
-
def _requires_calibration_image(self) -> bool:
"""Return whether the pipeline requires an image input."""
image_param = inspect.signature(self.pipe.__call__).parameters.get("image")
return image_param is not None and image_param.default is inspect.Parameter.empty
+ @staticmethod
+ def _load_calibration_images(image_inputs):
+ """Load image paths while preserving already decoded image inputs."""
+ from PIL import Image # pylint: disable=E0401
+
+ inputs = list(image_inputs) if isinstance(image_inputs, (list, tuple)) else [image_inputs]
+ images = []
+ for image_input in inputs:
+ if isinstance(image_input, (str, os.PathLike)):
+ image_path = os.fspath(image_input)
+ if not os.path.isfile(image_path):
+ raise ValueError(f"Calibration image does not exist: {image_path}")
+ with Image.open(image_path) as image:
+ images.append(image.convert("RGB"))
+ else:
+ images.append(image_input)
+ return images[0] if len(images) == 1 else images
+
@torch.no_grad()
def calib(self, nsamples: int, bs: int) -> None:
"""Drive the diffusion pipeline so block-forward hooks fire.
@@ -113,10 +109,11 @@ class DiffusionCalibrator(LLMCalibrator):
"Diffusion model will catch nsamples * num_inference_steps inputs, "
"you can reduce nsamples or num_inference_steps if OOM or take too much time."
)
+ requires_image = self._requires_calibration_image()
if isinstance(self.dataset, str):
dataset = self.dataset.replace(" ", "")
self.dataloader, self.batch_size = get_diffusion_dataloader(
- dataset=dataset, bs=bs, seed=self.seed, nsamples=nsamples
+ dataset=dataset, bs=bs, seed=self.seed, nsamples=nsamples, image_required=requires_image
)
else:
self.dataloader = self.dataset
@@ -144,11 +141,18 @@ class DiffusionCalibrator(LLMCalibrator):
low_gpu_mem_usage=self.low_gpu_mem_usage,
)
pipeline_fn = getattr(pipe, "_autoround_pipeline_fn", None)
- # Check if this is an I2V pipeline (needs calibration image)
- requires_image = self._requires_calibration_image()
with tqdm(range(1, total + 1), desc="cache block inputs") as pbar:
- for ids, prompts in self.dataloader:
+ for batch in self.dataloader:
+ if len(batch) == 2:
+ ids, prompts = batch
+ image_inputs = None
+ elif len(batch) == 3:
+ ids, prompts, image_inputs = batch
+ else:
+ raise ValueError(
+ "Diffusion calibration batches must contain (ids, prompts) or (ids, prompts, images)."
+ )
if isinstance(prompts, tuple):
prompts = list(prompts)
try:
@@ -158,7 +162,12 @@ class DiffusionCalibrator(LLMCalibrator):
else torch.Generator(device=pipe.device).manual_seed(self.generator_seed)
)
if requires_image:
- image = self._get_calibration_image(len(prompts) if isinstance(prompts, list) else 1)
+ if image_inputs is None:
+ raise ValueError(
+ "I2V calibration requires images. Use the default coco2014 dataset or provide an "
+ "'image' column in a local TSV."
+ )
+ image = self._load_calibration_images(image_inputs)
pipe(
image,
prompt=prompts,
diff --git a/auto_round/compressors/diffusion/dataset.py b/auto_round/compressors/diffusion/dataset.py
index 1b9e2363d..33da29c3b 100644
--- a/auto_round/compressors/diffusion/dataset.py
+++ b/auto_round/compressors/diffusion/dataset.py
@@ -12,8 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import json
import os
+import zipfile
from io import StringIO
+from pathlib import Path
from typing import Dict, Optional
import pandas as pd
@@ -33,6 +36,121 @@ COCO_URL = {
)
}
+COCO_ANNOTATIONS_URL = "https://s3.amazonaws.com/images.cocodataset.org/annotations/annotations_trainval2014.zip"
+COCO_CAPTIONS_MEMBER = "annotations/captions_val2014.json"
+COCO_ALLOWED_LICENSE_ID = 4 # Creative Commons Attribution 2.0
+
+
+def _get_coco_cache_dir() -> Path:
+ """Return the persistent cache directory for COCO calibration data."""
+ from auto_round import envs as _envs
+
+ cache_root = (
+ Path(_envs.AUTO_ROUND_CACHE).expanduser() if _envs.AUTO_ROUND_CACHE else Path.home() / ".cache" / "auto_round"
+ )
+ return cache_root / "datasets" / "coco2014"
+
+
+def _download_to_cache(url: str, destination: Path, timeout: int) -> None:
+ """Download a file atomically unless a non-empty cached copy exists."""
+ if destination.is_file() and destination.stat().st_size > 0:
+ return
+
+ import requests
+
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ temporary = destination.with_name(f"{destination.name}.{os.getpid()}.part")
+ try:
+ response = requests.get(url, timeout=timeout)
+ response.raise_for_status()
+ content = response.content
+ if not content:
+ raise RuntimeError(f"Downloaded an empty file from {url}")
+ temporary.write_bytes(content)
+ temporary.replace(destination)
+ finally:
+ temporary.unlink(missing_ok=True)
+
+
+def _load_cc_by_coco_manifest(dataframe: pd.DataFrame, cache_dir: Path) -> pd.DataFrame:
+ """Join official COCO metadata and retain only CC BY 2.0 (license ID 4) images."""
+ filtered_manifest = cache_dir / "captions_source_cc_by.tsv"
+ if filtered_manifest.is_file() and filtered_manifest.stat().st_size > 0:
+ filtered = pd.read_csv(filtered_manifest, sep="\t")
+ if "license_id" in filtered and filtered["license_id"].eq(COCO_ALLOWED_LICENSE_ID).all():
+ return filtered
+
+ annotation_archive = cache_dir / "annotations_trainval2014.zip"
+ _download_to_cache(COCO_ANNOTATIONS_URL, annotation_archive, timeout=300)
+ try:
+ with zipfile.ZipFile(annotation_archive) as archive, archive.open(COCO_CAPTIONS_MEMBER) as metadata_file:
+ metadata = json.load(metadata_file)
+ finally:
+ # The archive is about 242 MB; the compact filtered manifest is all subsequent runs need.
+ annotation_archive.unlink(missing_ok=True)
+
+ licenses = {item["id"]: item for item in metadata["licenses"]}
+ image_metadata = pd.DataFrame(
+ {
+ "image_id": image["id"],
+ "license_id": image["license"],
+ "flickr_url": image.get("flickr_url", ""),
+ }
+ for image in metadata["images"]
+ )
+ image_metadata["license_url"] = image_metadata["license_id"].map(
+ lambda license_id: licenses.get(license_id, {}).get("url", "")
+ )
+ image_metadata["license_name"] = image_metadata["license_id"].map(
+ lambda license_id: licenses.get(license_id, {}).get("name", "")
+ )
+ filtered = dataframe.merge(image_metadata, on="image_id", how="inner", validate="many_to_one")
+ filtered = filtered.loc[filtered["license_id"] == COCO_ALLOWED_LICENSE_ID].copy()
+ if filtered.empty:
+ raise ValueError("The COCO manifest contains no images with CC BY 2.0 license ID 4.")
+
+ filtered_manifest.parent.mkdir(parents=True, exist_ok=True)
+ temporary = filtered_manifest.with_name(f"{filtered_manifest.name}.{os.getpid()}.part")
+ try:
+ filtered.to_csv(temporary, sep="\t", index=False)
+ temporary.replace(filtered_manifest)
+ finally:
+ temporary.unlink(missing_ok=True)
+ return filtered
+
+
+def _load_coco_dataframe(dataset: str, nsamples: int, image_required: bool) -> pd.DataFrame:
+ """Load COCO metadata, caching the manifest and paired images only for I2V."""
+ cache_dir = _get_coco_cache_dir()
+ if not image_required:
+ import requests
+
+ response = requests.get(COCO_URL[dataset], timeout=30)
+ response.raise_for_status()
+ return pd.read_csv(StringIO(response.text), sep="\t")
+
+ manifest_path = cache_dir / "captions_source.tsv"
+ _download_to_cache(COCO_URL[dataset], manifest_path, timeout=30)
+ dataframe = pd.read_csv(manifest_path, sep="\t")
+
+ required_cols = {"image_id", "coco_url", "file_name"}
+ if not required_cols.issubset(dataframe.columns):
+ raise ValueError(f"COCO I2V calibration requires columns {sorted(required_cols)}.")
+
+ dataframe = _load_cc_by_coco_manifest(dataframe, cache_dir)
+ selected = dataframe.iloc[:nsamples].copy() if nsamples > 0 else dataframe.copy()
+ image_paths = []
+ logger.info(f"Caching {len(selected)} CC BY 2.0 COCO images for I2V calibration in {cache_dir / 'images'}")
+ for _, row in selected.iterrows():
+ image_path = cache_dir / "images" / Path(str(row["file_name"])).name
+ image_url = str(row["coco_url"]).replace(
+ "http://images.cocodataset.org/", "https://s3.amazonaws.com/images.cocodataset.org/", 1
+ )
+ _download_to_cache(image_url, image_path, timeout=60)
+ image_paths.append(str(image_path))
+ selected["image"] = image_paths
+ return selected
+
def register_dataset(name_list):
"""Class decorator to register a DATASET subclass to the registry.
@@ -66,6 +184,7 @@ class Text2ImgDataset(Dataset):
super().__init__()
self.captions = []
self.caption_ids = []
+ self.image_paths = None
if dataframe is not None:
df = dataframe
@@ -80,6 +199,14 @@ class Text2ImgDataset(Dataset):
f"expected columns {sorted(required_cols)}, got {list(df.columns)}"
)
+ if "image" in df.columns:
+ selected_df = df.iloc[:nsamples] if nsamples > 0 else df
+ image_values = selected_df["image"]
+ if image_values.isna().any() or image_values.astype(str).str.strip().eq("").any():
+ raise ValueError("Diffusion datasets with an 'image' column must provide an image for every sample.")
+ dataset_dir = Path(dataset_path).expanduser().resolve().parent
+ self.image_paths = []
+
for index, row in df.iterrows():
if nsamples > 0 and index + 1 > nsamples:
break
@@ -87,11 +214,18 @@ class Text2ImgDataset(Dataset):
caption_text = row["caption"]
self.caption_ids.append(caption_id)
self.captions.append(caption_text)
+ if self.image_paths is not None:
+ image_path = Path(str(row["image"])).expanduser()
+ if not image_path.is_absolute():
+ image_path = dataset_dir / image_path
+ self.image_paths.append(str(image_path))
def __len__(self):
return len(self.captions)
def __getitem__(self, i) -> Dict[str, torch.Tensor]:
+ if self.image_paths is not None:
+ return self.caption_ids[i], self.captions[i], self.image_paths[i]
return self.caption_ids[i], self.captions[i]
@@ -133,7 +267,7 @@ class AudioCapsDataset(Dataset):
return self.caption_ids[i], self.captions[i]
-def get_diffusion_dataloader(dataset="coco2014", bs=1, seed=42, nsamples=128):
+def get_diffusion_dataloader(dataset="coco2014", bs=1, seed=42, nsamples=128, image_required=False):
"""Generate a DataLoader for calibration using specified parameters.
Args:
Dataset_name (str): The name or path of the dataset.
@@ -142,12 +276,8 @@ def get_diffusion_dataloader(dataset="coco2014", bs=1, seed=42, nsamples=128):
DataLoader: The DataLoader for the calibrated datasets.
"""
if dataset in COCO_URL:
- import requests
-
- logger.info(f"use dataset {dataset}, downloading ...")
- response = requests.get(COCO_URL[dataset], timeout=30)
- response.raise_for_status()
- dataframe = pd.read_csv(StringIO(response.text), sep="\t")
+ logger.info(f"use dataset {dataset}, loading calibration data...")
+ dataframe = _load_coco_dataframe(dataset, nsamples, image_required)
dataset = DIFFUSION_DATASET["local"](dataset, nsamples, dataframe=dataframe)
if dataset in ("audiocaps",):
diff --git a/test/unit/common/calibration/test_diffusion.py b/test/unit/common/calibration/test_diffusion.py
index 8695d73bb..d42667708 100644
--- a/test/unit/common/calibration/test_diffusion.py
+++ b/test/unit/common/calibration/test_diffusion.py
@@ -267,15 +267,15 @@ class TestDiffusionCalibrator:
def test_calib_passes_image_when_required(self, calibrator):
seen_images = []
- calibrator.dataset = [("id0", ["p1"])]
+ calibration_image = torch.randn(1, 4, 64, 64)
+ calibrator.dataset = [("id0", ["p1"], [calibration_image])]
calibrator.pipe = ImagePipeline(fn=lambda image, prompt=None, **kwargs: seen_images.append(image))
calibrator._requires_calibration_image = lambda: True
- calibrator._get_calibration_image = lambda batch_size: torch.randn(batch_size, 4, 64, 64)
with patch("auto_round.calibration.diffusion.tqdm", FakeTqdm):
calibrator.calib(nsamples=1, bs=1)
- assert seen_images[0].shape == (1, 4, 64, 64)
+ assert seen_images[0] is calibration_image
def test_calib_not_implemented_error_is_swallowed(self, calibrator):
def failing_pipe(*args, **kwargs):
diff --git a/test/unit/common/models/test_diffusion_dataset.py b/test/unit/common/models/test_diffusion_dataset.py
index 140795e19..e22bb37bc 100644
--- a/test/unit/common/models/test_diffusion_dataset.py
+++ b/test/unit/common/models/test_diffusion_dataset.py
@@ -1,6 +1,9 @@
+import json
+import zipfile
+
import pytest
-from auto_round.compressors.diffusion.dataset import DIFFUSION_DATASET, get_diffusion_dataloader
+from auto_round.compressors.diffusion.dataset import DIFFUSION_DATASET, _load_coco_dataframe, get_diffusion_dataloader
def test_text2img_dataset_rejects_missing_required_columns(tmp_path):
@@ -11,6 +14,16 @@ def test_text2img_dataset_rejects_missing_required_columns(tmp_path):
DIFFUSION_DATASET["local"](str(dataset_path), nsamples=1)
+def test_text2img_dataset_returns_resolved_image_paths(tmp_path):
+ image_path = tmp_path / "images" / "sample.png"
+ dataset_path = tmp_path / "calibration.tsv"
+ dataset_path.write_text("id\tcaption\timage\n1\thello\timages/sample.png\n", encoding="utf-8")
+
+ dataset = DIFFUSION_DATASET["local"](str(dataset_path), nsamples=1)
+
+ assert dataset[0] == (1, "hello", str(image_path))
+
+
def test_get_diffusion_dataloader_parses_coco2014_response_without_temp_file(monkeypatch):
sample_tsv = "id\tcaption\n1\thello\n2\tworld\n"
@@ -33,3 +46,45 @@ def test_get_diffusion_dataloader_parses_coco2014_response_without_temp_file(mon
dataset = dataloader.dataset
assert dataset.caption_ids == [1, 2]
assert dataset.captions == ["hello", "world"]
+
+
+def test_coco_i2v_filters_cc_by_before_selecting_samples(monkeypatch, tmp_path):
+ manifest = (
+ "id\timage_id\tcaption\tfile_name\tcoco_url\n"
+ "1\t101\tnon-commercial\t101.jpg\thttp://images.cocodataset.org/val2014/101.jpg\n"
+ "2\t102\tallowed one\t102.jpg\thttp://images.cocodataset.org/val2014/102.jpg\n"
+ "3\t103\tallowed two\t103.jpg\thttp://images.cocodataset.org/val2014/103.jpg\n"
+ )
+ metadata = {
+ "licenses": [
+ {"id": 2, "url": "http://creativecommons.org/licenses/by-nc/2.0/", "name": "CC BY-NC"},
+ {"id": 4, "url": "http://creativecommons.org/licenses/by/2.0/", "name": "CC BY"},
+ ],
+ "images": [
+ {"id": 101, "license": 2, "flickr_url": "https://flickr.test/101"},
+ {"id": 102, "license": 4, "flickr_url": "https://flickr.test/102"},
+ {"id": 103, "license": 4, "flickr_url": "https://flickr.test/103"},
+ ],
+ }
+
+ def _fake_download(url, destination, timeout):
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ if destination.name == "captions_source.tsv":
+ destination.write_text(manifest, encoding="utf-8")
+ elif destination.suffix == ".zip":
+ with zipfile.ZipFile(destination, "w") as archive:
+ archive.writestr("annotations/captions_val2014.json", json.dumps(metadata))
+ else:
+ destination.write_bytes(b"image")
+
+ monkeypatch.setattr("auto_round.compressors.diffusion.dataset._get_coco_cache_dir", lambda: tmp_path)
+ monkeypatch.setattr("auto_round.compressors.diffusion.dataset._download_to_cache", _fake_download)
+
+ dataframe = _load_coco_dataframe("coco2014", nsamples=1, image_required=True)
+
+ assert dataframe[["image_id", "caption", "license_id"]].to_dict("records") == [
+ {"image_id": 102, "caption": "allowed one", "license_id": 4}
+ ]
+ assert dataframe.iloc[0]["flickr_url"] == "https://flickr.test/102"
+ assert dataframe.iloc[0]["license_url"] == "http://creativecommons.org/licenses/by/2.0/"
+ assert dataframe.iloc[0]["image"].endswith("/images/102.jpg")