PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix(diffusion): use real images for I2V calibration (#2272)
Wang, Chang1df1cef5fc0f
diff --git a/README.md b/README.mdindex 1f75c07c9..cdeaef4cb 100644--- a/README.md+++ b/README.md@@ -260,6 +260,7 @@ ar.quantize_and_save(output_dir="./qmodel", format="auto_round")##### Calibration Dataset- **`dataset` (str|list|tuple|torch.utils.data.DataLoader)**: The dataset for tuning (default is `"NeelNanda/pile-10k"`). Supports local JSON files and dataset combinations, e.g. `"./tmp.json,NeelNanda/pile-10k:train,mbpp:train+validation+test"`.+- I2V calibration uses real images paired with the default COCO2014 captions. Before sampling, it joins the official COCO metadata and retains only Creative Commons Attribution 2.0 images (license ID 4); the filtered manifest preserves the license and Flickr source URL. Its manifests and selected images are cached under `~/.cache/auto_round/datasets/coco2014`, or under `$AUTO_ROUND_CACHE/datasets/coco2014` when `AUTO_ROUND_CACHE` is set. T2V keeps parsing the caption manifest in memory and does not use this cache. A local TSV may instead provide `id`, `caption`, and `image` columns; `image` accepts absolute paths or paths relative to the TSV.- **`nsamples` (int)**: Number of samples for tuning (default is `128`).- **`seqlen` (int)**: Data length of the sequence for tuning (default is `2048`).diff --git a/README_CN.md b/README_CN.mdindex 73494030c..f149bec6c 100644--- a/README_CN.md+++ b/README_CN.md@@ -255,6 +255,7 @@ ar.quantize_and_save(output_dir="./qmodel", format="auto_round")##### 标定数据集- **`dataset`**(str | list | tuple | DataLoader):量化中用于校准的数据集(默认 `"NeelNanda/pile-10k"`)。支持本地 JSON 文件和数据集组合使用,如 `"./tmp.json,NeelNanda/pile-10k:train,mbpp:train+validation+test"`。+- I2V 标定默认使用与 COCO2014 caption 配对的真实图片。采样前会关联 COCO 官方元数据,仅保留知识共享署名 2.0(CC BY 2.0,license ID 4)图片;筛选后的清单会保留许可证和 Flickr 来源 URL。清单和选中的图片均缓存在 `~/.cache/auto_round/datasets/coco2014`;设置 `AUTO_ROUND_CACHE` 后则缓存在 `$AUTO_ROUND_CACHE/datasets/coco2014`。T2V 仍只在内存中解析 caption 清单,不使用该缓存。也可使用包含 `id`、`caption` 和 `image` 列的本地 TSV;`image` 支持绝对路径或相对于 TSV 的路径。- **`nsamples`**(int):校准时使用的样本数(默认 `128`)。- **`seqlen`**(int):每条样本在校准时使用 token 的序列长度(默认 `2048`)。diff --git a/auto_round/calibration/diffusion.py b/auto_round/calibration/diffusion.pyindex 8f5bcbe7e..c60a08279 100644--- a/auto_round/calibration/diffusion.py+++ b/auto_round/calibration/diffusion.py@@ -22,7 +22,7 @@ and customises:"""import inspect-from typing import Optional+import osimport torchfrom tqdm import tqdm@@ -68,33 +68,29 @@ class DiffusionCalibrator(LLMCalibrator):"""Diffusion calibration never early-stops: all denoising steps execute."""return False- def _get_calibration_image(self, batch_size: int):- """Return a synthetic PIL Image for I2V pipeline calibration."""- params = inspect.signature(self.pipe.__call__).parameters- width_param = params.get("width")- height_param = params.get("height")- width = (- 832- if width_param is None or width_param.default in (inspect.Parameter.empty, None)- else width_param.default- )- height = (- 480- if height_param is None or height_param.default in (inspect.Parameter.empty, None)- else height_param.default- )- from PIL import Image # pylint: disable=E0401-- image = Image.new("RGB", (int(width), int(height)), color=(127, 127, 127))- if batch_size == 1:- return image- return [image.copy() for _ in range(batch_size)]-def _requires_calibration_image(self) -> bool:"""Return whether the pipeline requires an image input."""image_param = inspect.signature(self.pipe.__call__).parameters.get("image")return image_param is not None and image_param.default is inspect.Parameter.empty+ @staticmethod+ def _load_calibration_images(image_inputs):+ """Load image paths while preserving already decoded image inputs."""+ from PIL import Image # pylint: disable=E0401++ inputs = list(image_inputs) if isinstance(image_inputs, (list, tuple)) else [image_inputs]+ images = []+ for image_input in inputs:+ if isinstance(image_input, (str, os.PathLike)):+ image_path = os.fspath(image_input)+ if not os.path.isfile(image_path):+ raise ValueError(f"Calibration image does not exist: {image_path}")+ with Image.open(image_path) as image:+ images.append(image.convert("RGB"))+ else:+ images.append(image_input)+ return images[0] if len(images) == 1 else images+@torch.no_grad()def calib(self, nsamples: int, bs: int) -> None:"""Drive the diffusion pipeline so block-forward hooks fire.@@ -113,10 +109,11 @@ class DiffusionCalibrator(LLMCalibrator):"Diffusion model will catch nsamples * num_inference_steps inputs, ""you can reduce nsamples or num_inference_steps if OOM or take too much time.")+ requires_image = self._requires_calibration_image()if isinstance(self.dataset, str):dataset = self.dataset.replace(" ", "")self.dataloader, self.batch_size = get_diffusion_dataloader(- dataset=dataset, bs=bs, seed=self.seed, nsamples=nsamples+ dataset=dataset, bs=bs, seed=self.seed, nsamples=nsamples, image_required=requires_image)else:self.dataloader = self.dataset@@ -144,11 +141,18 @@ class DiffusionCalibrator(LLMCalibrator):low_gpu_mem_usage=self.low_gpu_mem_usage,)pipeline_fn = getattr(pipe, "_autoround_pipeline_fn", None)- # Check if this is an I2V pipeline (needs calibration image)- requires_image = self._requires_calibration_image()with tqdm(range(1, total + 1), desc="cache block inputs") as pbar:- for ids, prompts in self.dataloader:+ for batch in self.dataloader:+ if len(batch) == 2:+ ids, prompts = batch+ image_inputs = None+ elif len(batch) == 3:+ ids, prompts, image_inputs = batch+ else:+ raise ValueError(+ "Diffusion calibration batches must contain (ids, prompts) or (ids, prompts, images)."+ )if isinstance(prompts, tuple):prompts = list(prompts)try:@@ -158,7 +162,12 @@ class DiffusionCalibrator(LLMCalibrator):else torch.Generator(device=pipe.device).manual_seed(self.generator_seed))if requires_image:- image = self._get_calibration_image(len(prompts) if isinstance(prompts, list) else 1)+ if image_inputs is None:+ raise ValueError(+ "I2V calibration requires images. Use the default coco2014 dataset or provide an "+ "'image' column in a local TSV."+ )+ image = self._load_calibration_images(image_inputs)pipe(image,prompt=prompts,diff --git a/auto_round/compressors/diffusion/dataset.py b/auto_round/compressors/diffusion/dataset.pyindex 1b9e2363d..33da29c3b 100644--- a/auto_round/compressors/diffusion/dataset.py+++ b/auto_round/compressors/diffusion/dataset.py@@ -12,8 +12,11 @@# See the License for the specific language governing permissions and# limitations under the License.+import jsonimport os+import zipfilefrom io import StringIO+from pathlib import Pathfrom typing import Dict, Optionalimport pandas as pd@@ -33,6 +36,121 @@ COCO_URL = {)}+COCO_ANNOTATIONS_URL = "https://s3.amazonaws.com/images.cocodataset.org/annotations/annotations_trainval2014.zip"+COCO_CAPTIONS_MEMBER = "annotations/captions_val2014.json"+COCO_ALLOWED_LICENSE_ID = 4 # Creative Commons Attribution 2.0+++def _get_coco_cache_dir() -> Path:+ """Return the persistent cache directory for COCO calibration data."""+ from auto_round import envs as _envs++ cache_root = (+ Path(_envs.AUTO_ROUND_CACHE).expanduser() if _envs.AUTO_ROUND_CACHE else Path.home() / ".cache" / "auto_round"+ )+ return cache_root / "datasets" / "coco2014"+++def _download_to_cache(url: str, destination: Path, timeout: int) -> None:+ """Download a file atomically unless a non-empty cached copy exists."""+ if destination.is_file() and destination.stat().st_size > 0:+ return++ import requests++ destination.parent.mkdir(parents=True, exist_ok=True)+ temporary = destination.with_name(f"{destination.name}.{os.getpid()}.part")+ try:+ response = requests.get(url, timeout=timeout)+ response.raise_for_status()+ content = response.content+ if not content:+ raise RuntimeError(f"Downloaded an empty file from {url}")+ temporary.write_bytes(content)+ temporary.replace(destination)+ finally:+ temporary.unlink(missing_ok=True)+++def _load_cc_by_coco_manifest(dataframe: pd.DataFrame, cache_dir: Path) -> pd.DataFrame:+ """Join official COCO metadata and retain only CC BY 2.0 (license ID 4) images."""+ filtered_manifest = cache_dir / "captions_source_cc_by.tsv"+ if filtered_manifest.is_file() and filtered_manifest.stat().st_size > 0:+ filtered = pd.read_csv(filtered_manifest, sep="\t")+ if "license_id" in filtered and filtered["license_id"].eq(COCO_ALLOWED_LICENSE_ID).all():+ return filtered++ annotation_archive = cache_dir / "annotations_trainval2014.zip"+ _download_to_cache(COCO_ANNOTATIONS_URL, annotation_archive, timeout=300)+ try:+ with zipfile.ZipFile(annotation_archive) as archive, archive.open(COCO_CAPTIONS_MEMBER) as metadata_file:+ metadata = json.load(metadata_file)+ finally:+ # The archive is about 242 MB; the compact filtered manifest is all subsequent runs need.+ annotation_archive.unlink(missing_ok=True)++ licenses = {item["id"]: item for item in metadata["licenses"]}+ image_metadata = pd.DataFrame(+ {+ "image_id": image["id"],+ "license_id": image["license"],+ "flickr_url": image.get("flickr_url", ""),+ }+ for image in metadata["images"]+ )+ image_metadata["license_url"] = image_metadata["license_id"].map(+ lambda license_id: licenses.get(license_id, {}).get("url", "")+ )+ image_metadata["license_name"] = image_metadata["license_id"].map(+ lambda license_id: licenses.get(license_id, {}).get("name", "")+ )+ filtered = dataframe.merge(image_metadata, on="image_id", how="inner", validate="many_to_one")+ filtered = filtered.loc[filtered["license_id"] == COCO_ALLOWED_LICENSE_ID].copy()+ if filtered.empty:+ raise ValueError("The COCO manifest contains no images with CC BY 2.0 license ID 4.")++ filtered_manifest.parent.mkdir(parents=True, exist_ok=True)+ temporary = filtered_manifest.with_name(f"{filtered_manifest.name}.{os.getpid()}.part")+ try:+ filtered.to_csv(temporary, sep="\t", index=False)+ temporary.replace(filtered_manifest)+ finally:+ temporary.unlink(missing_ok=True)+ return filtered+++def _load_coco_dataframe(dataset: str, nsamples: int, image_required: bool) -> pd.DataFrame:+ """Load COCO metadata, caching the manifest and paired images only for I2V."""+ cache_dir = _get_coco_cache_dir()+ if not image_required:+ import requests++ response = requests.get(COCO_URL[dataset], timeout=30)+ response.raise_for_status()+ return pd.read_csv(StringIO(response.text), sep="\t")++ manifest_path = cache_dir / "captions_source.tsv"+ _download_to_cache(COCO_URL[dataset], manifest_path, timeout=30)+ dataframe = pd.read_csv(manifest_path, sep="\t")++ required_cols = {"image_id", "coco_url", "file_name"}+ if not required_cols.issubset(dataframe.columns):+ raise ValueError(f"COCO I2V calibration requires columns {sorted(required_cols)}.")++ dataframe = _load_cc_by_coco_manifest(dataframe, cache_dir)+ selected = dataframe.iloc[:nsamples].copy() if nsamples > 0 else dataframe.copy()+ image_paths = []+ logger.info(f"Caching {len(selected)} CC BY 2.0 COCO images for I2V calibration in {cache_dir / 'images'}")+ for _, row in selected.iterrows():+ image_path = cache_dir / "images" / Path(str(row["file_name"])).name+ image_url = str(row["coco_url"]).replace(+ "http://images.cocodataset.org/", "https://s3.amazonaws.com/images.cocodataset.org/", 1+ )+ _download_to_cache(image_url, image_path, timeout=60)+ image_paths.append(str(image_path))+ selected["image"] = image_paths+ return selected+def register_dataset(name_list):"""Class decorator to register a DATASET subclass to the registry.@@ -66,6 +184,7 @@ class Text2ImgDataset(Dataset):super().__init__()self.captions = []self.caption_ids = []+ self.image_paths = Noneif dataframe is not None:df = dataframe@@ -80,6 +199,14 @@ class Text2ImgDataset(Dataset):f"expected columns {sorted(required_cols)}, got {list(df.columns)}")+ if "image" in df.columns:+ selected_df = df.iloc[:nsamples] if nsamples > 0 else df+ image_values = selected_df["image"]+ if image_values.isna().any() or image_values.astype(str).str.strip().eq("").any():+ raise ValueError("Diffusion datasets with an 'image' column must provide an image for every sample.")+ dataset_dir = Path(dataset_path).expanduser().resolve().parent+ self.image_paths = []+for index, row in df.iterrows():if nsamples > 0 and index + 1 > nsamples:break@@ -87,11 +214,18 @@ class Text2ImgDataset(Dataset):caption_text = row["caption"]self.caption_ids.append(caption_id)self.captions.append(caption_text)+ if self.image_paths is not None:+ image_path = Path(str(row["image"])).expanduser()+ if not image_path.is_absolute():+ image_path = dataset_dir / image_path+ self.image_paths.append(str(image_path))def __len__(self):return len(self.captions)def __getitem__(self, i) -> Dict[str, torch.Tensor]:+ if self.image_paths is not None:+ return self.caption_ids[i], self.captions[i], self.image_paths[i]return self.caption_ids[i], self.captions[i]@@ -133,7 +267,7 @@ class AudioCapsDataset(Dataset):return self.caption_ids[i], self.captions[i]-def get_diffusion_dataloader(dataset="coco2014", bs=1, seed=42, nsamples=128):+def get_diffusion_dataloader(dataset="coco2014", bs=1, seed=42, nsamples=128, image_required=False):"""Generate a DataLoader for calibration using specified parameters.Args:Dataset_name (str): The name or path of the dataset.@@ -142,12 +276,8 @@ def get_diffusion_dataloader(dataset="coco2014", bs=1, seed=42, nsamples=128):DataLoader: The DataLoader for the calibrated datasets."""if dataset in COCO_URL:- import requests-- logger.info(f"use dataset {dataset}, downloading ...")- response = requests.get(COCO_URL[dataset], timeout=30)- response.raise_for_status()- dataframe = pd.read_csv(StringIO(response.text), sep="\t")+ logger.info(f"use dataset {dataset}, loading calibration data...")+ dataframe = _load_coco_dataframe(dataset, nsamples, image_required)dataset = DIFFUSION_DATASET["local"](dataset, nsamples, dataframe=dataframe)if dataset in ("audiocaps",):diff --git a/test/unit/common/calibration/test_diffusion.py b/test/unit/common/calibration/test_diffusion.pyindex 8695d73bb..d42667708 100644--- a/test/unit/common/calibration/test_diffusion.py+++ b/test/unit/common/calibration/test_diffusion.py@@ -267,15 +267,15 @@ class TestDiffusionCalibrator:def test_calib_passes_image_when_required(self, calibrator):seen_images = []- calibrator.dataset = [("id0", ["p1"])]+ calibration_image = torch.randn(1, 4, 64, 64)+ calibrator.dataset = [("id0", ["p1"], [calibration_image])]calibrator.pipe = ImagePipeline(fn=lambda image, prompt=None, **kwargs: seen_images.append(image))calibrator._requires_calibration_image = lambda: True- calibrator._get_calibration_image = lambda batch_size: torch.randn(batch_size, 4, 64, 64)with patch("auto_round.calibration.diffusion.tqdm", FakeTqdm):calibrator.calib(nsamples=1, bs=1)- assert seen_images[0].shape == (1, 4, 64, 64)+ assert seen_images[0] is calibration_imagedef test_calib_not_implemented_error_is_swallowed(self, calibrator):def failing_pipe(*args, **kwargs):diff --git a/test/unit/common/models/test_diffusion_dataset.py b/test/unit/common/models/test_diffusion_dataset.pyindex 140795e19..e22bb37bc 100644--- a/test/unit/common/models/test_diffusion_dataset.py+++ b/test/unit/common/models/test_diffusion_dataset.py@@ -1,6 +1,9 @@+import json+import zipfile+import pytest-from auto_round.compressors.diffusion.dataset import DIFFUSION_DATASET, get_diffusion_dataloader+from auto_round.compressors.diffusion.dataset import DIFFUSION_DATASET, _load_coco_dataframe, get_diffusion_dataloaderdef test_text2img_dataset_rejects_missing_required_columns(tmp_path):@@ -11,6 +14,16 @@ def test_text2img_dataset_rejects_missing_required_columns(tmp_path):DIFFUSION_DATASET["local"](str(dataset_path), nsamples=1)+def test_text2img_dataset_returns_resolved_image_paths(tmp_path):+ image_path = tmp_path / "images" / "sample.png"+ dataset_path = tmp_path / "calibration.tsv"+ dataset_path.write_text("id\tcaption\timage\n1\thello\timages/sample.png\n", encoding="utf-8")++ dataset = DIFFUSION_DATASET["local"](str(dataset_path), nsamples=1)++ assert dataset[0] == (1, "hello", str(image_path))++def test_get_diffusion_dataloader_parses_coco2014_response_without_temp_file(monkeypatch):sample_tsv = "id\tcaption\n1\thello\n2\tworld\n"@@ -33,3 +46,45 @@ def test_get_diffusion_dataloader_parses_coco2014_response_without_temp_file(mondataset = dataloader.datasetassert dataset.caption_ids == [1, 2]assert dataset.captions == ["hello", "world"]+++def test_coco_i2v_filters_cc_by_before_selecting_samples(monkeypatch, tmp_path):+ manifest = (+ "id\timage_id\tcaption\tfile_name\tcoco_url\n"+ "1\t101\tnon-commercial\t101.jpg\thttp://images.cocodataset.org/val2014/101.jpg\n"+ "2\t102\tallowed one\t102.jpg\thttp://images.cocodataset.org/val2014/102.jpg\n"+ "3\t103\tallowed two\t103.jpg\thttp://images.cocodataset.org/val2014/103.jpg\n"+ )+ metadata = {+ "licenses": [+ {"id": 2, "url": "http://creativecommons.org/licenses/by-nc/2.0/", "name": "CC BY-NC"},+ {"id": 4, "url": "http://creativecommons.org/licenses/by/2.0/", "name": "CC BY"},+ ],+ "images": [+ {"id": 101, "license": 2, "flickr_url": "https://flickr.test/101"},+ {"id": 102, "license": 4, "flickr_url": "https://flickr.test/102"},+ {"id": 103, "license": 4, "flickr_url": "https://flickr.test/103"},+ ],+ }++ def _fake_download(url, destination, timeout):+ destination.parent.mkdir(parents=True, exist_ok=True)+ if destination.name == "captions_source.tsv":+ destination.write_text(manifest, encoding="utf-8")+ elif destination.suffix == ".zip":+ with zipfile.ZipFile(destination, "w") as archive:+ archive.writestr("annotations/captions_val2014.json", json.dumps(metadata))+ else:+ destination.write_bytes(b"image")++ monkeypatch.setattr("auto_round.compressors.diffusion.dataset._get_coco_cache_dir", lambda: tmp_path)+ monkeypatch.setattr("auto_round.compressors.diffusion.dataset._download_to_cache", _fake_download)++ dataframe = _load_coco_dataframe("coco2014", nsamples=1, image_required=True)++ assert dataframe[["image_id", "caption", "license_id"]].to_dict("records") == [+ {"image_id": 102, "caption": "allowed one", "license_id": 4}+ ]+ assert dataframe.iloc[0]["flickr_url"] == "https://flickr.test/102"+ assert dataframe.iloc[0]["license_url"] == "http://creativecommons.org/licenses/by/2.0/"+ assert dataframe.iloc[0]["image"].endswith("/images/102.jpg")