Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Add layer wise mode to save memory (#136)

n1ck-guoe2e33f036e62
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 46e254b7b..383ee5695 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -42,8 +42,10 @@ from .utils import (
to_device,
to_dtype,
get_layer_names_in_block,
+ mv_module_from_gpu,
)
+from .layer_wise.utils import get_layers_before_block
class AutoRound(object):
"""This is Signround+ which is an advanced version of Signround. For more information,
@@ -141,11 +143,14 @@ class AutoRound(object):
act_group_size: int = None,
act_sym: bool = None,
act_dynamic: bool = True,
+ low_cpu_mem_usage: bool = False,
**kwargs,
):
self.quantized = False
self.model_orig_dtype = model.dtype
- self.model = model.eval().to("cpu")
+ self.low_cpu_mem_usage = low_cpu_mem_usage
+ self.model = model.eval()
+ self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
self.amp = amp
self.enable_quanted_input = enable_quanted_input
self.enable_minmax_tuning = enable_minmax_tuning
@@ -254,15 +259,15 @@ class AutoRound(object):
self.train_bs = total_samples
logger.warning(f"force the train batch size to {total_samples} ")
- self.model = self.model.to("cpu")
- torch.cuda.empty_cache()
- self.quant_blocks(
- self.model,
- inputs,
- block_names,
- nblocks=self.nblocks,
- device=self.device,
- )
+ self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
+ torch.cuda.empty_cache()
+ self.quant_blocks(
+ self.model,
+ inputs,
+ block_names,
+ nblocks=self.nblocks,
+ device=self.device,
+ )
self.quant_layers(layer_names, all_inputs)
@@ -300,6 +305,9 @@ class AutoRound(object):
Returns:
None
"""
+ # load scale and zp if use low_cpu_memory
+ self.model = self.model.to('cpu')
+
for n, m in self.model.named_modules():
if n not in self.layer_config.keys():
continue
@@ -333,7 +341,7 @@ class AutoRound(object):
if self.enable_quanted_input:
q_layer_inputs = self.try_cache_inter_data_gpucpu([], self.nsamples, layer_names=layer_names)
- self.model = self.model.to("cpu")
+ self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
torch.cuda.empty_cache()
for layer_name in layer_names:
layer_input = layer_inputs[layer_name]
@@ -435,9 +443,9 @@ class AutoRound(object):
nsamples (int): The number of samples to use for calibration.
bs (int): The number of samples to use for calibration
"""
-
if isinstance(self.dataset, str):
dataset = self.dataset.replace(" ", "") ##remove all whitespaces
+ # slow here
self.dataloader = get_dataloader(
self.tokenizer,
self.seqlen,
@@ -449,11 +457,18 @@ class AutoRound(object):
else:
self.dataloader = self.dataset
total_cnt = 0
+
+ # load embed weight if use low_cpu_mem_usage
+ if self.low_cpu_mem_usage:
+ embed_layers = get_layers_before_block(self.model)
+ for n, m in embed_layers:
+ m = m.to(self.device)
+
for data in self.dataloader:
if data is None:
continue
if isinstance(data, torch.Tensor):
- input_ids = data.to(self.model.device)
+ input_ids = data.to(self.device)
data_new = input_ids
elif isinstance(data, str):
if self.tokenizer is None:
@@ -462,7 +477,7 @@ class AutoRound(object):
data = self.tokenizer(data, truncation=True, max_length=self.seqlen, return_tensors="pt").data
data_new = {}
for key in data.keys():
- data_new[key] = data[key].to(self.model.device)
+ data_new[key] = data[key].to(self.device)
input_ids = data_new["input_ids"]
elif isinstance(data, tuple) or isinstance(data, list):
data_new = data
@@ -502,6 +517,12 @@ class AutoRound(object):
f"Insufficient number of samples collected may affect the quantification. "
f"Valid samples size:{total_cnt}, Target sample size:{nsamples}"
)
+
+ # clean embed weight to save memory
+ if self.low_cpu_mem_usage:
+ for n, m in embed_layers:
+ m = m.to("meta")
+ torch.cuda.empty_cache()
@torch.no_grad()
def try_cache_inter_data_gpucpu(self, block_names, nsamples, layer_names=[], last_cache_name=None):
@@ -520,15 +541,16 @@ class AutoRound(object):
Exception: If caching on GPU fails, switches to CPU and caches there.
"""
try:
- self.model = self.model.to(self.device)
+ if not self.model.device.type == "meta":
+ self.model = self.model.to(self.device)
all_inputs = self.cache_inter_data(
block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name
)
- self.model = self.model.to("cpu")
+ self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
torch.cuda.empty_cache()
except:
logger.info("switch to cpu to cache inputs")
- self.model = self.model.to("cpu")
+ self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
torch.cuda.empty_cache()
all_inputs = self.cache_inter_data(
block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name
@@ -712,7 +734,7 @@ class AutoRound(object):
if q_inputs is not None:
q_inputs[i] = q_inputs[i].to(layer.weight.dtype)
- wrapper_linear = WrapperLinear(layer, self.enable_minmax_tuning).to(device)
+ wrapper_linear = WrapperLinear(layer, self.enable_minmax_tuning, device).to(device)
round_params = []
minmax_params = []
round_params.append(wrapper_linear.value)
@@ -824,8 +846,9 @@ class AutoRound(object):
if q_input is not None:
input_ids = q_input
- torch.cuda.empty_cache()
- quantized_layer_names, unquantized_layer_names = wrapper_block(block, self.enable_minmax_tuning)
+
+ quantized_layer_names, unquantized_layer_names = wrapper_block(
+ block, self.enable_minmax_tuning, device=self.device)
round_params = []
minmax_params = []
@@ -935,11 +958,12 @@ class AutoRound(object):
with torch.no_grad():
unwrapper_block(block, best_v, best_min_scale, best_max_scale)
if self.enable_quanted_input:
-
+ block = block.to(device)
q_outputs = self.get_block_outputs(
block, input_ids, input_others, self.train_bs * self.infer_bs_coeff, device,
cache_device=self.cache_device
)
+ block = mv_module_from_gpu(block, self.low_cpu_mem_usage)
for i in range(len(input_ids)):
input_ids[i] = None
torch.cuda.empty_cache()
@@ -1016,7 +1040,7 @@ class AutoRound(object):
q_input=q_input,
device=device,
)
- m = m.to("cpu")
+ self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
torch.cuda.empty_cache()
del q_input
@@ -1038,6 +1062,9 @@ class AutoRound(object):
Returns:
object: The compressed model object.
"""
+ if self.low_cpu_mem_usage:
+ self.model = self.model.to('cpu')
+
if not self.quantized:
logger.warning("please run autoround.quantize first")
return
diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.py
index d7220a78d..50680558e 100644
--- a/auto_round/export/export_to_itrex/export.py
+++ b/auto_round/export/export_to_itrex/export.py
@@ -191,6 +191,8 @@ def pack_model(
scale_dtype=convert_dtype_str2torch(config.scale_dtype),
device="xpu",
"""
+ if model.device.type == 'meta':
+ model = model.to("cuda" if device == "xpu" else device)
if inplace:
compressed_model = model
else:
diff --git a/auto_round/layer_wise/__init__.py b/auto_round/layer_wise/__init__.py
new file mode 100644
index 000000000..453a608c4
--- /dev/null
+++ b/auto_round/layer_wise/__init__.py
@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Torch layer-wise quantization module."""
+from .utils import *
diff --git a/auto_round/layer_wise/load.py b/auto_round/layer_wise/load.py
new file mode 100644
index 000000000..d50fdf734
--- /dev/null
+++ b/auto_round/layer_wise/load.py
@@ -0,0 +1,272 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Load one specify tensor from a bin file."""
+
+import io
+import os
+import warnings
+from typing import IO, Any, BinaryIO, Callable, Dict, Optional, Union
+
+from packaging.version import Version
+from torch.serialization import (
+ StorageType,
+ _get_restore_location,
+ _is_torchscript_zip,
+ _is_zipfile,
+ _maybe_decode_ascii,
+ _open_file_like,
+ _open_zipfile_reader,
+)
+
+from ..layer_wise import modified_pickle as pickle
+
+from .utils import torch
+
+torch_version = torch.__version__.split("+")[0]
+version = Version(torch_version)
+
+FILE_LIKE = Union[str, os.PathLike, BinaryIO, IO[bytes]]
+MAP_LOCATION = Optional[Union[Callable[[torch.Tensor, str], torch.Tensor], torch.device, str, Dict[str, str]]]
+
+if version.release < Version("1.13.0").release:
+ UntypedStorage = torch._UntypedStorage
+else:
+ UntypedStorage = torch.UntypedStorage
+
+
+def _load(zip_file, tensor_name, prefix, map_location, pickle_module, pickle_file="data.pkl", **pickle_load_args):
+ restore_location = _get_restore_location(map_location)
+
+ loaded_storages = {}
+
+ def load_tensor(dtype, numel, key, location):
+ name = f"data/{key}"
+
+ if version.release < Version("1.13.0").release:
+ storage = zip_file.get_storage_from_record(name, numel, torch._UntypedStorage).storage()._untyped()
+ typed_storage = torch.storage._TypedStorage(wrap_storage=restore_location(storage, location), dtype=dtype)
+ loaded_storages[key] = typed_storage
+ elif version.release < Version("2.0.0").release: # pragma: no cover
+ storage = zip_file.get_storage_from_record(name, numel, UntypedStorage).storage().untyped()
+ typed_storage = torch.storage.TypedStorage(wrap_storage=restore_location(storage, location), dtype=dtype)
+ loaded_storages[key] = typed_storage
+ else:
+ storage = zip_file.get_storage_from_record(name, numel, UntypedStorage)._typed_storage()._untyped_storage
+ typed_storage = torch.storage.TypedStorage(
+ wrap_storage=restore_location(storage, location), dtype=dtype, _internal=True
+ )
+
+ if typed_storage._data_ptr() != 0:
+ loaded_storages[key] = typed_storage
+
+ return typed_storage
+
+ load_module_mapping: Dict[str, str] = {"torch.tensor": "torch._tensor"}
+
+ # Need to subclass Unpickler instead of directly monkey-patching the find_class method
+ # because it's marked readonly in pickle.
+ # The type: ignore is because mypy can't statically determine the type of this class.
+ class UnpicklerWrapper(pickle_module.Unpickler): # type: ignore[name-defined]
+ def find_class(self, mod_name, name):
+ if type(name) is str and "Storage" in name:
+ try:
+ return StorageType(name)
+ except KeyError: # pragma: no cover
+ pass
+ mod_name = load_module_mapping.get(mod_name, mod_name)
+ return super().find_class(mod_name, name)
+
+ def persistent_load(self, saved_id):
+ assert isinstance(saved_id, tuple)
+ typename = _maybe_decode_ascii(saved_id[0])
+ data = saved_id[1:]
+
+ assert (
+ typename == "storage"
+ ), f"Unknown typename for persistent_load, expected 'storage' but got '{typename}'"
+ storage_type, key, location, numel = data
+
+ if storage_type is UntypedStorage: # pragma: no cover
+ dtype = torch.uint8
+ else:
+ dtype = storage_type.dtype
+
+ if key in loaded_storages:
+ typed_storage = loaded_storages[key]
+ else:
+ name_list = [self.tensor_name]
+ if prefix:
+ no_prefix_name = self.tensor_name.split(".")
+ if prefix in no_prefix_name:
+ no_prefix_name.remove(prefix)
+ no_prefix_name = ".".join(no_prefix_name)
+ name_list.append(no_prefix_name)
+ if self.tensor_name and self.metastack[-1][-2] not in name_list:
+ # typed_storage = None
+ # loaded_storages[key] = typed_storage
+ # nbytes = numel * torch._utils._element_size(dtype)
+ # typed_storage = load_tensor(dtype, nbytes, key, _maybe_decode_ascii(location))
+ typed_storage = None
+ else:
+ nbytes = numel * torch._utils._element_size(dtype)
+ typed_storage = load_tensor(dtype, nbytes, key, _maybe_decode_ascii(location))
+
+ return typed_storage
+
+ # Load the data (which may in turn use `persistent_load` to load tensors)
+ data_file = io.BytesIO(zip_file.get_record(pickle_file))
+
+ unpickler = UnpicklerWrapper(data_file, **pickle_load_args)
+ # unpickler.persistent_load = persistent_load
+ result = unpickler.load(tensor_name)
+
+ torch._utils._validate_loaded_sparse_tensors()
+ return result
+
+
+def load(
+ f: FILE_LIKE,
+ tensor_name: str = None,
+ prefix: str = None,
+ map_location: MAP_LOCATION = None,
+ pickle_module: Any = None,
+ *,
+ weights_only: bool = False,
+ **pickle_load_args: Any,
+) -> Any:
+ # Reference: https://github.com/pytorch/pytorch/issues/54354
+ # The first line of this docstring overrides the one Sphinx generates for the
+ # documentation. We need it so that Sphinx doesn't leak `pickle`s path from
+ # the build environment (e.g. `<module 'pickle' from '/leaked/path').
+
+ """Load(f, map_location=None, pickle_module=pickle, *, weights_only=False, **pickle_load_args)
+
+ Loads an object saved with :func:`torch.save` from a file.
+
+ :func:`torch.load` uses Python's unpickling facilities but treats storages,
+ which underlie tensors, specially. They are first deserialized on the
+ CPU and are then moved to the device they were saved from. If this fails
+ (e.g. because the run time system doesn't have certain devices), an exception
+ is raised. However, storages can be dynamically remapped to an alternative
+ set of devices using the :attr:`map_location` argument.
+
+ If :attr:`map_location` is a callable, it will be called once for each serialized
+ storage with two arguments: storage and location. The storage argument
+ will be the initial deserialization of the storage, residing on the CPU.
+ Each serialized storage has a location tag associated with it which
+ identifies the device it was saved from, and this tag is the second
+ argument passed to :attr:`map_location`. The builtin location tags are ``'cpu'``
+ for CPU tensors and ``'cuda:device_id'`` (e.g. ``'cuda:2'``) for CUDA tensors.
+ :attr:`map_location` should return either ``None`` or a storage. If
+ :attr:`map_location` returns a storage, it will be used as the final deserialized
+ object, already moved to the right device. Otherwise, :func:`torch.load` will
+ fall back to the default behavior, as if :attr:`map_location` wasn't specified.
+
+ If :attr:`map_location` is a :class:`torch.device` object or a string containing
+ a device tag, it indicates the location where all tensors should be loaded.
+
+ Otherwise, if :attr:`map_location` is a dict, it will be used to remap location tags
+ appearing in the file (keys), to ones that specify where to put the
+ storages (values).
+
+ User extensions can register their own location tags and tagging and
+ deserialization methods using :func:`torch.serialization.register_package`.
+
+ Args:
+ f: a file-like object (has to implement :meth:`read`, :meth:`readline`, :meth:`tell`, and :meth:`seek`),
+ or a string or os.PathLike object containing a file name
+ map_location: a function, :class:`torch.device`, string or a dict specifying how to remap storage
+ locations
+ pickle_module: module used for unpickling metadata and objects (has to
+ match the :attr:`pickle_module` used to serialize file)
+ weights_only: Indicates whether unpickler should be restricted to
+ loading only tensors, primitive types and dictionaries
+ pickle_load_args: (Python 3 only) optional keyword arguments passed over to
+ :func:`pickle_module.load` and :func:`pickle_module.Unpickler`, e.g.,
+ :attr:`errors=...`.
+
+ .. warning::
+ :func:`torch.load()` unless `weights_only` parameter is set to `True`,
+ uses ``pickle`` module implicitly, which is known to be insecure.
+ It is possible to construct malicious pickle data which will execute arbitrary code
+ during unpickling. Never load data that could have come from an untrusted
+ source in an unsafe mode, or that could have been tampered with. **Only load data you trust**.
+
+ .. note::
+ When you call :func:`torch.load()` on a file which contains GPU tensors, those tensors
+ will be loaded to GPU by default. You can call ``torch.load(.., map_location='cpu')``
+ and then :meth:`load_state_dict` to avoid GPU RAM surge when loading a model checkpoint.
+
+ .. note::
+ By default, we decode byte strings as ``utf-8``. This is to avoid a common error
+ case ``UnicodeDecodeError: 'ascii' codec can't decode byte 0x...``
+ when loading files saved by Python 2 in Python 3. If this default
+ is incorrect, you may use an extra :attr:`encoding` keyword argument to specify how
+ these objects should be loaded, e.g., :attr:`encoding='latin1'` decodes them
+ to strings using ``latin1`` encoding, and :attr:`encoding='bytes'` keeps them
+ as byte arrays which can be decoded later with ``byte_array.decode(...)``.
+
+ Example:
+ >>> # xdoctest: +SKIP("undefined filepaths")
+ >>> torch.load('tensors.pt')
+ # Load all tensors onto the CPU
+ >>> torch.load('tensors.pt', map_location=torch.device('cpu'))
+ # Load all tensors onto the CPU, using a function
+ >>> torch.load('tensors.pt', map_location=lambda storage, loc: storage)
+ # Load all tensors onto GPU 1
+ >>> torch.load('tensors.pt', map_location=lambda storage, loc: storage.cuda(1))
+ # Map tensors from GPU 1 to GPU 0
+ >>> torch.load('tensors.pt', map_location={'cuda:1': 'cuda:0'})
+ # Load tensor from io.BytesIO object
+ >>> with open('tensor.pt', 'rb') as f:
+ ... buffer = io.BytesIO(f.read())
+ >>> torch.load(buffer)
+ # Load a module with 'ascii' encoding for unpickling
+ >>> torch.load('module.pt', encoding='ascii')
+ """
+ torch._C._log_api_usage_once("torch.load")
+ # Add ability to force safe only weight loads via environment variable
+ if os.getenv("TORCH_FORCE_WEIGHTS_ONLY_LOAD", "0").lower() in ["1", "y", "yes", "true"]: # pragma: no cover
+ weights_only = True
+
+ if weights_only: # pragma: no cover
+ if pickle_module is not None:
+ raise RuntimeError("Can not safely load weights when explicit pickle_module is specified")
+ else:
+ if pickle_module is None:
+ pickle_module = pickle
+
+ if "encoding" not in pickle_load_args.keys():
+ pickle_load_args["encoding"] = "utf-8"
+
+ with _open_file_like(f, "rb") as opened_file:
+ if _is_zipfile(opened_file):
+ # The zipfile reader is going to advance the current file position.
+ # If we want to actually tail call to torch.jit.load, we need to
+ # reset back to the original position.
+ orig_position = opened_file.tell()
+ with _open_zipfile_reader(opened_file) as opened_zipfile:
+ if _is_torchscript_zip(opened_zipfile): # pragma: no cover
+ warnings.warn(
+ "'torch.load' received a zip file that looks like a TorchScript archive"
+ " dispatching to 'torch.jit.load' (call 'torch.jit.load' directly to"
+ " silence this warning)",
+ UserWarning,
+ )
+ opened_file.seek(orig_position)
+ return torch.jit.load(opened_file, map_location=map_location)
+ return _load(opened_zipfile, tensor_name, prefix, map_location, pickle_module, **pickle_load_args)
diff --git a/auto_round/layer_wise/modified_pickle.py b/auto_round/layer_wise/modified_pickle.py
new file mode 100644
index 000000000..fec381d60
--- /dev/null
+++ b/auto_round/layer_wise/modified_pickle.py
@@ -0,0 +1,1840 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Create portable serialized representations of Python objects.
+
+See module copyreg for a mechanism for registering custom picklers.
+See module pickletools source for extensive comments.
+
+Classes:
+
+ Pickler
+ Unpickler
+
+Functions:
+
+ dump(object, file)
+ dumps(object) -> string
+ load(file) -> object
+ loads(string) -> object
+
+Misc variables:
+
+ __version__
+ format_version
+ compatible_formats
+"""
+
+import codecs
+import io
+import re
+import sys
+from copyreg import _extension_cache, _extension_registry, _inverted_registry, dispatch_table
+from functools import partial
+from itertools import islice
+from struct import pack, unpack
+from sys import maxsize
+from types import FunctionType
+
+import _compat_pickle
+
+__all__ = ["PickleError", "PicklingError", "UnpicklingError", "Pickler", "Unpickler", "dump", "dumps", "load", "loads"]
+
+try:
+ from _pickle import PickleBuffer
+
+ __all__.append("PickleBuffer")
+ _HAVE_PICKLE_BUFFER = True
+except ImportError:
+ _HAVE_PICKLE_BUFFER = False
+
+
+# Shortcut for use in isinstance testing
+bytes_types = (bytes, bytearray)
+
+# These are purely informational; no code uses these.
+format_version = "4.0" # File format version we write
+compatible_formats = [
+ "1.0", # Original protocol 0
+ "1.1", # Protocol 0 with INST added
+ "1.2", # Original protocol 1
+ "1.3", # Protocol 1 with BINFLOAT added
+ "2.0", # Protocol 2
+ "3.0", # Protocol 3
+ "4.0", # Protocol 4
+ "5.0", # Protocol 5
+] # Old format versions we can read
+
+# This is the highest protocol number we know how to read.
+HIGHEST_PROTOCOL = 5
+
+# The protocol we write by default. May be less than HIGHEST_PROTOCOL.
+# Only bump this if the oldest still supported version of Python already
+# includes it.
+DEFAULT_PROTOCOL = 4
+
+
+class PickleError(Exception):
+ """A common base class for the other pickling exceptions."""
+
+ pass
+
+
+class PicklingError(PickleError):
+ """This exception is raised when an unpicklable object is passed to the
+ dump() method."""
+
+ pass
+
+
+class UnpicklingError(PickleError):
+ """This exception is raised when there is a problem unpickling an object,
+ such as a security violation.
+
+ Note that other exceptions may also be raised during unpickling, including
+ (but not necessarily limited to) AttributeError, EOFError, ImportError,
+ and IndexError.
+ """
+
+ pass
+
+
+# An instance of _Stop is raised by Unpickler.load_stop() in response to
+# the STOP opcode, passing the object that is the result of unpickling.
+class _Stop(Exception):
+ def __init__(self, value): # pragma: no cover
+ self.value = value
+
+
+# Jython has PyStringMap; it's a dict subclass with string keys
+try:
+ from org.python.core import PyStringMap
+except ImportError:
+ PyStringMap = None
+
+# Pickle opcodes. See pickletools.py for extensive docs. The listing
+# here is in kind-of alphabetical order of 1-character pickle code.
+# pickletools groups them by purpose.
+# fmt: off
+MARK = b'(' # push special markobject on stack
+STOP = b'.' # every pickle ends with STOP
+POP = b'0' # discard topmost stack item
+POP_MARK = b'1' # discard stack top through topmost markobject
+DUP = b'2' # duplicate top stack item
+FLOAT = b'F' # push float object; decimal string argument
+INT = b'I' # push integer or bool; decimal string argument
+BININT = b'J' # push four-byte signed int
+BININT1 = b'K' # push 1-byte unsigned int
+LONG = b'L' # push long; decimal string argument
+BININT2 = b'M' # push 2-byte unsigned int
+NONE = b'N' # push None
+PERSID = b'P' # push persistent object; id is taken from string arg
+BINPERSID = b'Q' # " " " ; " " " " stack
+REDUCE = b'R' # apply callable to argtuple, both on stack
+STRING = b'S' # push string; NL-terminated string argument
+BINSTRING = b'T' # push string; counted binary string argument
+SHORT_BINSTRING= b'U' # " " ; " " " " < 256 bytes
+UNICODE = b'V' # push Unicode string; raw-unicode-escaped'd argument
+BINUNICODE = b'X' # " " " ; counted UTF-8 string argument
+APPEND = b'a' # append stack top to list below it
+BUILD = b'b' # call __setstate__ or __dict__.update()
+GLOBAL = b'c' # push self.find_class(modname, name); 2 string args
+DICT = b'd' # build a dict from stack items
+EMPTY_DICT = b'}' # push empty dict
+APPENDS = b'e' # extend list on stack by topmost stack slice
+GET = b'g' # push item from memo on stack; index is string arg
+BINGET = b'h' # " " " " " " ; " " 1-byte arg
+INST = b'i' # build & push class instance
+LONG_BINGET = b'j' # push item from memo on stack; index is 4-byte arg
+LIST = b'l' # build list from topmost stack items
+EMPTY_LIST = b']' # push empty list
+OBJ = b'o' # build & push class instance
+PUT = b'p' # store stack top in memo; index is string arg
+BINPUT = b'q' # " " " " " ; " " 1-byte arg
+LONG_BINPUT = b'r' # " " " " " ; " " 4-byte arg
+SETITEM = b's' # add key+value pair to dict
+TUPLE = b't' # build tuple from topmost stack items
+EMPTY_TUPLE = b')' # push empty tuple
+SETITEMS = b'u' # modify dict by adding topmost key+value pairs
+BINFLOAT = b'G' # push float; arg is 8-byte float encoding
+
+TRUE = b'I01\n' # not an opcode; see INT docs in pickletools.py
+FALSE = b'I00\n' # not an opcode; see INT docs in pickletools.py
+
+# Protocol 2
+
+PROTO = b'\x80' # identify pickle protocol
+NEWOBJ = b'\x81' # build object by applying cls.__new__ to argtuple
+EXT1 = b'\x82' # push object from extension registry; 1-byte index
+EXT2 = b'\x83' # ditto, but 2-byte index
+EXT4 = b'\x84' # ditto, but 4-byte index
+TUPLE1 = b'\x85' # build 1-tuple from stack top
+TUPLE2 = b'\x86' # build 2-tuple from two topmost stack items
+TUPLE3 = b'\x87' # build 3-tuple from three topmost stack items
+NEWTRUE = b'\x88' # push True
+NEWFALSE = b'\x89' # push False
+LONG1 = b'\x8a' # push long from < 256 bytes
+LONG4 = b'\x8b' # push really big long
+
+_tuplesize2code = [EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3]
+
+# Protocol 3 (Python 3.x)
+
+BINBYTES = b'B' # push bytes; counted binary string argument
+SHORT_BINBYTES = b'C' # " " ; " " " " < 256 bytes
+
+# Protocol 4
+
+SHORT_BINUNICODE = b'\x8c' # push short string; UTF-8 length < 256 bytes
+BINUNICODE8 = b'\x8d' # push very long string
+BINBYTES8 = b'\x8e' # push very long bytes string
+EMPTY_SET = b'\x8f' # push empty set on the stack
+ADDITEMS = b'\x90' # modify set by adding topmost stack items
+FROZENSET = b'\x91' # build frozenset from topmost stack items
+NEWOBJ_EX = b'\x92' # like NEWOBJ but work with keyword only arguments
+STACK_GLOBAL = b'\x93' # same as GLOBAL but using names on the stacks
+MEMOIZE = b'\x94' # store top of the stack in memo
+FRAME = b'\x95' # indicate the beginning of a new frame
+
+# Protocol 5
+
+BYTEARRAY8 = b'\x96' # push bytearray
+NEXT_BUFFER = b'\x97' # push next out-of-band buffer
+READONLY_BUFFER = b'\x98' # make top of stack readonly
+# fmt: on
+__all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$", x)])
+
+
+class _Framer: # pragma: no cover
+ _FRAME_SIZE_MIN = 4
+ _FRAME_SIZE_TARGET = 64 * 1024
+
+ def __init__(self, file_write):
+ self.file_write = file_write
+ self.current_frame = None
+
+ def start_framing(self):
+ self.current_frame = io.BytesIO()
+
+ def end_framing(self):
+ if self.current_frame and self.current_frame.tell() > 0:
+ self.commit_frame(force=True)
+ self.current_frame = None
+
+ def commit_frame(self, force=False):
+ if self.current_frame:
+ f = self.current_frame
+ if f.tell() >= self._FRAME_SIZE_TARGET or force:
+ data = f.getbuffer()
+ write = self.file_write
+ if len(data) >= self._FRAME_SIZE_MIN:
+ # Issue a single call to the write method of the underlying
+ # file object for the frame opcode with the size of the
+ # frame. The concatenation is expected to be less expensive
+ # than issuing an additional call to write.
+ write(FRAME + pack("<Q", len(data)))
+
+ # Issue a separate call to write to append the frame
+ # contents without concatenation to the above to avoid a
+ # memory copy.
+ write(data)
+
+ # Start the new frame with a new io.BytesIO instance so that
+ # the file object can have delayed access to the previous frame
+ # contents via an unreleased memoryview of the previous
+ # io.BytesIO instance.
+ self.current_frame = io.BytesIO()
+
+ def write(self, data):
+ if self.current_frame:
+ return self.current_frame.write(data)
+ else:
+ return self.file_write(data)
+
+ def write_large_bytes(self, header, payload):
+ write = self.file_write
+ if self.current_frame:
+ # Terminate the current frame and flush it to the file.
+ self.commit_frame(force=True)
+
+ # Perform direct write of the header and payload of the large binary
+ # object. Be careful not to concatenate the header and the payload
+ # prior to calling 'write' as we do not want to allocate a large
+ # temporary bytes object.
+ # We intentionally do not insert a protocol 4 frame opcode to make
+ # it possible to optimize file.read calls in the loader.
+ write(header)
+ write(payload)
+
+
+class _Unframer: # pragma: no cover
+ def __init__(self, file_read, file_readline, file_tell=None):
+ self.file_read = file_read
+ self.file_readline = file_readline
+ self.current_frame = None
+
+ def readinto(self, buf):
+ if self.current_frame:
+ n = self.current_frame.readinto(buf)
+ if n == 0 and len(buf) != 0:
+ self.current_frame = None
+ n = len(buf)
+ buf[:] = self.file_read(n)
+ return n
+ if n < len(buf):
+ raise UnpicklingError("pickle exhausted before end of frame")
+ return n
+ else:
+ n = len(buf)
+ buf[:] = self.file_read(n)
+ return n
+
+ def read(self, n):
+ if self.current_frame:
+ data = self.current_frame.read(n)
+ if not data and n != 0:
+ self.current_frame = None
+ return self.file_read(n)
+ if len(data) < n:
+ raise UnpicklingError("pickle exhausted before end of frame")
+ return data
+ else:
+ return self.file_read(n)
+
+ def readline(self):
+ if self.current_frame:
+ data = self.current_frame.readline()
+ if not data:
+ self.current_frame = None
+ return self.file_readline()
+ if data[-1] != b"\n"[0]:
+ raise UnpicklingError("pickle exhausted before end of frame")
+ return data
+ else:
+ return self.file_readline()
+
+ def load_frame(self, frame_size):
+ if self.current_frame and self.current_frame.read() != b"":
+ raise UnpicklingError("beginning of a new frame before end of current frame")
+ self.current_frame = io.BytesIO(self.file_read(frame_size))
+
+
+# Tools used for pickling.
+
+
+def _getattribute(obj, name): # pragma: no cover
+ for subpath in name.split("."):
+ if subpath == "<locals>":
+ raise AttributeError("Can't get local attribute {!r} on {!r}".format(name, obj))
+ try:
+ parent = obj
+ obj = getattr(obj, subpath)
+ except AttributeError:
+ raise AttributeError("Can't get attribute {!r} on {!r}".format(name, obj)) from None
+ return obj, parent
+
+
+def whichmodule(obj, name): # pragma: no cover
+ """Find the module an object belong to."""
+ module_name = getattr(obj, "__module__", None)
+ if module_name is not None:
+ return module_name
+ # Protect the iteration by using a list copy of sys.modules against dynamic
+ # modules that trigger imports of other modules upon calls to getattr.
+ for module_name, module in sys.modules.copy().items():
+ if module_name == "__main__" or module_name == "__mp_main__" or module is None: # bpo-42406
+ continue
+ try:
+ if _getattribute(module, name)[0] is obj:
+ return module_name
+ except AttributeError:
+ pass
+ return "__main__"
+
+
+def encode_long(x): # pragma: no cover
+ r"""Encode a long to a two's complement little-endian binary string.
+ Note that 0 is a special case, returning an empty string, to save a
+ byte in the LONG1 pickling context.
+
+ >>> encode_long(0)
+ b''
+ >>> encode_long(255)
+ b'\xff\x00'
+ >>> encode_long(32767)
+ b'\xff\x7f'
+ >>> encode_long(-256)
+ b'\x00\xff'
+ >>> encode_long(-32768)
+ b'\x00\x80'
+ >>> encode_long(-128)
+ b'\x80'
+ >>> encode_long(127)
+ b'\x7f'
+ >>>
+ """
+ if x == 0:
+ return b""
+ nbytes = (x.bit_length() >> 3) + 1
+ result = x.to_bytes(nbytes, byteorder="little", signed=True)
+ if x < 0 and nbytes > 1:
+ if result[-1] == 0xFF and (result[-2] & 0x80) != 0:
+ result = result[:-1]
+ return result
+
+
+def decode_long(data): # pragma: no cover
+ r"""Decode a long from a two's complement little-endian binary string.
+
+ >>> decode_long(b'')
+ 0
+ >>> decode_long(b"\xff\x00")
+ 255
+ >>> decode_long(b"\xff\x7f")
+ 32767
+ >>> decode_long(b"\x00\xff")
+ -256
+ >>> decode_long(b"\x00\x80")
+ -32768
+ >>> decode_long(b"\x80")
+ -128
+ >>> decode_long(b"\x7f")
+ 127
+ """
+ return int.from_bytes(data, byteorder="little", signed=True)
+
+
+# Pickling machinery
+
+
+class _Pickler: # pragma: no cover
+ def __init__(self, file, protocol=None, *, fix_imports=True, buffer_callback=None):
+ """This takes a binary file for writing a pickle data stream.
+
+ The optional *protocol* argument tells the pickler to use the
+ given protocol; supported protocols are 0, 1, 2, 3, 4 and 5.
+ The default protocol is 4. It was introduced in Python 3.4, and
+ is incompatible with previous versions.
+
+ Specifying a negative protocol version selects the highest
+ protocol version supported. The higher the protocol used, the
+ more recent the version of Python needed to read the pickle
+ produced.
+
+ The *file* argument must have a write() method that accepts a
+ single bytes argument. It can thus be a file object opened for
+ binary writing, an io.BytesIO instance, or any other custom
+ object that meets this interface.
+
+ If *fix_imports* is True and *protocol* is less than 3, pickle
+ will try to map the new Python 3 names to the old module names
+ used in Python 2, so that the pickle data stream is readable
+ with Python 2.
+
+ If *buffer_callback* is None (the default), buffer views are
+ serialized into *file* as part of the pickle stream.
+
+ If *buffer_callback* is not None, then it can be called any number
+ of times with a buffer view. If the callback returns a false value
+ (such as None), the given buffer is out-of-band; otherwise the
+ buffer is serialized in-band, i.e. inside the pickle stream.
+
+ It is an error if *buffer_callback* is not None and *protocol*
+ is None or smaller than 5.
+ """
+ if protocol is None:
+ protocol = DEFAULT_PROTOCOL
+ if protocol < 0:
+ protocol = HIGHEST_PROTOCOL
+ elif not 0 <= protocol <= HIGHEST_PROTOCOL:
+ raise ValueError("pickle protocol must be <= %d" % HIGHEST_PROTOCOL)
+ if buffer_callback is not None and protocol < 5:
+ raise ValueError("buffer_callback needs protocol >= 5")
+ self._buffer_callback = buffer_callback
+ try:
+ self._file_write = file.write
+ except AttributeError:
+ raise TypeError("file must have a 'write' attribute")
+ self.framer = _Framer(self._file_write)
+ self.write = self.framer.write
+ self._write_large_bytes = self.framer.write_large_bytes
+ self.memo = {}
+ self.proto = int(protocol)
+ self.bin = protocol >= 1
+ self.fast = 0
+ self.fix_imports = fix_imports and protocol < 3
+
+ def clear_memo(self):
+ """Clears the pickler's "memo".
+
+ The memo is the data structure that remembers which objects the
+ pickler has already seen, so that shared or recursive objects
+ are pickled by reference and not by value. This method is
+ useful when re-using picklers.
+ """
+ self.memo.clear()
+
+ def dump(self, obj):
+ """Write a pickled representation of obj to the open file."""
+ # Check whether Pickler was initialized correctly. This is
+ # only needed to mimic the behavior of _pickle.Pickler.dump().
+ if not hasattr(self, "_file_write"):
+ raise PicklingError("Pickler.__init__() was not called by " "%s.__init__()" % (self.__class__.__name__,))
+ if self.proto >= 2:
+ self.write(PROTO + pack("<B", self.proto))
+ if self.proto >= 4:
+ self.framer.start_framing()
+ self.save(obj)
+ self.write(STOP)
+ self.framer.end_framing()
+
+ def memoize(self, obj):
+ """Store an object in the memo."""
+
+ # The Pickler memo is a dictionary mapping object ids to 2-tuples
+ # that contain the Unpickler memo key and the object being memoized.
+ # The memo key is written to the pickle and will become
+ # the key in the Unpickler's memo. The object is stored in the
+ # Pickler memo so that transient objects are kept alive during
+ # pickling.
+
+ # The use of the Unpickler memo length as the memo key is just a
+ # convention. The only requirement is that the memo values be unique.
+ # But there appears no advantage to any other scheme, and this
+ # scheme allows the Unpickler memo to be implemented as a plain (but
+ # growable) array, indexed by memo key.
+ if self.fast:
+ return
+ assert id(obj) not in self.memo
+ idx = len(self.memo)
+ self.write(self.put(idx))
+ self.memo[id(obj)] = idx, obj
+
+ # Return a PUT (BINPUT, LONG_BINPUT) opcode string, with argument i.
+ def put(self, idx):
+ if self.proto >= 4:
+ return MEMOIZE
+ elif self.bin:
+ if idx < 256:
+ return BINPUT + pack("<B", idx)
+ else:
+ return LONG_BINPUT + pack("<I", idx)
+ else:
+ return PUT + repr(idx).encode("ascii") + b"\n"
+
+ # Return a GET (BINGET, LONG_BINGET) opcode string, with argument i.
+ def get(self, i):
+ if self.bin:
+ if i < 256:
+ return BINGET + pack("<B", i)
+ else:
+ return LONG_BINGET + pack("<I", i)
+
+ return GET + repr(i).encode("ascii") + b"\n"
+
+ def save(self, obj, save_persistent_id=True):
+ self.framer.commit_frame()
+
+ # Check for persistent id (defined by a subclass)
+ pid = self.persistent_id(obj) # pylint: disable=E1128
+ if pid is not None and save_persistent_id:
+ self.save_pers(pid)
+ return
+
+ # Check the memo
+ x = self.memo.get(id(obj))
+ if x is not None:
+ self.write(self.get(x[0]))
+ return
+
+ rv = NotImplemented
+ reduce = getattr(self, "reducer_override", None)
+ if reduce is not None:
+ rv = reduce(obj) # pylint: disable=E1102
+
+ if rv is NotImplemented:
+ # Check the type dispatch table
+ t = type(obj)
+ f = self.dispatch.get(t)
+ if f is not None:
+ f(self, obj) # Call unbound method with explicit self
+ return
+
+ # Check private dispatch table if any, or else
+ # copyreg.dispatch_table
+ reduce = getattr(self, "dispatch_table", dispatch_table).get(t)
+ if reduce is not None:
+ rv = reduce(obj)
+ else:
+ # Check for a class with a custom metaclass; treat as regular
+ # class
+ if issubclass(t, type):
+ self.save_global(obj)
+ return
+
+ # Check for a __reduce_ex__ method, fall back to __reduce__
+ reduce = getattr(obj, "__reduce_ex__", None)
+ if reduce is not None:
+ rv = reduce(self.proto)
+ else:
+ reduce = getattr(obj, "__reduce__", None)
+ if reduce is not None:
+ rv = reduce()
+ else:
+ raise PicklingError("Can't pickle %r object: %r" % (t.__name__, obj))
+
+ # Check for string returned by reduce(), meaning "save as global"
+ if isinstance(rv, str):
+ self.save_global(obj, rv)
+ return
+
+ # Assert that reduce() returned a tuple
+ if not isinstance(rv, tuple):
+ raise PicklingError("%s must return string or tuple" % reduce)
+
+ # Assert that it returned an appropriately sized tuple
+ l = len(rv)
+ if not (2 <= l <= 6):
+ raise PicklingError("Tuple returned by %s must have " "two to six elements" % reduce)
+
+ # Save the reduce() output and finally memoize the object
+ self.save_reduce(obj=obj, *rv)
+
+ def persistent_id(self, obj):
+ # This exists so a subclass can override it
+ return None
+
+ def save_pers(self, pid):
+ # Save a persistent id reference
+ if self.bin:
+ self.save(pid, save_persistent_id=False)
+ self.write(BINPERSID)
+ else:
+ try:
+ self.write(PERSID + str(pid).encode("ascii") + b"\n")
+ except UnicodeEncodeError:
+ raise PicklingError("persistent IDs in protocol 0 must be ASCII strings")
+
+ def save_reduce(self, func, args, state=None, listitems=None, dictitems=None, state_setter=None, obj=None):
+ # This API is called by some subclasses
+
+ if not isinstance(args, tuple):
+ raise PicklingError("args from save_reduce() must be a tuple")
+ if not callable(func):
+ raise PicklingError("func from save_reduce() must be callable")
+
+ save = self.save
+ write = self.write
+
+ func_name = getattr(func, "__name__", "")
+ if self.proto >= 2 and func_name == "__newobj_ex__":
+ cls, args, kwargs = args
+ if not hasattr(cls, "__new__"):
+ raise PicklingError("args[0] from {} args has no __new__".format(func_name))
+ if obj is not None and cls is not obj.__class__:
+ raise PicklingError("args[0] from {} args has the wrong class".format(func_name))
+ if self.proto >= 4:
+ save(cls)
+ save(args)
+ save(kwargs)
+ write(NEWOBJ_EX)
+ else:
+ func = partial(cls.__new__, cls, *args, **kwargs)
+ save(func)
+ save(())
+ write(REDUCE)
+ elif self.proto >= 2 and func_name == "__newobj__":
+ # A __reduce__ implementation can direct protocol 2 or newer to
+ # use the more efficient NEWOBJ opcode, while still
+ # allowing protocol 0 and 1 to work normally. For this to
+ # work, the function returned by __reduce__ should be
+ # called __newobj__, and its first argument should be a
+ # class. The implementation for __newobj__
+ # should be as follows, although pickle has no way to
+ # verify this:
+ #
+ # def __newobj__(cls, *args):
+ # return cls.__new__(cls, *args)
+ #
+ # Protocols 0 and 1 will pickle a reference to __newobj__,
+ # while protocol 2 (and above) will pickle a reference to
+ # cls, the remaining args tuple, and the NEWOBJ code,
+ # which calls cls.__new__(cls, *args) at unpickling time
+ # (see load_newobj below). If __reduce__ returns a
+ # three-tuple, the state from the third tuple item will be
+ # pickled regardless of the protocol, calling __setstate__
+ # at unpickling time (see load_build below).
+ #
+ # Note that no standard __newobj__ implementation exists;
+ # you have to provide your own. This is to enforce
+ # compatibility with Python 2.2 (pickles written using
+ # protocol 0 or 1 in Python 2.3 should be unpicklable by
+ # Python 2.2).
+ cls = args[0]
+ if not hasattr(cls, "__new__"):
+ raise PicklingError("args[0] from __newobj__ args has no __new__")
+ if obj is not None and cls is not obj.__class__:
+ raise PicklingError("args[0] from __newobj__ args has the wrong class")
+ args = args[1:]
+ save(cls)
+ save(args)
+ write(NEWOBJ)
+ else:
+ save(func)
+ save(args)
+ write(REDUCE)
+
+ if obj is not None:
+ # If the object is already in the memo, this means it is
+ # recursive. In this case, throw away everything we put on the
+ # stack, and fetch the object back from the memo.
+ if id(obj) in self.memo:
+ write(POP + self.get(self.memo[id(obj)][0]))
+ else:
+ self.memoize(obj)
+
+ # More new special cases (that work with older protocols as
+ # well): when __reduce__ returns a tuple with 4 or 5 items,
+ # the 4th and 5th item should be iterators that provide list
+ # items and dict items (as (key, value) tuples), or None.
+
+ if listitems is not None:
+ self._batch_appends(listitems)
+
+ if dictitems is not None:
+ self._batch_setitems(dictitems)
+
+ if state is not None:
+ if state_setter is None:
+ save(state)
+ write(BUILD)
+ else:
+ # If a state_setter is specified, call it instead of load_build
+ # to update obj's with its previous state.
+ # First, push state_setter and its tuple of expected arguments
+ # (obj, state) onto the stack.
+ save(state_setter)
+ save(obj) # simple BINGET opcode as obj is already memoized.
+ save(state)
+ write(TUPLE2)
+ # Trigger a state_setter(obj, state) function call.
+ write(REDUCE)
+ # The purpose of state_setter is to carry-out an
+ # inplace modification of obj. We do not care about what the
+ # method might return, so its output is eventually removed from
+ # the stack.
+ write(POP)
+
+ # Methods below this point are dispatched through the dispatch table
+
+ dispatch = {}
+
+ def save_none(self, obj):
+ self.write(NONE)
+
+ dispatch[type(None)] = save_none
+
+ def save_bool(self, obj):
+ if self.proto >= 2:
+ self.write(NEWTRUE if obj else NEWFALSE)
+ else:
+ self.write(TRUE if obj else FALSE)
+
+ dispatch[bool] = save_bool
+
+ def save_long(self, obj):
+ if self.bin:
+ # If the int is small enough to fit in a signed 4-byte 2's-comp
+ # format, we can store it more efficiently than the general
+ # case.
+ # First one- and two-byte unsigned ints:
+ if obj >= 0:
+ if obj <= 0xFF:
+ self.write(BININT1 + pack("<B", obj))
+ return
+ if obj <= 0xFFFF:
+ self.write(BININT2 + pack("<H", obj))
+ return
+ # Next check for 4-byte signed ints:
+ if -0x80000000 <= obj <= 0x7FFFFFFF:
+ self.write(BININT + pack("<i", obj))
+ return
+ if self.proto >= 2:
+ encoded = encode_long(obj)
+ n = len(encoded)
+ if n < 256:
+ self.write(LONG1 + pack("<B", n) + encoded)
+ else:
+ self.write(LONG4 + pack("<i", n) + encoded)
+ return
+ if -0x80000000 <= obj <= 0x7FFFFFFF:
+ self.write(INT + repr(obj).encode("ascii") + b"\n")
+ else:
+ self.write(LONG + repr(obj).encode("ascii") + b"L\n")
+
+ dispatch[int] = save_long
+
+ def save_float(self, obj):
+ if self.bin:
+ self.write(BINFLOAT + pack(">d", obj))
+ else:
+ self.write(FLOAT + repr(obj).encode("ascii") + b"\n")
+
+ dispatch[float] = save_float
+
+ def save_bytes(self, obj):
+ if self.proto < 3:
+ if not obj: # bytes object is empty
+ self.save_reduce(bytes, (), obj=obj)
+ else:
+ self.save_reduce(codecs.encode, (str(obj, "latin1"), "latin1"), obj=obj)
+ return
+ n = len(obj)
+ if n <= 0xFF:
+ self.write(SHORT_BINBYTES + pack("<B", n) + obj)
+ elif n > 0xFFFFFFFF and self.proto >= 4:
+ self._write_large_bytes(BINBYTES8 + pack("<Q", n), obj)
+ elif n >= self.framer._FRAME_SIZE_TARGET:
+ self._write_large_bytes(BINBYTES + pack("<I", n), obj)
+ else:
+ self.write(BINBYTES + pack("<I", n) + obj)
+ self.memoize(obj)
+
+ dispatch[bytes] = save_bytes
+
+ def save_bytearray(self, obj):
+ if self.proto < 5:
+ if not obj: # bytearray is empty
+ self.save_reduce(bytearray, (), obj=obj)
+ else:
+ self.save_reduce(bytearray, (bytes(obj),), obj=obj)
+ return
+ n = len(obj)
+ if n >= self.framer._FRAME_SIZE_TARGET:
+ self._write_large_bytes(BYTEARRAY8 + pack("<Q", n), obj)
+ else:
+ self.write(BYTEARRAY8 + pack("<Q", n) + obj)
+
+ dispatch[bytearray] = save_bytearray
+
+ if _HAVE_PICKLE_BUFFER:
+
+ def save_picklebuffer(self, obj):
+ if self.proto < 5:
+ raise PicklingError("PickleBuffer can only pickled with " "protocol >= 5")
+ with obj.raw() as m:
+ if not m.contiguous:
+ raise PicklingError("PickleBuffer can not be pickled when " "pointing to a non-contiguous buffer")
+ in_band = True
+ if self._buffer_callback is not None:
+ in_band = bool(self._buffer_callback(obj))
+ if in_band:
+ # Write data in-band
+ # XXX The C implementation avoids a copy here
+ if m.readonly:
+ self.save_bytes(m.tobytes())
+ else:
+ self.save_bytearray(m.tobytes())
+ else:
+ # Write data out-of-band
+ self.write(NEXT_BUFFER)
+ if m.readonly:
+ self.write(READONLY_BUFFER)
+
+ dispatch[PickleBuffer] = save_picklebuffer
+
+ def save_str(self, obj):
+ if self.bin:
+ encoded = obj.encode("utf-8", "surrogatepass")
+ n = len(encoded)
+ if n <= 0xFF and self.proto >= 4:
+ self.write(SHORT_BINUNICODE + pack("<B", n) + encoded)
+ elif n > 0xFFFFFFFF and self.proto >= 4:
+ self._write_large_bytes(BINUNICODE8 + pack("<Q", n), encoded)
+ elif n >= self.framer._FRAME_SIZE_TARGET:
+ self._write_large_bytes(BINUNICODE + pack("<I", n), encoded)
+ else:
+ self.write(BINUNICODE + pack("<I", n) + encoded)
+ else:
+ obj = obj.replace("\\", "\\u005c")
+ obj = obj.replace("\0", "\\u0000")
+ obj = obj.replace("\n", "\\u000a")
+ obj = obj.replace("\r", "\\u000d")
+ obj = obj.replace("\x1a", "\\u001a") # EOF on DOS
+ self.write(UNICODE + obj.encode("raw-unicode-escape") + b"\n")
+ self.memoize(obj)
+
+ dispatch[str] = save_str
+
+ def save_tuple(self, obj):
+ if not obj: # tuple is empty
+ if self.bin:
+ self.write(EMPTY_TUPLE)
+ else:
+ self.write(MARK + TUPLE)
+ return
+
+ n = len(obj)
+ save = self.save
+ memo = self.memo
+ if n <= 3 and self.proto >= 2:
+ for element in obj:
+ save(element)
+ # Subtle. Same as in the big comment below.
+ if id(obj) in memo:
+ get = self.get(memo[id(obj)][0])
+ self.write(POP * n + get)
+ else:
+ self.write(_tuplesize2code[n])
+ self.memoize(obj)
+ return
+
+ # proto 0 or proto 1 and tuple isn't empty, or proto > 1 and tuple
+ # has more than 3 elements.
+ write = self.write
+ write(MARK)
+ for element in obj:
+ save(element)
+
+ if id(obj) in memo:
+ # Subtle. d was not in memo when we entered save_tuple(), so
+ # the process of saving the tuple's elements must have saved
+ # the tuple itself: the tuple is recursive. The proper action
+ # now is to throw away everything we put on the stack, and
+ # simply GET the tuple (it's already constructed). This check
+ # could have been done in the "for element" loop instead, but
+ # recursive tuples are a rare thing.
+ get = self.get(memo[id(obj)][0])
+ if self.bin:
+ write(POP_MARK + get)
+ else: # proto 0 -- POP_MARK not available
+ write(POP * (n + 1) + get)
+ return
+
+ # No recursion.
+ write(TUPLE)
+ self.memoize(obj)
+
+ dispatch[tuple] = save_tuple
+
+ def save_list(self, obj):
+ if self.bin:
+ self.write(EMPTY_LIST)
+ else: # proto 0 -- can't use EMPTY_LIST
+ self.write(MARK + LIST)
+
+ self.memoize(obj)
+ self._batch_appends(obj)
+
+ dispatch[list] = save_list
+
+ _BATCHSIZE = 1000
+
+ def _batch_appends(self, items):
+ # Helper to batch up APPENDS sequences
+ save = self.save
+ write = self.write
+
+ if not self.bin:
+ for x in items:
+ save(x)
+ write(APPEND)
+ return
+
+ it = iter(items)
+ while True:
+ tmp = list(islice(it, self._BATCHSIZE))
+ n = len(tmp)
+ if n > 1:
+ write(MARK)
+ for x in tmp:
+ save(x)
+ write(APPENDS)
+ elif n:
+ save(tmp[0])
+ write(APPEND)
+ # else tmp is empty, and we're done
+ if n < self._BATCHSIZE:
+ return
+
+ def save_dict(self, obj):
+ if self.bin:
+ self.write(EMPTY_DICT)
+ else: # proto 0 -- can't use EMPTY_DICT
+ self.write(MARK + DICT)
+
+ self.memoize(obj)
+ self._batch_setitems(obj.items())
+
+ dispatch[dict] = save_dict
+ if PyStringMap is not None:
+ dispatch[PyStringMap] = save_dict
+
+ def _batch_setitems(self, items):
+ # Helper to batch up SETITEMS sequences; proto >= 1 only
+ save = self.save
+ write = self.write
+
+ if not self.bin:
+ for k, v in items:
+ save(k)
+ save(v)
+ write(SETITEM)
+ return
+
+ it = iter(items)
+ while True:
+ tmp = list(islice(it, self._BATCHSIZE))
+ n = len(tmp)
+ if n > 1:
+ write(MARK)
+ for k, v in tmp:
+ save(k)
+ save(v)
+ write(SETITEMS)
+ elif n:
+ k, v = tmp[0]
+ save(k)
+ save(v)
+ write(SETITEM)
+ # else tmp is empty, and we're done
+ if n < self._BATCHSIZE:
+ return
+
+ def save_set(self, obj):
+ save = self.save
+ write = self.write
+
+ if self.proto < 4:
+ self.save_reduce(set, (list(obj),), obj=obj)
+ return
+
+ write(EMPTY_SET)
+ self.memoize(obj)
+
+ it = iter(obj)
+ while True:
+ batch = list(islice(it, self._BATCHSIZE))
+ n = len(batch)
+ if n > 0:
+ write(MARK)
+ for item in batch:
+ save(item)
+ write(ADDITEMS)
+ if n < self._BATCHSIZE:
+ return
+
+ dispatch[set] = save_set
+
+ def save_frozenset(self, obj):
+ save = self.save
+ write = self.write
+
+ if self.proto < 4:
+ self.save_reduce(frozenset, (list(obj),), obj=obj)
+ return
+
+ write(MARK)
+ for item in obj:
+ save(item)
+
+ if id(obj) in self.memo:
+ # If the object is already in the memo, this means it is
+ # recursive. In this case, throw away everything we put on the
+ # stack, and fetch the object back from the memo.
+ write(POP_MARK + self.get(self.memo[id(obj)][0]))
+ return
+
+ write(FROZENSET)
+ self.memoize(obj)
+
+ dispatch[frozenset] = save_frozenset
+
+ def save_global(self, obj, name=None):
+ write = self.write
+ memo = self.memo
+
+ if name is None:
+ name = getattr(obj, "__qualname__", None)
+ if name is None:
+ name = obj.__name__
+
+ module_name = whichmodule(obj, name)
+ try:
+ __import__(module_name, level=0)
+ module = sys.modules[module_name]
+ obj2, parent = _getattribute(module, name)
+ except (ImportError, KeyError, AttributeError):
+ raise PicklingError("Can't pickle %r: it's not found as %s.%s" % (obj, module_name, name)) from None
+ else:
+ if obj2 is not obj:
+ raise PicklingError("Can't pickle %r: it's not the same object as %s.%s" % (obj, module_name, name))
+
+ if self.proto >= 2:
+ code = _extension_registry.get((module_name, name))
+ if code:
+ assert code > 0
+ if code <= 0xFF:
+ write(EXT1 + pack("<B", code))
+ elif code <= 0xFFFF:
+ write(EXT2 + pack("<H", code))
+ else:
+ write(EXT4 + pack("<i", code))
+ return
+ lastname = name.rpartition(".")[2]
+ if parent is module:
+ name = lastname
+ # Non-ASCII identifiers are supported only with protocols >= 3.
+ if self.proto >= 4:
+ self.save(module_name)
+ self.save(name)
+ write(STACK_GLOBAL)
+ elif parent is not module:
+ self.save_reduce(getattr, (parent, lastname))
+ elif self.proto >= 3:
+ write(GLOBAL + bytes(module_name, "utf-8") + b"\n" + bytes(name, "utf-8") + b"\n")
+ else:
+ if self.fix_imports:
+ r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING
+ r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING
+ if (module_name, name) in r_name_mapping:
+ module_name, name = r_name_mapping[(module_name, name)]
+ elif module_name in r_import_mapping:
+ module_name = r_import_mapping[module_name]
+ try:
+ write(GLOBAL + bytes(module_name, "ascii") + b"\n" + bytes(name, "ascii") + b"\n")
+ except UnicodeEncodeError:
+ raise PicklingError(
+ "can't pickle global identifier '%s.%s' using " "pickle protocol %i" % (module, name, self.proto)
+ ) from None
+
+ self.memoize(obj)
+
+ def save_type(self, obj):
+ if obj is type(None):
+ return self.save_reduce(type, (None,), obj=obj)
+ elif obj is type(NotImplemented):
+ return self.save_reduce(type, (NotImplemented,), obj=obj)
+ elif obj is type(...):
+ return self.save_reduce(type, (...,), obj=obj)
+ return self.save_global(obj)
+
+ dispatch[FunctionType] = save_global
+ dispatch[type] = save_type
+
+
+# Unpickling machinery
+
+
+class _Unpickler: # pragma: no cover
+ def __init__(self, file, *, fix_imports=True, encoding="ASCII", errors="strict", buffers=None):
+ """This takes a binary file for reading a pickle data stream.
+
+ The protocol version of the pickle is detected automatically, so
+ no proto argument is needed.
+
+ The argument *file* must have two methods, a read() method that
+ takes an integer argument, and a readline() method that requires
+ no arguments. Both methods should return bytes. Thus *file*
+ can be a binary file object opened for reading, an io.BytesIO
+ object, or any other custom object that meets this interface.
+
+ The file-like object must have two methods, a read() method
+ that takes an integer argument, and a readline() method that
+ requires no arguments. Both methods should return bytes.
+ Thus file-like object can be a binary file object opened for
+ reading, a BytesIO object, or any other custom object that
+ meets this interface.
+
+ If *buffers* is not None, it should be an iterable of buffer-enabled
+ objects that is consumed each time the pickle stream references
+ an out-of-band buffer view. Such buffers have been given in order
+ to the *buffer_callback* of a Pickler object.
+
+ If *buffers* is None (the default), then the buffers are taken
+ from the pickle stream, assuming they are serialized there.
+ It is an error for *buffers* to be None if the pickle stream
+ was produced with a non-None *buffer_callback*.
+
+ Other optional arguments are *fix_imports*, *encoding* and
+ *errors*, which are used to control compatibility support for
+ pickle stream generated by Python 2. If *fix_imports* is True,
+ pickle will try to map the old Python 2 names to the new names
+ used in Python 3. The *encoding* and *errors* tell pickle how
+ to decode 8-bit string instances pickled by Python 2; these
+ default to 'ASCII' and 'strict', respectively. *encoding* can be
+ 'bytes' to read these 8-bit string instances as bytes objects.
+ """
+ self._buffers = iter(buffers) if buffers is not None else None
+ self._file_readline = file.readline
+ self._file_read = file.read
+ self.memo = {}
+ self.encoding = encoding
+ self.errors = errors
+ self.proto = 0
+ self.fix_imports = fix_imports
+
+ def load(self, tensor_name=None):
+ """Read a pickled object representation from the open file.
+
+ Return the reconstituted object hierarchy specified in the file.
+ """
+ # Check whether Unpickler was initialized correctly. This is
+ # only needed to mimic the behavior of _pickle.Unpickler.dump().
+
+ if not hasattr(self, "_file_read"):
+ raise UnpicklingError(
+ "Unpickler.__init__() was not called by " "%s.__init__()" % (self.__class__.__name__,)
+ )
+ self.tensor_name = tensor_name
+ self._unframer = _Unframer(self._file_read, self._file_readline)
+ self.read = self._unframer.read
+ self.readinto = self._unframer.readinto
+ self.readline = self._unframer.readline
+ self.metastack = []
+ self.stack = []
+ self.append = self.stack.append
+ self.proto = 0
+ read = self.read
+ dispatch = self.dispatch
+ try:
+ while True:
+ key = read(1)
+ if not key:
+ raise EOFError
+ assert isinstance(key, bytes_types)
+ dispatch[key[0]](self)
+ except _Stop as stopinst:
+ return stopinst.value
+
+ # Return a list of items pushed in the stack after last MARK instruction.
+ def pop_mark(self):
+ items = self.stack
+ self.stack = self.metastack.pop()
+ self.append = self.stack.append
+ return items
+
+ def persistent_load(self, pid):
+ raise UnpicklingError("unsupported persistent id encountered")
+
+ dispatch = {}
+
+ def load_proto(self):
+ proto = self.read(1)[0]
+ if not 0 <= proto <= HIGHEST_PROTOCOL:
+ raise ValueError("unsupported pickle protocol: %d" % proto)
+ self.proto = proto
+
+ dispatch[PROTO[0]] = load_proto
+
+ def load_frame(self):
+ (frame_size,) = unpack("<Q", self.read(8))
+ if frame_size > sys.maxsize:
+ raise ValueError("frame size > sys.maxsize: %d" % frame_size)
+ self._unframer.load_frame(frame_size)
+
+ dispatch[FRAME[0]] = load_frame
+
+ def load_persid(self):
+ try:
+ pid = self.readline()[:-1].decode("ascii")
+ except UnicodeDecodeError:
+ raise UnpicklingError("persistent IDs in protocol 0 must be ASCII strings")
+ self.append(self.persistent_load(pid))
+
+ dispatch[PERSID[0]] = load_persid
+
+ def load_binpersid(self):
+ pid = self.stack.pop()
+ self.append(self.persistent_load(pid))
+
+ dispatch[BINPERSID[0]] = load_binpersid
+
+ def load_none(self):
+ self.append(None)
+
+ dispatch[NONE[0]] = load_none
+
+ def load_false(self):
+ self.append(False)
+
+ dispatch[NEWFALSE[0]] = load_false
+
+ def load_true(self):
+ self.append(True)
+
+ dispatch[NEWTRUE[0]] = load_true
+
+ def load_int(self):
+ data = self.readline()
+ if data == FALSE[1:]:
+ val = False
+ elif data == TRUE[1:]:
+ val = True
+ else:
+ val = int(data, 0)
+ self.append(val)
+
+ dispatch[INT[0]] = load_int
+
+ def load_binint(self):
+ self.append(unpack("<i", self.read(4))[0])
+
+ dispatch[BININT[0]] = load_binint
+
+ def load_binint1(self):
+ self.append(self.read(1)[0])
+
+ dispatch[BININT1[0]] = load_binint1
+
+ def load_binint2(self):
+ self.append(unpack("<H", self.read(2))[0])
+
+ dispatch[BININT2[0]] = load_binint2
+
+ def load_long(self):
+ val = self.readline()[:-1]
+ if val and val[-1] == b"L"[0]:
+ val = val[:-1]
+ self.append(int(val, 0))
+
+ dispatch[LONG[0]] = load_long
+
+ def load_long1(self):
+ n = self.read(1)[0]
+ data = self.read(n)
+ self.append(decode_long(data))
+
+ dispatch[LONG1[0]] = load_long1
+
+ def load_long4(self):
+ (n,) = unpack("<i", self.read(4))
+ if n < 0:
+ # Corrupt or hostile pickle -- we never write one like this
+ raise UnpicklingError("LONG pickle has negative byte count")
+ data = self.read(n)
+ self.append(decode_long(data))
+
+ dispatch[LONG4[0]] = load_long4
+
+ def load_float(self):
+ self.append(float(self.readline()[:-1]))
+
+ dispatch[FLOAT[0]] = load_float
+
+ def load_binfloat(self):
+ self.append(unpack(">d", self.read(8))[0])
+
+ dispatch[BINFLOAT[0]] = load_binfloat
+
+ def _decode_string(self, value):
+ # Used to allow strings from Python 2 to be decoded either as
+ # bytes or Unicode strings. This should be used only with the
+ # STRING, BINSTRING and SHORT_BINSTRING opcodes.
+ if self.encoding == "bytes":
+ return value
+ else:
+ return value.decode(self.encoding, self.errors)
+
+ def load_string(self):
+ data = self.readline()[:-1]
+ # Strip outermost quotes
+ if len(data) >= 2 and data[0] == data[-1] and data[0] in b"\"'":
+ data = data[1:-1]
+ else:
+ raise UnpicklingError("the STRING opcode argument must be quoted")
+ self.append(self._decode_string(codecs.escape_decode(data)[0]))
+
+ dispatch[STRING[0]] = load_string
+
+ def load_binstring(self):
+ # Deprecated BINSTRING uses signed 32-bit length
+ (len,) = unpack("<i", self.read(4))
+ if len < 0:
+ raise UnpicklingError("BINSTRING pickle has negative byte count")
+ data = self.read(len)
+ self.append(self._decode_string(data))
+
+ dispatch[BINSTRING[0]] = load_binstring
+
+ def load_binbytes(self):
+ (len,) = unpack("<I", self.read(4))
+ if len > maxsize:
+ raise UnpicklingError("BINBYTES exceeds system's maximum size " "of %d bytes" % maxsize)
+ self.append(self.read(len))
+
+ dispatch[BINBYTES[0]] = load_binbytes
+
+ def load_unicode(self):
+ self.append(str(self.readline()[:-1], "raw-unicode-escape"))
+
+ dispatch[UNICODE[0]] = load_unicode
+
+ def load_binunicode(self):
+ (len,) = unpack("<I", self.read(4))
+ if len > maxsize:
+ raise UnpicklingError("BINUNICODE exceeds system's maximum size " "of %d bytes" % maxsize)
+ self.append(str(self.read(len), "utf-8", "surrogatepass"))
+
+ dispatch[BINUNICODE[0]] = load_binunicode
+
+ def load_binunicode8(self):
+ (len,) = unpack("<Q", self.read(8))
+ if len > maxsize:
+ raise UnpicklingError("BINUNICODE8 exceeds system's maximum size " "of %d bytes" % maxsize)
+ self.append(str(self.read(len), "utf-8", "surrogatepass"))
+
+ dispatch[BINUNICODE8[0]] = load_binunicode8
+
+ def load_binbytes8(self):
+ (len,) = unpack("<Q", self.read(8))
+ if len > maxsize:
+ raise UnpicklingError("BINBYTES8 exceeds system's maximum size " "of %d bytes" % maxsize)
+ self.append(self.read(len))
+
+ dispatch[BINBYTES8[0]] = load_binbytes8
+
+ def load_bytearray8(self):
+ (len,) = unpack("<Q", self.read(8))
+ if len > maxsize:
+ raise UnpicklingError("BYTEARRAY8 exceeds system's maximum size " "of %d bytes" % maxsize)
+ b = bytearray(len)
+ self.readinto(b)
+ self.append(b)
+
+ dispatch[BYTEARRAY8[0]] = load_bytearray8
+
+ def load_next_buffer(self):
+ if self._buffers is None:
+ raise UnpicklingError("pickle stream refers to out-of-band data " "but no *buffers* argument was given")
+ try:
+ buf = next(self._buffers)
+ except StopIteration:
+ raise UnpicklingError("not enough out-of-band buffers")
+ self.append(buf)
+
+ dispatch[NEXT_BUFFER[0]] = load_next_buffer
+
+ def load_readonly_buffer(self):
+ buf = self.stack[-1]
+ with memoryview(buf) as m:
+ if not m.readonly:
+ self.stack[-1] = m.toreadonly()
+
+ dispatch[READONLY_BUFFER[0]] = load_readonly_buffer
+
+ def load_short_binstring(self):
+ len = self.read(1)[0]
+ data = self.read(len)
+ self.append(self._decode_string(data))
+
+ dispatch[SHORT_BINSTRING[0]] = load_short_binstring
+
+ def load_short_binbytes(self):
+ len = self.read(1)[0]
+ self.append(self.read(len))
+
+ dispatch[SHORT_BINBYTES[0]] = load_short_binbytes
+
+ def load_short_binunicode(self):
+ len = self.read(1)[0]
+ self.append(str(self.read(len), "utf-8", "surrogatepass"))
+
+ dispatch[SHORT_BINUNICODE[0]] = load_short_binunicode
+
+ def load_tuple(self):
+ items = self.pop_mark()
+ self.append(tuple(items))
+
+ dispatch[TUPLE[0]] = load_tuple
+
+ def load_empty_tuple(self):
+ self.append(())
+
+ dispatch[EMPTY_TUPLE[0]] = load_empty_tuple
+
+ def load_tuple1(self):
+ self.stack[-1] = (self.stack[-1],)
+
+ dispatch[TUPLE1[0]] = load_tuple1
+
+ def load_tuple2(self):
+ self.stack[-2:] = [(self.stack[-2], self.stack[-1])]
+
+ dispatch[TUPLE2[0]] = load_tuple2
+
+ def load_tuple3(self):
+ self.stack[-3:] = [(self.stack[-3], self.stack[-2], self.stack[-1])]
+
+ dispatch[TUPLE3[0]] = load_tuple3
+
+ def load_empty_list(self):
+ self.append([])
+
+ dispatch[EMPTY_LIST[0]] = load_empty_list
+
+ def load_empty_dictionary(self):
+ self.append({})
+
+ dispatch[EMPTY_DICT[0]] = load_empty_dictionary
+
+ def load_empty_set(self):
+ self.append(set())
+
+ dispatch[EMPTY_SET[0]] = load_empty_set
+
+ def load_frozenset(self):
+ items = self.pop_mark()
+ self.append(frozenset(items))
+
+ dispatch[FROZENSET[0]] = load_frozenset
+
+ def load_list(self):
+ items = self.pop_mark()
+ self.append(items)
+
+ dispatch[LIST[0]] = load_list
+
+ def load_dict(self):
+ items = self.pop_mark()
+ d = {items[i]: items[i + 1] for i in range(0, len(items), 2)}
+ self.append(d)
+
+ dispatch[DICT[0]] = load_dict
+
+ # INST and OBJ differ only in how they get a class object. It's not
+ # only sensible to do the rest in a common routine, the two routines
+ # previously diverged and grew different bugs.
+ # klass is the class to instantiate, and k points to the topmost mark
+ # object, following which are the arguments for klass.__init__.
+ def _instantiate(self, klass, args):
+ if args or not isinstance(klass, type) or hasattr(klass, "__getinitargs__"):
+ try:
+ value = klass(*args)
+ except TypeError as err:
+ raise TypeError("in constructor for %s: %s" % (klass.__name__, str(err)), sys.exc_info()[2])
+ else:
+ value = klass.__new__(klass)
+ self.append(value)
+
+ def load_inst(self):
+ module = self.readline()[:-1].decode("ascii")
+ name = self.readline()[:-1].decode("ascii")
+ klass = self.find_class(module, name)
+ self._instantiate(klass, self.pop_mark())
+
+ dispatch[INST[0]] = load_inst
+
+ def load_obj(self):
+ # Stack is ... markobject classobject arg1 arg2 ...
+ args = self.pop_mark()
+ cls = args.pop(0)
+ self._instantiate(cls, args)
+
+ dispatch[OBJ[0]] = load_obj
+
+ def load_newobj(self):
+ args = self.stack.pop()
+ cls = self.stack.pop()
+ obj = cls.__new__(cls, *args)
+ self.append(obj)
+
+ dispatch[NEWOBJ[0]] = load_newobj
+
+ def load_newobj_ex(self):
+ kwargs = self.stack.pop()
+ args = self.stack.pop()
+ cls = self.stack.pop()
+ obj = cls.__new__(cls, *args, **kwargs)
+ self.append(obj)
+
+ dispatch[NEWOBJ_EX[0]] = load_newobj_ex
+
+ def load_global(self):
+ module = self.readline()[:-1].decode("utf-8")
+ name = self.readline()[:-1].decode("utf-8")
+ klass = self.find_class(module, name)
+ self.append(klass)
+
+ dispatch[GLOBAL[0]] = load_global
+
+ def load_stack_global(self):
+ name = self.stack.pop()
+ module = self.stack.pop()
+ if type(name) is not str or type(module) is not str:
+ raise UnpicklingError("STACK_GLOBAL requires str")
+ self.append(self.find_class(module, name))
+
+ dispatch[STACK_GLOBAL[0]] = load_stack_global
+
+ def load_ext1(self):
+ code = self.read(1)[0]
+ self.get_extension(code)
+
+ dispatch[EXT1[0]] = load_ext1
+
+ def load_ext2(self):
+ (code,) = unpack("<H", self.read(2))
+ self.get_extension(code)
+
+ dispatch[EXT2[0]] = load_ext2
+
+ def load_ext4(self):
+ (code,) = unpack("<i", self.read(4))
+ self.get_extension(code)
+
+ dispatch[EXT4[0]] = load_ext4
+
+ def get_extension(self, code):
+ nil = []
+ obj = _extension_cache.get(code, nil)
+ if obj is not nil:
+ self.append(obj)
+ return
+ key = _inverted_registry.get(code)
+ if not key:
+ if code <= 0: # note that 0 is forbidden
+ # Corrupt or hostile pickle.
+ raise UnpicklingError("EXT specifies code <= 0")
+ raise ValueError("unregistered extension code %d" % code)
+ obj = self.find_class(*key)
+ _extension_cache[code] = obj
+ self.append(obj)
+
+ def find_class(self, module, name):
+ # Subclasses may override this.
+ sys.audit("pickle.find_class", module, name)
+ if self.proto < 3 and self.fix_imports:
+ if (module, name) in _compat_pickle.NAME_MAPPING:
+ module, name = _compat_pickle.NAME_MAPPING[(module, name)]
+ elif module in _compat_pickle.IMPORT_MAPPING:
+ module = _compat_pickle.IMPORT_MAPPING[module]
+ __import__(module, level=0)
+ if self.proto >= 4:
+ return _getattribute(sys.modules[module], name)[0]
+ else:
+ return getattr(sys.modules[module], name)
+
+ def load_reduce(self):
+ stack = self.stack
+ args = stack.pop()
+ func = stack[-1]
+ if len(args) > 0 and args[0] is None:
+ stack[-1] = None
+ else:
+ stack[-1] = func(*args)
+ # stack[-1] = func(*args)
+
+ dispatch[REDUCE[0]] = load_reduce
+
+ def load_pop(self):
+ if self.stack:
+ del self.stack[-1]
+ else:
+ self.pop_mark()
+
+ dispatch[POP[0]] = load_pop
+
+ def load_pop_mark(self):
+ self.pop_mark()
+
+ dispatch[POP_MARK[0]] = load_pop_mark
+
+ def load_dup(self):
+ self.append(self.stack[-1])
+
+ dispatch[DUP[0]] = load_dup
+
+ def load_get(self):
+ i = int(self.readline()[:-1])
+ self.append(self.memo[i])
+
+ dispatch[GET[0]] = load_get
+
+ def load_binget(self):
+ i = self.read(1)[0]
+ self.append(self.memo[i])
+
+ dispatch[BINGET[0]] = load_binget
+
+ def load_long_binget(self):
+ (i,) = unpack("<I", self.read(4))
+ self.append(self.memo[i])
+
+ dispatch[LONG_BINGET[0]] = load_long_binget
+
+ def load_put(self):
+ i = int(self.readline()[:-1])
+ if i < 0:
+ raise ValueError("negative PUT argument")
+ self.memo[i] = self.stack[-1]
+
+ dispatch[PUT[0]] = load_put
+
+ def load_binput(self):
+ i = self.read(1)[0]
+ if i < 0:
+ raise ValueError("negative BINPUT argument")
+
+ self.memo[i] = self.stack[-1]
+
+ dispatch[BINPUT[0]] = load_binput
+
+ def load_long_binput(self):
+ (i,) = unpack("<I", self.read(4))
+ if i > maxsize:
+ raise ValueError("negative LONG_BINPUT argument")
+ self.memo[i] = self.stack[-1]
+
+ dispatch[LONG_BINPUT[0]] = load_long_binput
+
+ def load_memoize(self):
+ memo = self.memo
+ memo[len(memo)] = self.stack[-1]
+
+ dispatch[MEMOIZE[0]] = load_memoize
+
+ def load_append(self):
+ stack = self.stack
+ value = stack.pop()
+ list = stack[-1]
+ list.append(value)
+
+ dispatch[APPEND[0]] = load_append
+
+ def load_appends(self):
+ items = self.pop_mark()
+ list_obj = self.stack[-1]
+ try:
+ extend = list_obj.extend
+ except AttributeError:
+ pass
+ else:
+ extend(items)
+ return
+ # Even if the PEP 307 requires extend() and append() methods,
+ # fall back on append() if the object has no extend() method
+ # for backward compatibility.
+ append = list_obj.append
+ for item in items:
+ append(item)
+
+ dispatch[APPENDS[0]] = load_appends
+
+ def load_setitem(self):
+ stack = self.stack
+ value = stack.pop()
+ key = stack.pop()
+ dict = stack[-1]
+ dict[key] = value
+
+ dispatch[SETITEM[0]] = load_setitem
+
+ def load_setitems(self):
+ items = self.pop_mark()
+ dict = self.stack[-1]
+ for i in range(0, len(items), 2):
+ dict[items[i]] = items[i + 1]
+
+ dispatch[SETITEMS[0]] = load_setitems
+
+ def load_additems(self):
+ items = self.pop_mark()
+ set_obj = self.stack[-1]
+ if isinstance(set_obj, set):
+ set_obj.update(items)
+ else:
+ add = set_obj.add
+ for item in items:
+ add(item)
+
+ dispatch[ADDITEMS[0]] = load_additems
+
+ def load_build(self):
+ stack = self.stack
+ state = stack.pop()
+ inst = stack[-1]
+ setstate = getattr(inst, "__setstate__", None)
+ if setstate is not None:
+ setstate(state)
+ return
+ slotstate = None
+ if isinstance(state, tuple) and len(state) == 2:
+ state, slotstate = state
+ if state:
+ inst_dict = inst.__dict__
+ intern = sys.intern
+ for k, v in state.items():
+ if type(k) is str:
+ inst_dict[intern(k)] = v
+ else:
+ inst_dict[k] = v
+ if slotstate:
+ for k, v in slotstate.items():
+ setattr(inst, k, v)
+
+ dispatch[BUILD[0]] = load_build
+
+ def load_mark(self):
+ self.metastack.append(self.stack)
+ self.stack = []
+ self.append = self.stack.append
+
+ dispatch[MARK[0]] = load_mark
+
+ def load_stop(self):
+ value = self.stack.pop()
+ raise _Stop(value)
+
+ dispatch[STOP[0]] = load_stop
+
+
+# Shorthands
+
+
+def _dump(obj, file, protocol=None, *, fix_imports=True, buffer_callback=None): # pragma: no cover
+ _Pickler(file, protocol, fix_imports=fix_imports, buffer_callback=buffer_callback).dump(obj)
+
+
+def _dumps(obj, protocol=None, *, fix_imports=True, buffer_callback=None): # pragma: no cover
+ f = io.BytesIO()
+ _Pickler(f, protocol, fix_imports=fix_imports, buffer_callback=buffer_callback).dump(obj)
+ res = f.getvalue()
+ assert isinstance(res, bytes_types)
+ return res
+
+
+def _load(file, *, fix_imports=True, encoding="ASCII", errors="strict", buffers=None): # pragma: no cover
+ return _Unpickler(file, fix_imports=fix_imports, buffers=buffers, encoding=encoding, errors=errors).load()
+
+
+def _loads(s, *, fix_imports=True, encoding="ASCII", errors="strict", buffers=None): # pragma: no cover
+ if isinstance(s, str):
+ raise TypeError("Can't load pickle from unicode string")
+ file = io.BytesIO(s)
+ return _Unpickler(file, fix_imports=fix_imports, buffers=buffers, encoding=encoding, errors=errors).load()
+
+
+# Use the faster _pickle if possible
+Pickler, Unpickler = _Pickler, _Unpickler
+dump, dumps, load, loads = _dump, _dumps, _load, _loads
+
+
+# Doctest
+def _test(): # pragma: no cover
+ import doctest
+
+ return doctest.testmod()
\ No newline at end of file
diff --git a/auto_round/layer_wise/utils.py b/auto_round/layer_wise/utils.py
new file mode 100644
index 000000000..92c14bbac
--- /dev/null
+++ b/auto_round/layer_wise/utils.py
@@ -0,0 +1,476 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utils for layer wise quantization."""
+
+import os
+import gc
+import json
+import pickle
+from functools import partial
+import logging
+from collections import OrderedDict
+
+import torch
+from accelerate import init_empty_weights
+from accelerate.utils import set_module_tensor_to_device
+from transformers import AutoConfig, AutoModelForCausalLM
+from transformers.models.auto.auto_factory import _BaseAutoModelClass
+
+from .load import load
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(filename)s L%(lineno)d: %(message)s")
+logger = logging.getLogger("layer_wise_tools")
+
+LWQ_WORKSPACE = os.path.join("layer_wise_tmp")
+
+
+def get_module(model, key):
+ """Get module from model by key name.
+
+ Args:
+ model (torch.nn.Module): original model
+ key (str): module name to be replaced
+ """
+ attrs = key.split(".")
+ module = model
+ for attr in attrs:
+ try:
+ attr = int(attr)
+ module = module[attr]
+ except:
+ module = getattr(module, attr)
+ return module
+
+
+def get_children(model):
+ """Get all the children of given model."""
+ module_list = []
+ children = list(model.children())
+ if len(children) == 0:
+ return [model]
+ for child in children:
+ module_list += get_children(child)
+ return module_list
+
+
+def get_named_children(model, pre=[]):
+ """Get all the name and children of given model."""
+ module_list = []
+ if len(list(model.children())) == 0:
+ return [(".".join(pre), model)]
+ for name, module in model.named_children():
+ module_list += get_named_children(module, pre=pre + [name])
+ return module_list
+
+
+def dowload_hf_model(repo_id, cache_dir=None, repo_type=None, revision=None):
+ """Download hugging face model from hf hub."""
+ from huggingface_hub.constants import DEFAULT_REVISION, HUGGINGFACE_HUB_CACHE
+ from huggingface_hub.file_download import REGEX_COMMIT_HASH, repo_folder_name
+ from huggingface_hub.utils import EntryNotFoundError
+
+ if cache_dir is None:
+ cache_dir = HUGGINGFACE_HUB_CACHE
+ if revision is None:
+ revision = DEFAULT_REVISION
+ if repo_type is None:
+ repo_type = "model"
+ storage_folder = os.path.join(cache_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type))
+ commit_hash = None
+ if REGEX_COMMIT_HASH.match(revision):
+ commit_hash = revision
+ else:
+ ref_path = os.path.join(storage_folder, "refs", revision)
+ if os.path.exists(ref_path):
+ with open(ref_path) as f:
+ commit_hash = f.read()
+ if storage_folder and commit_hash:
+ pointer_path = os.path.join(storage_folder, "snapshots", commit_hash)
+ if os.path.isdir(pointer_path):
+ return pointer_path
+ else: # pragma: no cover
+ from huggingface_hub import snapshot_download
+
+ file_path = snapshot_download(repo_id)
+ return file_path
+
+
+def load_empty_model(pretrained_model_name_or_path, cls=AutoModelForCausalLM, saved_path=None, **kwargs):
+ """Load a empty model."""
+ is_local = os.path.isdir(pretrained_model_name_or_path)
+ if is_local: # pragma: no cover
+ path = pretrained_model_name_or_path
+ else:
+ path = dowload_hf_model(pretrained_model_name_or_path)
+ torch_dtype = kwargs.pop("torch_dtype", None)
+ if cls.__base__ == _BaseAutoModelClass:
+ config = AutoConfig.from_pretrained(path, **kwargs)
+ if str(torch_dtype) == "auto":
+ if hasattr(config, "torch_dtype") and config.torch_dtype is not None:
+ torch_dtype = config.torch_dtype
+ else:
+ torch_dtype = torch.float32
+ with init_empty_weights():
+ model = cls.from_config(config, torch_dtype=torch_dtype)
+ else: # pragma: no cover
+ config = cls.config_class.from_pretrained(path, **kwargs)
+ if hasattr(config, "torch_dtype") and config.torch_dtype is not None:
+ torch_dtype = config.torch_dtype
+ else:
+ torch_dtype = torch.float32
+ with init_empty_weights():
+ model = cls(config, torch_dtype=torch_dtype)
+ model.tie_weights()
+ model.eval()
+ model.path = path
+
+ if saved_path is None:
+ logger.warning(f"saved_path is not set, use default working space: {LWQ_WORKSPACE}")
+ saved_path = LWQ_WORKSPACE
+ convert_model(model, saved_path)
+ return model
+
+
+def get_super_module_by_name(model, module_name):
+ """Get the father module with given name of child module."""
+ name_list = module_name.split(".")
+ for name in name_list[:-1]:
+ if hasattr(model, name):
+ model = getattr(model, name)
+ else: # pragma: no cover
+ return None
+ if hasattr(model, name_list[-1]):
+ return model
+ else: # pragma: no cover
+ return None
+
+
+def update_module(model, module_name, new_module):
+ """Update module."""
+ super_module = get_super_module_by_name(model, module_name)
+ if super_module:
+ setattr(super_module, module_name.split(".")[-1], new_module)
+
+
+def get_layers_before_block(model):
+ """get the embed layers before blocks.
+ validate on: Llama, opt, bloom, gpt-j, Qwen, Baichuan, Mistral, Mixtral
+ not work for: phi
+ """
+ return_layers = []
+ block_name = None
+ def _forward(module, name, *args, **kwargs):
+ if name == block_name:
+ raise NotImplementedError
+ if len(module._modules) == 0:
+ return_layers.append((name, module))
+ return module.ori_forward(*args, **kwargs)
+
+ for n, m in model.named_modules():
+ if isinstance(m, torch.nn.ModuleList):
+ block_name = n + '.' + m.named_children().__next__()[0]
+ m.ori_forward = m.forward
+ m.forward = partial(_forward, m, n)
+
+ try:
+ if model.device.type == 'meta':
+ target_device = 'meta'
+
+ else:
+ target_device = model.device
+ input = {
+ "input_ids": torch.zeros((1,1), device=target_device, dtype=torch.int),
+ "attention_mask": torch.zeros((1,1), device=target_device, dtype=torch.int)
+ }
+ model.forward(**input)
+ except NotImplementedError:
+ pass
+
+ for n, m in model.named_modules():
+ m.forward = m.ori_forward
+ del m.ori_forward
+
+ return return_layers
+
+
+
+def load_layer_wise_quantized_model(path): # pragma: no cover
+ """Load layer wise quantized model."""
+ model = torch.load(os.path.join(path, "model_arch.pt"))
+ for name, _ in model.named_modules():
+ if name + ".pt" in os.listdir(path):
+ update_module(model, name, torch.load(os.path.join(path, name + ".pt")))
+ model.eval()
+ return model
+
+
+def load_tensor_from_shard(pretrained_model_name_or_path, tensor_name, prefix=None): # pragma: no cover
+ """Load tensor from shard."""
+ path = _get_path(pretrained_model_name_or_path)
+ idx_dict = json.load(open(os.path.join(path, "pytorch_model.bin.index.json"), "r"))["weight_map"]
+ if tensor_name not in idx_dict.keys():
+ if tensor_name.replace(f"{prefix}.", "") in idx_dict.keys():
+ tensor_name = tensor_name.replace(f"{prefix}.", "")
+ else:
+ assert False, "{} not in the index.json".format(tensor_name)
+ return load_tensor(os.path.join(path, idx_dict[tensor_name]), tensor_name, None)
+
+
+def load_tensor(path, tensor_name=None, prefix=None):
+ """Load a tensor from bin file with given tensor name."""
+ # transformers.modeling_utils
+ if tensor_name:
+ if "gamma" in tensor_name: # pragma: no cover
+ tensor_name = tensor_name.replace("gamma", "weight")
+ if "beta" in tensor_name: # pragma: no cover
+ tensor_name = tensor_name.replace("beta", "bias")
+
+ if os.path.isdir(path):
+ path = os.path.join(path, "pytorch_model.bin")
+ state_dict = load(path, tensor_name, prefix)
+ if tensor_name:
+ if tensor_name in state_dict:
+ return state_dict[tensor_name]
+ else: # pragma: no cover
+ return state_dict[tensor_name.replace(f"{prefix}.", "")]
+ else: # pragma: no cover
+ return state_dict
+
+
+def _get_path(pretrained_model_name_or_path):
+ if pretrained_model_name_or_path is None:
+ return None
+ is_local = os.path.isdir(pretrained_model_name_or_path)
+ if is_local: # pragma: no cover
+ path = pretrained_model_name_or_path
+ else:
+ path = dowload_hf_model(pretrained_model_name_or_path)
+ return path
+
+
+def load_value(model, param_name, path):
+ logger.debug(f'load value for layer: {param_name}')
+ if "lm_head" in param_name and getattr(model.config, "tie_word_embeddings", True):
+ input_embeddings = model.get_input_embeddings()
+ modules = get_named_children(model)
+ for name, module in modules:
+ if module == input_embeddings:
+ param_name = name + "." + param_name.split(".")[-1]
+ prefix = model.base_model_prefix
+ if "pytorch_model.bin.index.json" in os.listdir(path):
+ value = load_tensor_from_shard(path, param_name, prefix)
+ else:
+ value = load_tensor(os.path.join(path, "pytorch_model.bin"), param_name, prefix)
+ return value
+
+
+def load_module(model, module_name, path, device="cpu"):
+ module = get_module(model, module_name)
+ for n, p in module.named_parameters():
+ param_name = module_name + "." + n
+ value = load_value(model, param_name, path)
+ set_module_tensor_to_device(model, param_name, device, value)
+
+
+def register_weight_hooks(model, path, device="cpu", clean_weight=True, saved_path=None):
+ if saved_path:
+ os.makedirs(saved_path, exist_ok=True)
+
+ def forward_pre_hook(name):
+ def hook(module, input):
+ logger.debug(f"{name} forward hood load value")
+ state_dict = None
+ if os.path.exists(os.path.join(saved_path, f"{name}.pt")):
+ state_dict = torch.load(os.path.join(saved_path, f"{name}.pt"))
+ for n, p in module.named_parameters():
+ param_name = name + "." + n
+ if state_dict:
+ value = state_dict[n]
+ else:
+ value = load_value(model, param_name, path)
+ set_module_tensor_to_device(model, param_name, device, value)
+ module = module.to(device)
+
+ return hook
+
+ def forward_hook(name):
+ def hook(module, input, output):
+ logger.debug(f"{name} forward hood clean value")
+ if saved_path:
+ file_path = os.path.join(saved_path, f"{name}.pt")
+ torch.save(module.state_dict(), file_path)
+ clean_module_weight(module)
+
+ return hook
+
+ handle = {}
+ modules = get_named_children(model)
+ for name, module in modules:
+ handle[name] = [module.register_forward_pre_hook(forward_pre_hook(name))]
+ if clean_weight:
+ handle[name] += [module.register_forward_hook(forward_hook(name))]
+ return handle
+
+
+def clean_module_weight(submodule): # pragma: no cover
+ for n, m in submodule.named_parameters():
+ is_buffer = n in submodule._buffers
+ old_value = getattr(submodule, n)
+ with torch.no_grad():
+ if is_buffer:
+ submodule._buffers[n] = torch.zeros(old_value.shape, device="meta")
+ else:
+ param_cls = type(submodule._parameters[n])
+ kwargs = submodule._parameters[n].__dict__
+ new_value = torch.zeros(old_value.shape, device="meta")
+ new_value = param_cls(new_value, requires_grad=old_value.requires_grad, **kwargs).to("meta")
+ submodule._parameters[n] = new_value
+ gc.collect()
+
+
+def convert_model(empty_model, saved_path=LWQ_WORKSPACE):
+ os.makedirs(saved_path, exist_ok=True)
+ def _get_value(name, n):
+ state_dict = None
+ if os.path.exists(os.path.join(saved_path, f"{name}.pt")):
+ state_dict = torch.load(os.path.join(saved_path, f"{name}.pt"))
+ param_name = name + "." + n
+ if state_dict:
+ value = state_dict[n]
+ else:
+ value = load_value(empty_model, param_name, empty_model.path)
+ return value
+
+ def _update(name, module):
+ state_dict = None
+ if os.path.exists(os.path.join(saved_path, f"{name}.pt")):
+ state_dict = torch.load(os.path.join(saved_path, f"{name}.pt"))
+ for n, p in module.named_parameters():
+ if str(p.device) != 'meta':
+ continue
+ param_name = name + "." + n
+ if state_dict:
+ value = state_dict[n]
+ else:
+ value = load_value(empty_model, param_name, saved_path)
+ set_module_tensor_to_device(empty_model, param_name, 'cpu', value)
+ file_path = os.path.join(saved_path, f"{name}.pt")
+ torch.save(module.state_dict(), file_path)
+
+ # save quant_info
+ quant_info = {}
+ if hasattr(module, "scale"):
+ quant_info["scale"] = module.scale
+ if hasattr(module, "zp"):
+ quant_info["zp"]= module.zp
+ logger.debug(f"save quant info for layer: {name}")
+ f = open(os.path.join(saved_path, f"{name}_quant_info.pkl"), 'wb')
+ pickle.dump(quant_info, f)
+ f.close()
+
+ def _layer_wise_to(module, name, device_or_dtype):
+ if isinstance(device_or_dtype, torch.dtype):
+ return module.ori_to(device_or_dtype)
+ elif len(module._modules) == 0:
+ # skip method type
+ if len(module._parameters) == 0 or module.weight.device.type != 'meta':
+ if hasattr(module, "scale"):
+ module.scale = module.scale.to(device_or_dtype)
+ if hasattr(module, "zp"):
+ module.zp = module.zp.to(device_or_dtype)
+ return module.ori_to(device_or_dtype)
+ else:
+ for n, _ in module.named_parameters():
+ param_name = name + "." + n
+ value = load_value(empty_model, param_name, empty_model.path)
+ dtype = None
+ if hasattr(module, 'dtype'):
+ dtype = module.dtype
+ set_module_tensor_to_device(module, n, device_or_dtype, value, dtype=dtype)
+
+ if hasattr(module, "scale"):
+ f = open(os.path.join(saved_path, f"{name}_quant_info.pkl"), 'rb')
+ quant_info = pickle.load(f)
+ f.close()
+ module.scale = quant_info["scale"].to(device_or_dtype)
+ if "zp" in quant_info:
+ module.zp = quant_info["zp"].to(device_or_dtype)
+ return module.ori_to(device_or_dtype)
+ else:
+ for n, m in module.named_children():
+ m.to(device_or_dtype)
+ return module
+
+ modules = get_named_children(empty_model)
+ for name, module in modules:
+ if hasattr(module, 'weight'):
+ # delattr(module, 'weight')
+ # module.weight = partial(_get_value, name, 'weight')()
+ module.get_weight = partial(_get_value, name, 'weight')
+ if hasattr(module, 'bias') and module.bias is not None:
+ module.get_bias = partial(_get_value, name, 'bias')
+ module.update = partial(_update, name, module)
+
+ def _repalce_to(module, name):
+ if len(module._modules) > 0:
+ for n, m in module.named_children():
+ if len(name) > 0:
+ n = name + '.' + n
+ _repalce_to(m, n)
+ module.ori_to = module.to
+ module.to = partial(_layer_wise_to, module, name)
+ _repalce_to(empty_model, '')
+
+def load_model_with_hooks(
+ pretrained_model_name_or_path,
+ cls=AutoModelForCausalLM,
+ device="cpu",
+ clean_weight=True,
+ saved_path=None,
+ **kwargs):
+ if saved_path is None:
+ logger.warning(f"saved_path is not set, use default working space: {LWQ_WORKSPACE}")
+ saved_path = LWQ_WORKSPACE
+ empty_model = load_empty_model(pretrained_model_name_or_path, cls=cls, saved_path=saved_path, **kwargs)
+ register_weight_hooks(empty_model, empty_model.path, device, clean_weight, saved_path)
+ return empty_model
+
+
+def layer_wise_save(model, path):
+ os.makedirs(path, exist_ok=True)
+ file_path = os.path.join(path, 'layer_wise_model.bin')
+ modules = get_named_children(model)
+ with open(file_path, 'wb') as f:
+ for name, module in modules:
+ output = OrderedDict()
+ if hasattr(module, "get_weight"):
+ output[f"{name}.weight"] = module.get_weight()
+ if hasattr(module, "get_bias"):
+ output[f"{name}.bias"] = module.get_bias()
+ output = pickle.dumps(output)
+ f.write(output + b'split_tag')
+
+def layer_wise_load(path):
+ file_path = os.path.join(path, 'layer_wise_model.bin')
+ state_dict = OrderedDict()
+ with open(file_path, 'rb') as f:
+ data = f.read().split(b'split_tag')
+ for d in data:
+ if len(d) > 0:
+ d = pickle.loads(d)
+ state_dict.update(d)
+ return state_dict
\ No newline at end of file
diff --git a/auto_round/quantizer.py b/auto_round/quantizer.py
index 151d14266..89ec18a08 100644
--- a/auto_round/quantizer.py
+++ b/auto_round/quantizer.py
@@ -249,7 +249,7 @@ class WrapperWALayer(torch.nn.Module):
class WrapperLinear(torch.nn.Module):
- def __init__(self, orig_layer, enable_minmax_tuning=True):
+ def __init__(self, orig_layer, enable_minmax_tuning=True, device='cpu'):
"""A wrapper module for linear layers that enables quantization and min-max tuning of weights.
Args:
@@ -268,6 +268,7 @@ class WrapperLinear(torch.nn.Module):
"""
super(WrapperLinear, self).__init__()
self.orig_layer = orig_layer
+ self.device = device
self.num_bits = self.orig_layer.bits
self.group_size = self.orig_layer.group_size
self.scale_dtype = self.orig_layer.scale_dtype
@@ -280,12 +281,14 @@ class WrapperLinear(torch.nn.Module):
self.q_scale_thresh = 1e-5
weight_dtype = torch.float32
+ orig_layer_weight = self.orig_layer.weight if not hasattr(self.orig_layer, 'get_weight') \
+ else self.orig_layer.get_weight()
self.value = torch.nn.Parameter(
reshape_tensor(
- torch.zeros(self.orig_layer.weight.shape, device=self.orig_layer.weight.device, dtype=weight_dtype),
+ torch.zeros(self.orig_layer.weight.shape, device=self.device, dtype=weight_dtype),
self.group_size),
requires_grad=True)
- weight_reshape = reshape_tensor(self.orig_layer.weight.data, self.group_size)
+ weight_reshape = reshape_tensor(orig_layer_weight.data, self.group_size)
self.weight_min = torch.clamp(weight_reshape.min(1)[0], max=0)
self.weight_max = torch.clamp(weight_reshape.max(1)[0], min=0)
@@ -293,14 +296,14 @@ class WrapperLinear(torch.nn.Module):
shape = get_scale_shape(self.orig_layer.weight, self.group_size)
if self.enable_minmax_tuning:
self.min_scale = torch.nn.Parameter(
- torch.ones(shape, device=self.orig_layer.weight.device, dtype=weight_dtype), requires_grad=True
+ torch.ones(shape, device=self.device, dtype=weight_dtype), requires_grad=True
)
self.max_scale = torch.nn.Parameter(
- torch.ones(shape, device=self.orig_layer.weight.device, dtype=weight_dtype), requires_grad=True
+ torch.ones(shape, device=self.device, dtype=weight_dtype), requires_grad=True
)
else:
- self.min_scale = torch.tensor(1.0, device=self.orig_layer.weight.device, dtype=weight_dtype)
- self.max_scale = torch.tensor(1.0, device=self.orig_layer.weight.device, dtype=weight_dtype)
+ self.min_scale = torch.tensor(1.0, device=self.device, dtype=weight_dtype)
+ self.max_scale = torch.tensor(1.0, device=self.device, dtype=weight_dtype)
def unwrapper(self, v, min_scale, max_scale):
"""Unwrapper the layer to the original layer.
@@ -316,6 +319,8 @@ class WrapperLinear(torch.nn.Module):
min_scale.clamp_(0, 1.0)
max_scale.clamp_(0, 1.0)
+ if self.orig_layer.weight.device.type == 'meta':
+ self.orig_layer.to(self.device)
qdq_weight, scale, zp = quant_tensor(self.orig_layer.weight, self.num_bits, self.group_size, self.sym, v,
min_scale, max_scale, self.scale_dtype, self.weight_min, self.weight_max)
scale = scale.reshape(qdq_weight.shape[0], -1)
@@ -326,6 +331,9 @@ class WrapperLinear(torch.nn.Module):
self.orig_layer.scale = scale.to("cpu")
self.orig_layer.zp = zp.to("cpu") if zp is not None else None
self.orig_layer.q_scale_thresh = self.q_scale_thresh
+ if hasattr(self.orig_layer, 'update'):
+ self.orig_layer.update()
+ self.orig_layer.to('meta')
if self.act_quant:
wrapper_layer = WrapperWALayer(self.orig_layer)
return wrapper_layer
@@ -343,6 +351,8 @@ class WrapperLinear(torch.nn.Module):
from torch.functional import F
weight = self.orig_layer.weight
+ if weight.device.type == 'meta':
+ weight = self.orig_layer.get_weight().to(self.device)
self.min_scale.data.copy_(torch.clamp(self.min_scale.data, 0, 1.0))
self.max_scale.data.copy_(torch.clamp(self.max_scale.data, 0, 1.0))
weight_q, _, _ = quant_tensor(weight, self.num_bits, self.group_size, self.sym, self.value, self.min_scale,
@@ -352,11 +362,14 @@ class WrapperLinear(torch.nn.Module):
x, _, _ = quant_tensor(x, self.act_bits, self.act_group_size, self.act_sym,
scale_dtype=self.scale_dtype, q_scale_thresh=self.q_scale_thresh)
# pylint: disable=not-callable
- return F.linear(x, weight_q, self.orig_layer.bias)
+ bias = self.orig_layer.bias
+ if bias is not None and bias.device.type == 'meta':
+ bias = self.orig_layer.get_bias().to(self.device)
+ return F.linear(x, weight_q, bias)
class WrapperTransformerConv1d(torch.nn.Module):
- def __init__(self, orig_layer, enable_minmax_tuning=True):
+ def __init__(self, orig_layer, enable_minmax_tuning=True, device='cpu'):
"""A wrapper module for transformers 1D convolutional layers used in transformers,
enabling quantization and min-max tuning of weights.
@@ -391,8 +404,12 @@ class WrapperTransformerConv1d(torch.nn.Module):
self.act_quant = self.act_bits <= 8
self.q_scale_thresh = 1e-5
weight_dtype = torch.float32
- device = self.orig_layer.weight.device
- self.weight_t = self.orig_layer.weight.t()
+ self.device = device
+ if hasattr(self.orig_layer, 'get_weight'):
+ self.weight_t = self.orig_layer.get_weight().t()
+ else:
+ self.weight_t = self.orig_layer.weight.t()
+ self.weight_t = self.weight_t.to(self.device)
self.value = torch.nn.Parameter(
reshape_tensor(torch.zeros(self.weight_t.shape, device=device, dtype=weight_dtype),
group_size=self.group_size),
@@ -433,6 +450,8 @@ class WrapperTransformerConv1d(torch.nn.Module):
scale = scale.reshape(qdq_weight.shape[0], -1)
if zp is not None:
zp = zp.reshape(qdq_weight.shape[0], -1)
+ if self.orig_layer.weight.device.type == 'meta':
+ self.orig_layer.weight.to(self.device)
self.orig_layer.weight.data.copy_(qdq_weight.t())
self.orig_layer.weight.grad = None
self.orig_layer.scale = scale.to("cpu")
@@ -441,6 +460,9 @@ class WrapperTransformerConv1d(torch.nn.Module):
if self.act_quant:
wrapper_layer = WrapperWALayer(self.orig_layer)
return wrapper_layer
+ if hasattr(self.orig_layer, 'update'):
+ self.orig_layer.update()
+ self.orig_layer.to('meta')
return self.orig_layer
def forward(self, x):
@@ -489,7 +511,7 @@ class WrapperMultiblock(torch.nn.Module):
return hidden_states
-def wrapper_block(block, enable_minmax_tuning):
+def wrapper_block(block, enable_minmax_tuning, device='cpu'):
"""Wraps the layers in the given block with a custom Wrapper module.
Args:
@@ -506,7 +528,7 @@ def wrapper_block(block, enable_minmax_tuning):
if not check_to_quantized(m):
unquantized_layers.append(n)
continue
- new_m = WrapperLinear(m, enable_minmax_tuning=enable_minmax_tuning)
+ new_m = WrapperLinear(m, enable_minmax_tuning=enable_minmax_tuning, device=device)
set_module(block, n, new_m)
quantized_layers.append(n)
@@ -514,7 +536,7 @@ def wrapper_block(block, enable_minmax_tuning):
if not check_to_quantized(m):
unquantized_layers.append(n)
continue
- new_m = WrapperTransformerConv1d(m, enable_minmax_tuning=enable_minmax_tuning)
+ new_m = WrapperTransformerConv1d(m, enable_minmax_tuning=enable_minmax_tuning, device=device)
set_module(block, n, new_m)
quantized_layers.append(n)
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 8edc92d19..aae086a75 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -165,6 +165,19 @@ def to_device(input, device=torch.device("cpu")):
return input
+def mv_module_from_gpu(module, low_cpu_mem_usage=False):
+ if hasattr(module, "device"):
+ target_device ="meta" if low_cpu_mem_usage else "cpu"
+ if module.device.type == target_device:
+ return module
+ else:
+ return module.to(target_device)
+ else:
+ if low_cpu_mem_usage:
+ return module.to('meta')
+ else:
+ return module.to('cpu')
+
def to_dtype(input, dtype=torch.float32):
"""Moves input data to the specified data type.
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 5d2554de6..5f74de672 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -116,6 +116,8 @@ if __name__ == '__main__':
parser.add_argument("--quant_lm_head", action='store_true',
help="quant_lm_head")
+ parser.add_argument("--low_cpu_mem_mode", default=0, type=int,
+ help="choose low cpu memory mode, 1 for block-wise, 2 for layer-wise, others means not use low cpu memory.")
parser.add_argument("--model_dtype", default=None, type=str,
help="force to convert the dtype, some backends supports fp16 dtype better")
@@ -162,10 +164,32 @@ if __name__ == '__main__':
torch_device = torch.device(device_str)
is_glm = bool(re.search("chatglm", model_name.lower()))
- if is_glm:
- model = AutoModel.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)
+ low_cpu_mem_usage = False
+ model_cls = AutoModel if is_glm else AutoModelForCausalLM
+ if args.low_cpu_mem_mode == 2:
+ from auto_round.layer_wise.utils import load_model_with_hooks
+ model = load_model_with_hooks(
+ model_name,
+ model_cls,
+ device=device_str,
+ clean_weight=True,
+ saved_path=os.path.join(args.output_dir, "layer_wise_tmp"),
+ torch_dtype=torch_dtype,
+ trust_remote_code=not args.disable_trust_remote_code
+ )
+ elif args.low_cpu_mem_mode == 1:
+ from auto_round.layer_wise.utils import load_empty_model
+ low_cpu_mem_usage = True
+ model = load_empty_model(
+ model_name,
+ model_cls,
+ device=device_str,
+ saved_path=os.path.join(args.output_dir, "layer_wise_tmp"),
+ torch_dtype=torch_dtype,
+ trust_remote_code=not args.disable_trust_remote_code
+ )
else:
- model = AutoModelForCausalLM.from_pretrained(
+ model = model_cls.from_pretrained(
model_name, low_cpu_mem_usage=True, torch_dtype=torch_dtype,
trust_remote_code=not args.disable_trust_remote_code
)
@@ -256,9 +280,13 @@ if __name__ == '__main__':
low_gpu_mem_usage=args.low_gpu_mem_usage,
seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,
scale_dtype=args.scale_dtype, layer_config=layer_config,
- enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits)
+ enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits,
+ low_cpu_mem_usage=low_cpu_mem_usage)
model, _ = autoround.quantize()
model_name = args.model_name.rstrip("/")
+ if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:
+ import shutil
+ shutil.rmtree(os.path.join(args.output_dir, "layer_wise_tmp"), ignore_errors=True)
model.eval()
if "cpu" not in device_str:
diff --git a/test/test_low_cpu_mem.py b/test/test_low_cpu_mem.py
new file mode 100644
index 000000000..3f8dd0211
--- /dev/null
+++ b/test/test_low_cpu_mem.py
@@ -0,0 +1,107 @@
+import shutil
+import sys
+import os
+import unittest
+sys.path.insert(0, "..")
+
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from auto_round.layer_wise.utils import (
+ load_model_with_hooks,
+ load_empty_model,
+ get_layers_before_block,
+ layer_wise_load,
+ layer_wise_save,
+ )
+
+from auto_round import AutoRound
+
+
+class LLMDataLoader:
+ def __init__(self):
+ self.batch_size = 1
+
+ def __iter__(self):
+ for i in range(2):
+ yield torch.ones([1, 10], dtype=torch.long)
+
+class TestLowCPUMem(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ self.model_name = "facebook/opt-125m"
+ self.saved_path = './test_tmp_saved'
+ self.ori_model = AutoModelForCausalLM.from_pretrained(self.model_name, trust_remote_code=True)
+ self.model = load_model_with_hooks(self.model_name, AutoModelForCausalLM, saved_path=self.saved_path, device='cpu')
+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ self.llm_dataloader = LLMDataLoader()
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree(self.saved_path, ignore_errors=True)
+
+ def test_default(self):
+ self.assertTrue(self.model.device.type, 'meta')
+
+ layers = get_layers_before_block(self.model)
+ self.assertEqual(layers[0][0], 'model.decoder.embed_tokens')
+
+ # test get_weight bias
+ self.assertTrue(torch.equal(
+ self.model.model.decoder.layers[0].self_attn.k_proj.get_weight(),
+ self.ori_model.model.decoder.layers[0].self_attn.k_proj.weight,
+ ))
+ self.assertTrue(torch.equal(
+ self.model.model.decoder.layers[0].self_attn.k_proj.get_bias(),
+ self.ori_model.model.decoder.layers[0].self_attn.k_proj.bias,
+ ))
+
+ # test hooks
+ text = ["Hello, my dog is cute"]
+ input = self.tokenizer(text)
+ for key in input:
+ input[key] = torch.tensor(input[key])
+ ori_output = self.ori_model(**input)
+ output = self.model(**input)
+ self.assertTrue(torch.equal(ori_output[0], output[0]))
+
+ # test save and load
+ layer_wise_save(self.model, self.saved_path)
+ state_dict = layer_wise_load(self.saved_path)
+ self.assertTrue(torch.equal(
+ state_dict['lm_head.weight'],
+ self.ori_model.lm_head.weight
+ ))
+
+ # test layer-wise auto_round
+ bits, group_size, sym = 4, 128, False
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ device='cpu',
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ autoround.quantize()
+
+ # test block-wise auto_round
+ self.model = load_empty_model(self.model_name, AutoModelForCausalLM, saved_path=self.saved_path, device='cpu')
+ bits, group_size, sym = 4, 128, False
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ device='cpu',
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ autoround.quantize()
+
+if __name__ == "__main__":
+ unittest.main()
\ No newline at end of file