Skip to content

AirLLM 2.8T LLM inference with single 4GB GPU

Rearchitect core to hook-based layer streaming (Phase 2)

Yu Lie42a0fecee1c
diff --git a/air_llm/airllm/airllm_base.py b/air_llm/airllm/airllm_base.py
index f7449b2..411373b 100644
--- a/air_llm/airllm/airllm_base.py
+++ b/air_llm/airllm/airllm_base.py
@@ -6,12 +6,10 @@ import time
from concurrent.futures import ThreadPoolExecutor
import torch
-from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer, AutoModel, GenerationMixin, LlamaForCausalLM, GenerationConfig
-from transformers.modeling_outputs import CausalLMOutputWithPast
+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer, GenerationConfig
from accelerate import init_empty_weights
-
from accelerate.utils.modeling import set_module_tensor_to_device
-from transformers.quantizers import AutoHfQuantizer, HfQuantizer
+from transformers.quantizers import AutoHfQuantizer
from .profiler import LayeredProfiler
@@ -27,39 +25,32 @@ except ImportError:
bitsandbytes_installed = False
+class AirLLMBaseModel:
+ """
+ Memory-frugal wrapper around a Hugging Face ``*ForCausalLM`` model.
-try:
- from transformers.cache_utils import Cache, DynamicCache
-
- cache_utils_installed = True
- print('>>>> cache_utils installed')
-except ImportError:
- cache_utils_installed = False
-
-
-
+ The checkpoint is split into per-layer shards on disk. The real transformers model is
+ instantiated on the ``meta`` device (no memory used) and owns the full forward / generation
+ logic. AirLLM only attaches forward hooks to each big module (embeddings, every decoder
+ layer, the final norm and the lm_head) to stream that module's weights disk -> GPU right
+ before it runs and free them right after, prefetching the next module on a worker thread.
+ Because transformers drives the forward pass, AirLLM no longer needs to track per-architecture
+ attention/rotary/cache details: new model architectures work as soon as transformers supports
+ them.
+ """
-
-class AirLLMBaseModel(GenerationMixin):
-
- # customize layer names here
+ # Subclasses override this to point at non-standard module names.
def set_layer_names_dict(self):
self.layer_names_dict = {'embed': 'model.embed_tokens',
- 'layer_prefix': 'model.layers',
- 'norm': 'model.norm',
- 'lm_head': 'lm_head',}
-
-
+ 'layer_prefix': 'model.layers',
+ 'norm': 'model.norm',
+ 'lm_head': 'lm_head'}
def __init__(self, model_local_path_or_repo_id, device="cuda:0", dtype=torch.float16, max_seq_len=512,
layer_shards_saving_path=None, profiling_mode=False, compression=None,
hf_token=None, prefetching=True, delete_original=False):
"""
- Sharded version of LlamaForCausalLM : the model is splitted into layer shards to reduce GPU memory usage.
- During the forward pass, the inputs are processed layer by layer, and the GPU memory is freed after each layer.
- To avoid loading the layers multiple times, we could save all the intermediate activations in RAM.
-
Parameters
----------
model_local_path_or_repo_id : str or Path
@@ -69,158 +60,155 @@ class AirLLMBaseModel(GenerationMixin):
dtype : torch.dtype, optional
dtype, by default torch.float16
max_seq_len : int, optional
- max seq lenght, by default 512
+ max seq length, by default 512
layer_shards_saving_path : str, optional
- optional path to save layered shards model file, by default just save to the local cache of model, subdir named splitted_model will be saved
- profiling_mode : book, optional
- if to profile the model loading time, default to False
- compression: str, optinal
- setting to '4bit' or '8bit' to enable compression from 16 bits to 4 bits/8 bits which speeed up 4x or 2x inference time with a tiny accuracy loss.
+ optional path to save the splitted shards, by default next to the model cache
+ profiling_mode : bool, optional
+ whether to profile the model loading time, default False
+ compression: str, optional
+ '4bit' or '8bit' to enable block-wise quantization of the on-disk shards
hf_token: str, optional
- huggingface api token could be provided, by default None
+ huggingface api token
+ prefetching: bool, optional
+ overlap the next layer's disk load with the current layer's compute
+ delete_original: bool, optional
+ delete the original downloaded checkpoint after splitting to save disk space
"""
-
self.profiling_mode = profiling_mode
self.profiler = LayeredProfiler()
self.total_disk_loading_time = None
self.total_gpu_loading_time = None
self.total_compression_overhead_time = None
- self._supports_cache_class = False
self.hf_quantizer = None
- if compression is not None:
- if not bitsandbytes_installed:
- raise ImportError('WARNING: bitsandbytes not found. Compression needs bitsandbytes. To use compression, please install bitsandbytes: `pip install bitsandbytes`')
-
+ if compression is not None and not bitsandbytes_installed:
+ raise ImportError('WARNING: bitsandbytes not found. Compression needs bitsandbytes. '
+ 'To use compression, please install bitsandbytes: `pip install bitsandbytes`')
self.compression = compression
self.hf_token = hf_token
- # Save parameters
-
self.set_layer_names_dict()
+ self.model_local_path, self.checkpoint_path = find_or_create_local_splitted_path(
+ model_local_path_or_repo_id,
+ layer_shards_saving_path,
+ compression=compression,
+ layer_names=self.layer_names_dict,
+ hf_token=hf_token,
+ delete_original=delete_original)
- self.model_local_path, self.checkpoint_path = find_or_create_local_splitted_path(model_local_path_or_repo_id,
- layer_shards_saving_path,
- compression=compression,
- layer_names=self.layer_names_dict,
- hf_token=hf_token,
- delete_original=delete_original)
self.running_device = device
self.device = torch.device(self.running_device)
self.running_dtype = dtype
self.dtype = self.running_dtype
- # Create model
if hf_token is not None:
self.config = AutoConfig.from_pretrained(self.model_local_path, token=hf_token, trust_remote_code=True)
else:
self.config = AutoConfig.from_pretrained(self.model_local_path, trust_remote_code=True)
self.generation_config = self.get_generation_config()
- #print(f"using generation_config: {self.generation_config}")
-
self.tokenizer = self.get_tokenizer(hf_token=hf_token)
+ # prefetch executor / state
+ self.prefetching = prefetching
+ if self.compression is not None and self.prefetching:
+ print("prefetching is not supported together with compression for now; disabling prefetching.")
+ self.prefetching = False
+ self._executor = ThreadPoolExecutor(max_workers=1) if self.prefetching else None
+ self._prefetch_future = None
+ self._prefetched_idx = None
self.init_model()
- # get layer count:
+ # compute layer count from the instantiated model
model_attr = self.model
for attr_name in self.layer_names_dict["layer_prefix"].split("."):
model_attr = getattr(model_attr, attr_name)
-
layers_count = len(model_attr)
-
- self.layer_names = [self.layer_names_dict['embed']] + [f'{self.layer_names_dict["layer_prefix"]}.{i}' for i in
- range(layers_count)] + \
+ self.layer_names = [self.layer_names_dict['embed']] + \
+ [f'{self.layer_names_dict["layer_prefix"]}.{i}' for i in range(layers_count)] + \
[self.layer_names_dict['norm'], self.layer_names_dict['lm_head']]
self.max_seq_len = max_seq_len
- self.main_input_name = "input_ids"
-
- # model weights prefetch cuda stream
- self.prefetching = prefetching
+ self.set_layers_from_layer_names()
+ self._install_streaming_hooks()
- if self.compression is not None:
- self.prefetching = False
- print(f"not support prefetching for compression for now. loading with no prepetching mode.")
+ # ---- customization hooks for subclasses -------------------------------------------------
- # this operation should run only if gpu is available
- if prefetching and device.startswith("cuda"):
- self.stream = torch.cuda.Stream()
- else:
- self.stream = None
-
- # if derived class needs to create generation config differently, like Mistrial, this function can be overridden
def get_generation_config(self):
- # protective on generation config
-
try:
return GenerationConfig.from_pretrained(self.model_local_path)
- except Exception as e:
+ except Exception:
return GenerationConfig()
- # a chance to customize tokenizer
def get_tokenizer(self, hf_token=None):
if hf_token is not None:
return AutoTokenizer.from_pretrained(self.model_local_path, token=hf_token, trust_remote_code=True)
else:
return AutoTokenizer.from_pretrained(self.model_local_path, trust_remote_code=True)
- def get_use_better_transformer(self):
- return True
+ # ---- model construction -----------------------------------------------------------------
def init_model(self):
-
- # Load meta model (no real memory used). Prefer PyTorch's SDPA (flash / memory-efficient
- # attention), which is built into transformers now. BetterTransformer is deprecated and
- # has been removed from optimum, so we no longer depend on it.
+ # Build the real model on meta (no memory). include_buffers=False so non-persistent
+ # buffers such as rotary inv_freq are actually computed (they aren't in the checkpoint).
self.model = None
-
try:
- with init_empty_weights():
+ with init_empty_weights(include_buffers=False):
self.model = AutoModelForCausalLM.from_config(
self.config, attn_implementation="sdpa", trust_remote_code=True)
except (ValueError, TypeError) as e:
- # This architecture may not implement an SDPA path; fall back to the default attention.
- print(f"attn_implementation='sdpa' not available ({e}), creating model with default attn implementation")
- del self.model
- clean_memory()
+ print(f"attn_implementation='sdpa' not available ({e}), using default attn implementation")
self.model = None
-
if self.model is None:
- with init_empty_weights():
+ with init_empty_weights(include_buffers=False):
self.model = AutoModelForCausalLM.from_config(self.config, trust_remote_code=True)
quantization_config = getattr(self.config, "quantization_config", None)
-
if quantization_config is not None:
self.hf_quantizer = AutoHfQuantizer.from_config(quantization_config, pre_quantized=True)
device_map = self.hf_quantizer.update_device_map(None)
- self.hf_quantizer.preprocess_model(model = self.model, device_map = device_map)
+ self.hf_quantizer.preprocess_model(model=self.model, device_map=device_map)
self.model.eval()
self.model.tie_weights()
+ self.model.generation_config = self.generation_config
- self.set_layers_from_layer_names()
-
- # Move buffers to device (not that much GPU memory used)
+ # Move all (already-materialized) buffers to the running device, preserving their dtype.
+ # This includes rotary inv_freq, which transformers computes once at the model level and
+ # passes down to every decoder layer.
for buffer_name, buffer in self.model.named_buffers():
- set_module_tensor_to_device(self.model, buffer_name, self.running_device, value=buffer,
- dtype=self.running_dtype)
+ if buffer is not None and buffer.device.type != 'meta':
+ set_module_tensor_to_device(self.model, buffer_name, self.running_device, value=buffer)
- if 'rotary_pos_emb' in self.layer_names_dict:
- # for glm keep rotary_pos_emb in gpu
- self.load_rotary_pos_emb_to_device()
+ # Force the model to report the running (cuda) device even though its parameters live on
+ # meta between layer executions, so transformers' generation utilities place inputs/cache
+ # tensors on the right device.
+ self._patch_device_property()
- def set_layers_from_layer_names(self):
+ def _patch_device_property(self):
+ running_device = torch.device(self.running_device)
+ running_dtype = self.running_dtype
+ base_cls = type(self.model)
+
+ class _AirLLMRuntimeModel(base_cls):
+ @property
+ def device(self):
+ return running_device
+ @property
+ def dtype(self):
+ return running_dtype
+
+ self.model.__class__ = _AirLLMRuntimeModel
+
+ def set_layers_from_layer_names(self):
self.layers = []
model_attr = self.model
@@ -231,7 +219,6 @@ class AirLLMBaseModel(GenerationMixin):
model_attr = self.model
for attr_name in self.layer_names_dict["layer_prefix"].split("."):
model_attr = getattr(model_attr, attr_name)
-
self.layers.extend(list(model_attr))
model_attr = self.model
@@ -244,382 +231,92 @@ class AirLLMBaseModel(GenerationMixin):
model_attr = getattr(model_attr, attr_name)
self.layers.append(model_attr)
- def load_rotary_pos_emb_to_device(self):
- state_dict = load_layer(self.checkpoint_path, self.layer_names_dict['rotary_pos_emb'])
- self.move_layer_to_device(state_dict)
+ # ---- weight streaming -------------------------------------------------------------------
def load_layer_to_cpu(self, layer_name):
-
t = time.time()
-
load_layer_output = load_layer(self.checkpoint_path, layer_name, self.profiling_mode)
elapsed_time = time.time() - t
if self.profiling_mode:
state_dict, compression_time = load_layer_output
disk_loading_time = elapsed_time - compression_time
-
self.profiler.add_profiling_time('load_safe_tensor', disk_loading_time)
-
self.profiler.add_profiling_time('compression_time', compression_time)
else:
state_dict = load_layer_output
- # pin memory:
- if self.prefetching:
- t = time.time()
- if torch.cuda.is_available(): # Check if CUDA is available
- for k in state_dict.keys():
- state_dict[k].pin_memory()
- else:
- # For CPU, no action is needed, but you could optionally add a log or message
- print("Prefetching is enabled, but no pin_memory operation is needed for CPU.")
-
- elapsed_time = time.time() - t
- if self.profiling_mode:
- self.profiler.add_profiling_time('pin_memory_to_trigger_load', elapsed_time)
+ if self.prefetching and torch.cuda.is_available():
+ for k in state_dict.keys():
+ state_dict[k].pin_memory()
return state_dict
def move_layer_to_device(self, state_dict):
- layers = []
- for param_name, param in state_dict.items():
- if self.hf_quantizer is None:
- layers.append(param_name)
- else:
- if '.weight' in param_name:
- layer_name = param_name[:param_name.index(".weight") + len(".weight")]
- if layer_name not in layers:
- layers.append(layer_name)
-
- for param_name in layers:
+ moved = []
+ for param_name in self._param_names_from_state_dict(state_dict):
if (self.hf_quantizer is None or
- not self.hf_quantizer.check_quantized_param(self.model, param_value=None, param_name=param_name, state_dict={})
- ):
- set_module_tensor_to_device(self.model, param_name, self.running_device, value=state_dict[param_name],
- dtype=self.running_dtype,
- )
+ not self.hf_quantizer.check_quantized_param(self.model, param_value=None,
+ param_name=param_name, state_dict={})):
+ set_module_tensor_to_device(self.model, param_name, self.running_device,
+ value=state_dict[param_name], dtype=self.running_dtype)
else:
- torch_dtype = self.hf_quantizer.update_torch_dtype(None)
- self.hf_quantizer.create_quantized_param(self.model, state_dict[param_name], param_name, self.running_device, state_dict)
- return layers
-
- # make GenerationMixin happy
- def can_generate(self):
- return True
-
- def prepare_inputs_for_generation(
- self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs
- ):
- if past_key_values is not None:
- past_length = self.get_past_key_values_cache_seq_len(past_key_values) #[0][0].shape[2]
-
- # Some generation methods already pass only the last input ID
- if input_ids.shape[1] > past_length:
- remove_prefix_length = past_length
+ self.hf_quantizer.create_quantized_param(self.model, state_dict[param_name], param_name,
+ self.running_device, state_dict)
+ moved.append(param_name)
+ return moved
+
+ def _param_names_from_state_dict(self, state_dict):
+ if self.hf_quantizer is None:
+ return list(state_dict.keys())
+ names = []
+ for param_name in state_dict.keys():
+ if '.weight' in param_name:
+ layer_name = param_name[:param_name.index(".weight") + len(".weight")]
+ if layer_name not in names:
+ names.append(layer_name)
else:
- # Default to old behavior: keep only final ID
- remove_prefix_length = input_ids.shape[1] - 1
-
- input_ids = input_ids[:, remove_prefix_length:]
-
- position_ids = kwargs.get("position_ids", None)
- if attention_mask is not None and position_ids is None:
- # create position_ids on the fly for batch generation
- position_ids = attention_mask.long().cumsum(-1) - 1
- position_ids.masked_fill_(attention_mask == 0, 1)
- if past_key_values:
- position_ids = position_ids[:, -input_ids.shape[1]:]
-
- # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
- if inputs_embeds is not None and past_key_values is None:
- model_inputs = {"inputs_embeds": inputs_embeds}
+ names.append(param_name)
+ return names
+
+ def _install_streaming_hooks(self):
+ # All modules execute in this order during a forward pass: embed -> layers -> norm -> lm_head.
+ for idx, (name, module) in enumerate(zip(self.layer_names, self.layers)):
+ module._airllm_idx = idx
+ module.register_forward_pre_hook(self._pre_hook)
+ module.register_forward_hook(self._post_hook)
+
+ def _pre_hook(self, module, args):
+ idx = module._airllm_idx
+
+ if self.prefetching and self._prefetch_future is not None and self._prefetched_idx == idx:
+ state_dict = self._prefetch_future.result()
+ self._prefetch_future = None
else:
- model_inputs = {"input_ids": input_ids}
-
- model_inputs.update(
- {
- "position_ids": position_ids,
- "past_key_values": past_key_values,
- "use_cache": kwargs.get("use_cache"),
- "attention_mask": attention_mask,
- }
- )
- return model_inputs
-
- def __call__(self, *args, **kwargs):
- return self.forward(*args, **kwargs)
-
- def get_past_key_values_cache_seq_len(self, past_key_values):
- return past_key_values[0][0].shape[2]
- def get_sequence_len(self, seq):
- return seq.shape[1]
-
- def get_pos_emb_args(self, len_p, len_s):
- return {}
-
- def get_past_key_value_args(self, k_cache, v_cache):
- return {'past_key_value': (k_cache, v_cache)}
-
- def get_attention_mask_args(self, full_attention_mask, len_p, len_s):
- return {'attention_mask': full_attention_mask[:, :, -len_s:, -len_p - len_s:]}
-
- def get_position_ids_args(self, full_position_ids, len_p, len_s):
-
- return {'position_ids': full_position_ids[:, len_p:len_p + len_s]}
-
+ state_dict = self.load_layer_to_cpu(self.layer_names[idx])
- def run_lm_head(self, layer, seq):
- return layer(seq).float()
+ module._airllm_moved = self.move_layer_to_device(state_dict)
- def run_norm(self, layer, seq):
- return layer(seq)
+ if self.prefetching and (idx + 1) < len(self.layer_names):
+ self._prefetch_future = self._executor.submit(self.load_layer_to_cpu, self.layer_names[idx + 1])
+ self._prefetched_idx = idx + 1
- def forward(
- self,
- input_ids: torch.LongTensor = None,
- attention_mask: Optional[torch.Tensor] = None,
- position_ids: Optional[torch.LongTensor] = None,
- past_key_values: Optional[List[torch.FloatTensor]] = None,
- inputs_embeds: Optional[torch.FloatTensor] = None,
- labels: Optional[torch.LongTensor] = None,
- use_cache: Optional[bool] = None,
- output_attentions: Optional[bool] = None,
- output_hidden_states: Optional[bool] = None,
- return_dict: Optional[bool] = None,
- ) -> Union[Tuple, CausalLMOutputWithPast]:
-
- if cache_utils_installed:
- # we don't support kv cache for new version yet
- use_cache = False
-
- if self.profiling_mode:
- self.profiler.clear_profiling_time()
-
- forward_start = time.process_time()
- forward_start_wall = time.time()
-
- # Reboot the model to make sure buffers are loaded and memory is clean
- del self.model
+ def _post_hook(self, module, args, output):
+ if self.hf_quantizer is not None:
+ for param_name in getattr(module, '_airllm_moved', []):
+ set_module_tensor_to_device(self.model, param_name, 'meta')
+ else:
+ module.to('meta')
clean_memory()
- self.init_model()
-
- batch = [input_ids_unit.to(self.running_device).unsqueeze(0) for input_ids_unit in input_ids]
- n_seq = len(batch[0])
-
- # Create attention mask for the largest input, and position ids to use KV cache
- attention_mask = torch.ones(self.max_seq_len, self.max_seq_len)
- attention_mask = attention_mask.triu(diagonal=1)[None, None, ...] == 0
- attention_mask = attention_mask.to(self.running_device)
- position_ids = torch.arange(self.max_seq_len, dtype=torch.long, device=self.running_device)[None, :]
-
- kv_cache_list = [] if use_cache else None
- if use_cache:
- for x in self.layers:
- kv_cache_list.append(([], []))
- all_hidden_states = [] * len(self.layers) if output_hidden_states else None
- all_self_attns = [] * len(self.layers) if output_attentions else None
-
- with torch.inference_mode(), ThreadPoolExecutor() as executor:
-
- # Load first layer
- if self.prefetching:
- #with torch.cuda.stream(self.stream):
- #state_dict = self.load_layer_to_cpu(self.layer_names[0])
- future = executor.submit(self.load_layer_to_cpu, self.layer_names[0])
-
-
- for i, (layer_name, layer) in tqdm(enumerate(zip(self.layer_names, self.layers)),
- desc=f'running layers({self.running_device})',
- total=len(self.layers)):
-
- if self.prefetching:
- if self.profiling_mode:
- t = time.time()
- # Load current layer and prepare next layer
- state_dict = future.result()
- #torch.cuda.current_stream().wait_stream(self.stream)
- if self.profiling_mode:
- elapsed_time = time.time() - t
- self.profiler.add_profiling_time('load_safe_tensor_cpu_wait', elapsed_time)
-
- #for param_name, param in state_dict.items():
- # state_dict[param_name] = param.to('cuda', non_blocking=True)
-
- if self.profiling_mode:
- t = time.time()
- moved_layers = self.move_layer_to_device(state_dict)
- if self.profiling_mode:
- elapsed_time = time.time() - t
- self.profiler.add_profiling_time('create_layer_from_state_dict', elapsed_time)
-
- # kick off next layer loading
-
- if (i + 1) < len(self.layer_names):
- #with torch.cuda.stream(self.stream):
- #state_dict = self.load_layer_to_cpu(self.layer_names[i + 1])
- if self.profiling_mode:
- t = time.time()
- future = executor.submit(self.load_layer_to_cpu, self.layer_names[i+1])
- #for param_name, param in state_dict.items():
- # state_dict[param_name] = param.to('cuda', non_blocking=True)
-
- if self.profiling_mode:
- elapsed_time = time.time() - t
- self.profiler.add_profiling_time('kick_off_load_cpu', elapsed_time)
-
- else:
- state_dict = self.load_layer_to_cpu(layer_name)
- if self.profiling_mode:
- t = time.time()
- moved_layers = self.move_layer_to_device(state_dict)
- if self.profiling_mode:
- elapsed_time = time.time() - t
- self.profiler.add_profiling_time('create_layer_from_safe_tensor', elapsed_time)
-
- # Run layer
-
- for j, seq in enumerate(batch):
-
- if layer_name == self.layer_names_dict['embed']:
- batch[j] = layer(seq)
- elif layer_name == self.layer_names_dict['norm']:
- #batch[j] = layer(seq[torch.arange(n_seq), batch_eos[j]][:, None])
- batch[j] = self.run_norm(layer, seq)
-
- if output_attentions:
- all_hidden_states[i].append(batch[j])
- elif layer_name == self.layer_names_dict['lm_head']:
- batch[j] = self.run_lm_head(layer, seq)
- else:
-
- if output_attentions:
- all_hidden_states[i].append(new_seq)
-
- if past_key_values is not None:
- # join past kv
- k_cache, v_cache = past_key_values[i - 1]
- len_p = self.get_past_key_values_cache_seq_len(past_key_values)
- len_s = self.get_sequence_len(seq)
-
- position_ids_args = self.get_position_ids_args(position_ids, len_p, len_s)
- attention_mask_args = self.get_attention_mask_args(attention_mask, len_p, len_s)
- past_key_value_args = self.get_past_key_value_args(k_cache, v_cache)
-
- kwargs = {'use_cache':True,
- }
-
- pos_embed_args = self.get_pos_emb_args(len_p, len_s)
- kwargs = {**kwargs, **past_key_value_args, **pos_embed_args, **attention_mask_args,
- **position_ids_args}
-
-
- layer_outputs = layer(seq,
- **kwargs
- )
- new_seq = layer_outputs[0]
-
- if output_attentions:
- all_self_attns[i].append(layer_outputs[1])
-
- if use_cache:
- (k_cache, v_cache) = layer_outputs[2 if output_attentions else 1]
- kv_cache_list[i][0].append(k_cache)
- kv_cache_list[i][1].append(v_cache)
+ return output
+ # ---- delegation to the underlying transformers model ------------------------------------
- else:
- len_seq = self.get_sequence_len(seq)
+ def generate(self, *args, **kwargs):
+ return self.model.generate(*args, **kwargs)
+ def forward(self, *args, **kwargs):
+ return self.model(*args, **kwargs)
-
- pos_embed_args = self.get_pos_emb_args(0, len_seq)
- attention_mask_args = self.get_attention_mask_args(attention_mask, 0, len_seq)
- position_ids_args = self.get_position_ids_args(position_ids, 0, len_seq)
-
-
-
-
- if not use_cache:
-
- kwargs = {'use_cache': False,
- 'attention_mask': attention_mask[:, :, -len_seq:, -len_seq:],
- }
- kwargs = {**kwargs, **pos_embed_args, **attention_mask_args, **position_ids_args}
-
-
- new_seq = layer(seq, **kwargs)[0]
- else:
-
- kwargs = {'use_cache': True,
- 'attention_mask': attention_mask[:, :, -len_seq:, -len_seq:],
- }
- kwargs = {**kwargs, **pos_embed_args, **attention_mask_args, **position_ids_args}
-
- layer_out = layer(seq, **kwargs)
-
- # TODO: adopt Cache mechanism in 4.36
- new_seq, (k_cache, v_cache) = layer_out
- kv_cache_list[i][0].append(k_cache)
- kv_cache_list[i][1].append(v_cache)
-
- # print(f"k_cache sizes: {[len(x[1]) for x in kv_cache_list]}")
-
- batch[j] = new_seq
-
- if output_hidden_states:
- all_hidden_states += (torch.cat(batch, 0),)
-
- # Remove previous layer from memory (including buffers)
-
- if self.hf_quantizer is not None:
- for param_name in moved_layers:#param_name, param in state_dict.items():
- set_module_tensor_to_device(self.model, param_name,'meta')
- else:
- layer.to("meta")
-
- layer.to("meta")
- clean_memory() # proposed by CPMP
-
- logits = torch.cat(batch, 0)
- if use_cache:
- kv_cache_list = kv_cache_list[1:-2]
- for i in range(len(kv_cache_list)):
- # print(f"{i} - {kv_cache_list[i][0].shape}")
- kv_cache_list[i] = (torch.cat(kv_cache_list[i][0], 0), torch.cat(kv_cache_list[i][1], 0))
- #print(f"returning kvcache size: {kv_cache_list[0][0].shape}")
-
- if output_attentions:
- all_self_attns = all_self_attns[0:-2]
- for i in range(len(all_self_attns)):
- all_self_attns[i] = torch.cat(all_self_attns[i], 0)
-
- if output_hidden_states:
- all_hidden_states = all_hidden_states[0:-2]
- for i in range(len(all_hidden_states)):
- all_hidden_states[i] = torch.cat(all_hidden_states[i], 0)
-
- if not return_dict:
- return tuple(v for v in [logits,
- tuple(kv_cache_list) if kv_cache_list is not None else None,
- tuple(all_hidden_states) if all_hidden_states is not None else None,
- tuple(all_self_attns) if all_self_attns is not None else None] if v is not None)
- if self.profiling_mode:
- forward_elapsed_time = time.process_time() - forward_start
- forward_elapsed_time_wall = time.time() - forward_start_wall
- self.profiler.print_profiling_time()
-
-
- print(f"total infer process time(including all above plus gpu compute): {forward_elapsed_time:.04f}")
- print(f"total infer wall time(including all above plus gpu compute): {forward_elapsed_time_wall:.04f}")
-
- self.profiler.clear_profiling_time()
-
-
- return CausalLMOutputWithPast(
- loss=None,
- logits=logits,
- past_key_values=tuple(kv_cache_list) if kv_cache_list is not None else None,
- hidden_states=tuple(all_hidden_states) if all_hidden_states is not None else None,
- attentions=tuple(all_self_attns) if all_hidden_states is not None else None,
- )
\ No newline at end of file
+ def __call__(self, *args, **kwargs):
+ return self.model(*args, **kwargs)
diff --git a/air_llm/tests/test_streaming_gpu.py b/air_llm/tests/test_streaming_gpu.py
new file mode 100644
index 0000000..24b4601
--- /dev/null
+++ b/air_llm/tests/test_streaming_gpu.py
@@ -0,0 +1,106 @@
+"""
+Manual GPU test harness for AirLLM layer-streaming inference.
+
+Runs a model through AirLLM (one layer on the GPU at a time) and reports peak VRAM.
+Optionally caps the visible VRAM to emulate a small card, and/or compares the output
+against a normal full-load run of the same model for correctness.
+
+Examples
+--------
+# Tiny model, verify output matches a normal full-load run, report peak VRAM:
+python test_streaming_gpu.py --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 --compare
+
+# Emulate a 4GB card running a 7B model:
+python test_streaming_gpu.py --model Qwen/Qwen2.5-7B-Instruct --max-vram-gb 4
+
+# A real 70B on whatever card you have, capped to 8GB:
+python test_streaming_gpu.py --model meta-llama/Llama-3.3-70B-Instruct --max-vram-gb 8 \
+ --prompt "The capital of France is" --max-new-tokens 8
+"""
+import argparse
+import time
+
+import torch
+
+
+def cap_vram(max_vram_gb):
+ """Limit how much of the GPU this process may allocate, to emulate a smaller card."""
+ if max_vram_gb is None:
+ return
+ total = torch.cuda.get_device_properties(0).total_memory
+ frac = (max_vram_gb * (1024 ** 3)) / total
+ if frac >= 1.0:
+ print(f"requested cap {max_vram_gb}GB >= device memory; not capping")
+ return
+ torch.cuda.set_per_process_memory_fraction(frac, 0)
+ print(f"capped process VRAM to ~{max_vram_gb}GB ({frac*100:.1f}% of device)")
+
+
+def run_airllm(args):
+ from airllm import AutoModel
+
+ model = AutoModel.from_pretrained(args.model, compression=args.compression)
+ ids = model.tokenizer([args.prompt], return_tensors="pt",
+ return_attention_mask=False)["input_ids"].cuda()
+
+ torch.cuda.reset_peak_memory_stats()
+ t = time.time()
+ out = model.generate(ids, max_new_tokens=args.max_new_tokens, do_sample=False,
+ return_dict_in_generate=True)
+ elapsed = time.time() - t
+
+ text = model.tokenizer.decode(out.sequences[0])
+ peak = torch.cuda.max_memory_allocated() / 1e6
+ print(f"\n=== AirLLM ===")
+ print(f"output : {text!r}")
+ print(f"peak VRAM: {peak:.1f} MB")
+ print(f"time : {elapsed:.1f}s ({args.max_new_tokens} new tokens)")
+ return out.sequences[0].tolist()
+
+
+def run_reference(args):
+ """Full-load run for correctness comparison (only feasible for small models)."""
+ from transformers import AutoModelForCausalLM, AutoTokenizer
+
+ tok = AutoTokenizer.from_pretrained(args.model)
+ ref = AutoModelForCausalLM.from_pretrained(args.model, dtype=torch.float16).cuda().eval()
+ ids = tok([args.prompt], return_tensors="pt", return_attention_mask=False)["input_ids"].cuda()
+ out = ref.generate(ids, max_new_tokens=args.max_new_tokens, do_sample=False)
+ text = tok.decode(out[0])
+ print(f"\n=== Reference (full load) ===")
+ print(f"output : {text!r}")
+ del ref
+ torch.cuda.empty_cache()
+ return out[0].tolist()
+
+
+def main():
+ p = argparse.ArgumentParser()
+ p.add_argument("--model", required=True)
+ p.add_argument("--prompt", default="The capital of France is")
+ p.add_argument("--max-new-tokens", type=int, default=12)
+ p.add_argument("--max-vram-gb", type=float, default=None)
+ p.add_argument("--compression", default=None, choices=[None, "4bit", "8bit"])
+ p.add_argument("--compare", action="store_true",
+ help="also run a full-load reference and assert outputs match")
+ args = p.parse_args()
+
+ ref_seq = None
+ if args.compare:
+ # run reference first (before capping) so the full model fits
+ ref_seq = run_reference(args)
+
+ cap_vram(args.max_vram_gb)
+ air_seq = run_airllm(args)
+
+ if ref_seq is not None:
+ match = ref_seq == air_seq
+ print(f"\n=== correctness: {'MATCH' if match else 'MISMATCH'} ===")
+ if not match:
+ print(f"ref : {ref_seq}")
+ print(f"air : {air_seq}")
+ raise SystemExit(1)
+
+
+if __name__ == "__main__":
+ main()