Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix: serialize PyTorch device transfers with Kompress execution (#3601)

lorenzozane3dfa7daadbab
diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.py
index 29d0486ef..ac09291b8 100644
--- a/headroom/transforms/kompress_compressor.py
+++ b/headroom/transforms/kompress_compressor.py
@@ -933,8 +933,6 @@ def _validate_pytorch_device(model: Any, tokenizer: Any, device: str) -> None:
padding=True,
return_tensors="pt",
)
- input_ids = encoding["input_ids"].to(device)
- attention_mask = encoding["attention_mask"].to(device)
semaphore, _wait_ms = _acquire_execution_slot(
"pytorch",
device,
@@ -943,6 +941,8 @@ def _validate_pytorch_device(model: Any, tokenizer: Any, device: str) -> None:
assert semaphore is not None
with contextlib.ExitStack() as stack:
stack.callback(semaphore.release)
+ input_ids = encoding["input_ids"].to(device)
+ attention_mask = encoding["attention_mask"].to(device)
scores = model.get_scores(input_ids, attention_mask)
_ = scores[0].detach().cpu()
@@ -1495,12 +1495,21 @@ class KompressCompressor(Transform):
)
input_ids = encoding["input_ids"]
attention_mask = encoding["attention_mask"]
- if not is_onnx:
- device = next(model.parameters()).device
- input_ids = input_ids.to(device)
- attention_mask = attention_mask.to(device)
+ device_type = _model_device_type(model, backend)
+ semaphore, _wait_ms = _acquire_execution_slot(
+ backend,
+ device_type,
+ timeout_seconds=None,
+ )
+ assert semaphore is not None
started = time.perf_counter()
- model.get_keep_mask(input_ids, attention_mask)
+ with contextlib.ExitStack() as stack:
+ stack.callback(semaphore.release)
+ if not is_onnx:
+ device = next(model.parameters()).device
+ input_ids = input_ids.to(device)
+ attention_mask = attention_mask.to(device)
+ model.get_keep_mask(input_ids, attention_mask)
return time.perf_counter() - started
def is_ready(self) -> bool:
@@ -1654,8 +1663,6 @@ class KompressCompressor(Transform):
if not is_onnx:
device = next(model.parameters()).device
- input_ids = input_ids.to(device)
- attention_mask = attention_mask.to(device)
request_remaining: float | None = None
if deadline_s:
@@ -1702,6 +1709,9 @@ class KompressCompressor(Transform):
with contextlib.ExitStack() as stack:
stack.callback(semaphore.release)
+ if not is_onnx:
+ input_ids = input_ids.to(device)
+ attention_mask = attention_mask.to(device)
inference_started = time.perf_counter()
if target_ratio is not None:
scores = model.get_scores(input_ids, attention_mask)
@@ -2065,8 +2075,6 @@ class KompressCompressor(Transform):
if not is_onnx:
device = next(model.parameters()).device
- input_ids = input_ids.to(device)
- attention_mask = attention_mask.to(device)
request_remaining: float | None = None
if deadline_s:
@@ -2103,6 +2111,9 @@ class KompressCompressor(Transform):
with contextlib.ExitStack() as stack:
stack.callback(semaphore.release)
+ if not is_onnx:
+ input_ids = input_ids.to(device)
+ attention_mask = attention_mask.to(device)
inference_started = time.perf_counter()
scores = model.get_scores(input_ids, attention_mask)
inference_ms += (time.perf_counter() - inference_started) * 1000
diff --git a/tests/test_kompress_mps_serialization.py b/tests/test_kompress_mps_serialization.py
new file mode 100644
index 000000000..e3afff808
--- /dev/null
+++ b/tests/test_kompress_mps_serialization.py
@@ -0,0 +1,149 @@
+"""Device transfers must share Kompress's execution slot with inference."""
+
+import threading
+
+import headroom.transforms.kompress_compressor as kc
+
+
+def test_validation_device_copy_holds_execution_slot(monkeypatch):
+ semaphore = threading.BoundedSemaphore(1)
+ copy_saw_slot: list[bool] = []
+
+ class FakeTensor:
+ def to(self, _device):
+ acquired = semaphore.acquire(blocking=False)
+ if acquired:
+ semaphore.release()
+ copy_saw_slot.append(not acquired)
+ return self
+
+ class FakeEncoding(dict):
+ def __init__(self):
+ super().__init__(input_ids=FakeTensor(), attention_mask=FakeTensor())
+
+ class FakeTokenizer:
+ def __call__(self, *_args, **_kwargs):
+ return FakeEncoding()
+
+ class FakeScore:
+ def detach(self):
+ return self
+
+ def cpu(self):
+ return self
+
+ class FakeModel:
+ def get_scores(self, input_ids, attention_mask):
+ return [FakeScore()]
+
+ monkeypatch.setattr(kc, "_execution_semaphore", lambda *_args, **_kwargs: semaphore)
+
+ kc._validate_pytorch_device(FakeModel(), FakeTokenizer(), "mps")
+
+ assert copy_saw_slot == [True, True]
+
+
+def test_single_and_batch_device_copies_hold_execution_slot(monkeypatch):
+ semaphore = threading.BoundedSemaphore(1)
+ copy_saw_slot: list[bool] = []
+
+ class FakeTensor:
+ def __init__(self, row_lengths):
+ self.row_lengths = row_lengths
+
+ def to(self, _device):
+ acquired = semaphore.acquire(blocking=False)
+ if acquired:
+ semaphore.release()
+ copy_saw_slot.append(not acquired)
+ return self
+
+ class FakeEncoding(dict):
+ def __init__(self, row_lengths):
+ super().__init__(
+ input_ids=FakeTensor(row_lengths),
+ attention_mask=FakeTensor(row_lengths),
+ )
+ self.row_lengths = row_lengths
+
+ def word_ids(self, batch_index=0):
+ return list(range(self.row_lengths[batch_index]))
+
+ class FakeTokenizer:
+ def __call__(self, words, **_kwargs):
+ rows = words if words and isinstance(words[0], list) else [words]
+ return FakeEncoding([len(row) for row in rows])
+
+ class FakeScores:
+ def __init__(self, row_length):
+ self.values = [float(index) for index in range(row_length)]
+
+ def cpu(self):
+ return self.values
+
+ class FakeParameter:
+ device = "mps"
+
+ class FakeModel:
+ def parameters(self):
+ return iter([FakeParameter()])
+
+ def get_scores(self, input_ids, _attention_mask):
+ return [FakeScores(row_length) for row_length in input_ids.row_lengths]
+
+ model = FakeModel()
+ monkeypatch.setattr(kc, "_execution_semaphore", lambda *_args, **_kwargs: semaphore)
+ monkeypatch.setattr(
+ kc,
+ "_load_kompress",
+ lambda *args, **kwargs: (model, FakeTokenizer(), "pytorch"),
+ )
+
+ compressor = kc.KompressCompressor(kc.KompressConfig(min_input_words=10, enable_ccr=False))
+ compressor.compress(" ".join(["word"] * 20), target_ratio=0.5)
+ assert copy_saw_slot == [True, True]
+
+ copy_saw_slot.clear()
+ monkeypatch.setattr(
+ kc.KompressCompressor,
+ "_should_use_sequential_fallback",
+ lambda self: False,
+ )
+ compressor.compress_batch(
+ [" ".join(["word"] * 20), " ".join(["token"] * 20)],
+ target_ratio=0.5,
+ )
+ assert copy_saw_slot == [True, True]
+
+
+def test_canary_device_copy_holds_execution_slot(monkeypatch):
+ semaphore = threading.BoundedSemaphore(1)
+ copy_saw_slot: list[bool] = []
+
+ class FakeTensor:
+ def to(self, _device):
+ acquired = semaphore.acquire(blocking=False)
+ if acquired:
+ semaphore.release()
+ copy_saw_slot.append(not acquired)
+ return self
+
+ class FakeTokenizer:
+ def __call__(self, *_args, **_kwargs):
+ return {"input_ids": FakeTensor(), "attention_mask": FakeTensor()}
+
+ class FakeParameter:
+ device = "mps"
+
+ class FakeModel:
+ def parameters(self):
+ return iter([FakeParameter()])
+
+ def get_keep_mask(self, input_ids, attention_mask):
+ return []
+
+ monkeypatch.setattr(kc, "_execution_semaphore", lambda *_args, **_kwargs: semaphore)
+
+ kc.KompressCompressor()._timed_canary(FakeModel(), FakeTokenizer(), "pytorch")
+
+ assert copy_saw_slot == [True, True]