Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix: serialize PyTorch device transfers with Kompress execution (#3601)
lorenzozane3dfa7daadbab
diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.pyindex 29d0486ef..ac09291b8 100644--- a/headroom/transforms/kompress_compressor.py+++ b/headroom/transforms/kompress_compressor.py@@ -933,8 +933,6 @@ def _validate_pytorch_device(model: Any, tokenizer: Any, device: str) -> None:padding=True,return_tensors="pt",)- input_ids = encoding["input_ids"].to(device)- attention_mask = encoding["attention_mask"].to(device)semaphore, _wait_ms = _acquire_execution_slot("pytorch",device,@@ -943,6 +941,8 @@ def _validate_pytorch_device(model: Any, tokenizer: Any, device: str) -> None:assert semaphore is not Nonewith contextlib.ExitStack() as stack:stack.callback(semaphore.release)+ input_ids = encoding["input_ids"].to(device)+ attention_mask = encoding["attention_mask"].to(device)scores = model.get_scores(input_ids, attention_mask)_ = scores[0].detach().cpu()@@ -1495,12 +1495,21 @@ class KompressCompressor(Transform):)input_ids = encoding["input_ids"]attention_mask = encoding["attention_mask"]- if not is_onnx:- device = next(model.parameters()).device- input_ids = input_ids.to(device)- attention_mask = attention_mask.to(device)+ device_type = _model_device_type(model, backend)+ semaphore, _wait_ms = _acquire_execution_slot(+ backend,+ device_type,+ timeout_seconds=None,+ )+ assert semaphore is not Nonestarted = time.perf_counter()- model.get_keep_mask(input_ids, attention_mask)+ with contextlib.ExitStack() as stack:+ stack.callback(semaphore.release)+ if not is_onnx:+ device = next(model.parameters()).device+ input_ids = input_ids.to(device)+ attention_mask = attention_mask.to(device)+ model.get_keep_mask(input_ids, attention_mask)return time.perf_counter() - starteddef is_ready(self) -> bool:@@ -1654,8 +1663,6 @@ class KompressCompressor(Transform):if not is_onnx:device = next(model.parameters()).device- input_ids = input_ids.to(device)- attention_mask = attention_mask.to(device)request_remaining: float | None = Noneif deadline_s:@@ -1702,6 +1709,9 @@ class KompressCompressor(Transform):with contextlib.ExitStack() as stack:stack.callback(semaphore.release)+ if not is_onnx:+ input_ids = input_ids.to(device)+ attention_mask = attention_mask.to(device)inference_started = time.perf_counter()if target_ratio is not None:scores = model.get_scores(input_ids, attention_mask)@@ -2065,8 +2075,6 @@ class KompressCompressor(Transform):if not is_onnx:device = next(model.parameters()).device- input_ids = input_ids.to(device)- attention_mask = attention_mask.to(device)request_remaining: float | None = Noneif deadline_s:@@ -2103,6 +2111,9 @@ class KompressCompressor(Transform):with contextlib.ExitStack() as stack:stack.callback(semaphore.release)+ if not is_onnx:+ input_ids = input_ids.to(device)+ attention_mask = attention_mask.to(device)inference_started = time.perf_counter()scores = model.get_scores(input_ids, attention_mask)inference_ms += (time.perf_counter() - inference_started) * 1000diff --git a/tests/test_kompress_mps_serialization.py b/tests/test_kompress_mps_serialization.pynew file mode 100644index 000000000..e3afff808--- /dev/null+++ b/tests/test_kompress_mps_serialization.py@@ -0,0 +1,149 @@+"""Device transfers must share Kompress's execution slot with inference."""++import threading++import headroom.transforms.kompress_compressor as kc+++def test_validation_device_copy_holds_execution_slot(monkeypatch):+ semaphore = threading.BoundedSemaphore(1)+ copy_saw_slot: list[bool] = []++ class FakeTensor:+ def to(self, _device):+ acquired = semaphore.acquire(blocking=False)+ if acquired:+ semaphore.release()+ copy_saw_slot.append(not acquired)+ return self++ class FakeEncoding(dict):+ def __init__(self):+ super().__init__(input_ids=FakeTensor(), attention_mask=FakeTensor())++ class FakeTokenizer:+ def __call__(self, *_args, **_kwargs):+ return FakeEncoding()++ class FakeScore:+ def detach(self):+ return self++ def cpu(self):+ return self++ class FakeModel:+ def get_scores(self, input_ids, attention_mask):+ return [FakeScore()]++ monkeypatch.setattr(kc, "_execution_semaphore", lambda *_args, **_kwargs: semaphore)++ kc._validate_pytorch_device(FakeModel(), FakeTokenizer(), "mps")++ assert copy_saw_slot == [True, True]+++def test_single_and_batch_device_copies_hold_execution_slot(monkeypatch):+ semaphore = threading.BoundedSemaphore(1)+ copy_saw_slot: list[bool] = []++ class FakeTensor:+ def __init__(self, row_lengths):+ self.row_lengths = row_lengths++ def to(self, _device):+ acquired = semaphore.acquire(blocking=False)+ if acquired:+ semaphore.release()+ copy_saw_slot.append(not acquired)+ return self++ class FakeEncoding(dict):+ def __init__(self, row_lengths):+ super().__init__(+ input_ids=FakeTensor(row_lengths),+ attention_mask=FakeTensor(row_lengths),+ )+ self.row_lengths = row_lengths++ def word_ids(self, batch_index=0):+ return list(range(self.row_lengths[batch_index]))++ class FakeTokenizer:+ def __call__(self, words, **_kwargs):+ rows = words if words and isinstance(words[0], list) else [words]+ return FakeEncoding([len(row) for row in rows])++ class FakeScores:+ def __init__(self, row_length):+ self.values = [float(index) for index in range(row_length)]++ def cpu(self):+ return self.values++ class FakeParameter:+ device = "mps"++ class FakeModel:+ def parameters(self):+ return iter([FakeParameter()])++ def get_scores(self, input_ids, _attention_mask):+ return [FakeScores(row_length) for row_length in input_ids.row_lengths]++ model = FakeModel()+ monkeypatch.setattr(kc, "_execution_semaphore", lambda *_args, **_kwargs: semaphore)+ monkeypatch.setattr(+ kc,+ "_load_kompress",+ lambda *args, **kwargs: (model, FakeTokenizer(), "pytorch"),+ )++ compressor = kc.KompressCompressor(kc.KompressConfig(min_input_words=10, enable_ccr=False))+ compressor.compress(" ".join(["word"] * 20), target_ratio=0.5)+ assert copy_saw_slot == [True, True]++ copy_saw_slot.clear()+ monkeypatch.setattr(+ kc.KompressCompressor,+ "_should_use_sequential_fallback",+ lambda self: False,+ )+ compressor.compress_batch(+ [" ".join(["word"] * 20), " ".join(["token"] * 20)],+ target_ratio=0.5,+ )+ assert copy_saw_slot == [True, True]+++def test_canary_device_copy_holds_execution_slot(monkeypatch):+ semaphore = threading.BoundedSemaphore(1)+ copy_saw_slot: list[bool] = []++ class FakeTensor:+ def to(self, _device):+ acquired = semaphore.acquire(blocking=False)+ if acquired:+ semaphore.release()+ copy_saw_slot.append(not acquired)+ return self++ class FakeTokenizer:+ def __call__(self, *_args, **_kwargs):+ return {"input_ids": FakeTensor(), "attention_mask": FakeTensor()}++ class FakeParameter:+ device = "mps"++ class FakeModel:+ def parameters(self):+ return iter([FakeParameter()])++ def get_keep_mask(self, input_ids, attention_mask):+ return []++ monkeypatch.setattr(kc, "_execution_semaphore", lambda *_args, **_kwargs: semaphore)++ kc.KompressCompressor()._timed_canary(FakeModel(), FakeTokenizer(), "pytorch")++ assert copy_saw_slot == [True, True]