Fully automatic censorship removal for language models
Fix multi-GPU support and memory management (#17)
Nikolai Kolodziejc8b6663b930e
diff --git a/src/heretic/main.py b/src/heretic/main.pyindex 4a810e7..a662ec6 100644--- a/src/heretic/main.py+++ b/src/heretic/main.py@@ -1,6 +1,7 @@# SPDX-License-Identifier: AGPL-3.0-or-later# Copyright (C) 2025 Philipp Emanuel Weidmann <[email protected]>+import osimport mathimport sysimport time@@ -34,6 +35,7 @@ from .config import Settingsfrom .evaluator import Evaluatorfrom .model import AbliterationParameters, Modelfrom .utils import (+ empty_cache,format_duration,get_readme_intro,get_trial_parameters,@@ -44,6 +46,10 @@ from .utils import (def run():+ # Enable expandable segments to reduce memory fragmentation on multi-GPU setups.+ if "PYTORCH_CUDA_ALLOC_CONF" not in os.environ:+ os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"+# Modified "Pagga" font from https://budavariam.github.io/asciiart-text/print(f"[cyan]█░█░█▀▀░█▀▄░█▀▀░▀█▀░█░█▀▀[/] v{version('heretic-llm')}")print("[cyan]█▀█░█▀▀░█▀▄░█▀▀░░█░░█░█░░[/]")diff --git a/src/heretic/model.py b/src/heretic/model.pyindex b617052..d1797e6 100644--- a/src/heretic/model.py+++ b/src/heretic/model.py@@ -213,8 +213,10 @@ class Model:).to(self.model.dtype)for matrix in matrices:+ # Ensure projector is on the same device as the matrix for multi-GPU support.+ device_projector = projector.to(matrix.device)# In-place subtraction is safe as we're not using Autograd.- matrix.sub_(weight * (projector @ matrix))+ matrix.sub_(weight * (device_projector @ matrix))def get_chat(self, prompt: str) -> list[dict[str, str]]:return [diff --git a/src/heretic/utils.py b/src/heretic/utils.pyindex 74f0874..bae2a11 100644--- a/src/heretic/utils.py+++ b/src/heretic/utils.py@@ -48,6 +48,8 @@ def batchify(items: list[T], batch_size: int) -> list[list[T]]:def empty_cache():+ gc.collect()+if torch.cuda.is_available():torch.cuda.empty_cache()elif is_xpu_available():