Skip to content

Fully automatic censorship removal for language models

Fix multi-GPU support and memory management (#17)

Nikolai Kolodziejc8b6663b930e
diff --git a/src/heretic/main.py b/src/heretic/main.py
index 4a810e7..a662ec6 100644
--- a/src/heretic/main.py
+++ b/src/heretic/main.py
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
# Copyright (C) 2025 Philipp Emanuel Weidmann <[email protected]>
+import os
import math
import sys
import time
@@ -34,6 +35,7 @@ from .config import Settings
from .evaluator import Evaluator
from .model import AbliterationParameters, Model
from .utils import (
+ empty_cache,
format_duration,
get_readme_intro,
get_trial_parameters,
@@ -44,6 +46,10 @@ from .utils import (
def run():
+ # Enable expandable segments to reduce memory fragmentation on multi-GPU setups.
+ if "PYTORCH_CUDA_ALLOC_CONF" not in os.environ:
+ os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
+
# Modified "Pagga" font from https://budavariam.github.io/asciiart-text/
print(f"[cyan]█░█░█▀▀░█▀▄░█▀▀░▀█▀░█░█▀▀[/] v{version('heretic-llm')}")
print("[cyan]█▀█░█▀▀░█▀▄░█▀▀░░█░░█░█░░[/]")
diff --git a/src/heretic/model.py b/src/heretic/model.py
index b617052..d1797e6 100644
--- a/src/heretic/model.py
+++ b/src/heretic/model.py
@@ -213,8 +213,10 @@ class Model:
).to(self.model.dtype)
for matrix in matrices:
+ # Ensure projector is on the same device as the matrix for multi-GPU support.
+ device_projector = projector.to(matrix.device)
# In-place subtraction is safe as we're not using Autograd.
- matrix.sub_(weight * (projector @ matrix))
+ matrix.sub_(weight * (device_projector @ matrix))
def get_chat(self, prompt: str) -> list[dict[str, str]]:
return [
diff --git a/src/heretic/utils.py b/src/heretic/utils.py
index 74f0874..bae2a11 100644
--- a/src/heretic/utils.py
+++ b/src/heretic/utils.py
@@ -48,6 +48,8 @@ def batchify(items: list[T], batch_size: int) -> list[list[T]]:
def empty_cache():
+ gc.collect()
+
if torch.cuda.is_available():
torch.cuda.empty_cache()
elif is_xpu_available():