Fully automatic censorship removal for language models
Separate abliteration parameters for different layer components
Philipp Emanuel Weidmann6359aa44bb77
diff --git a/src/heretic/main.py b/src/heretic/main.pyindex ec8f41c..788e50b 100644--- a/src/heretic/main.py+++ b/src/heretic/main.py@@ -26,7 +26,7 @@ from rich.traceback import installfrom .config import Settingsfrom .evaluator import Evaluator-from .model import Model+from .model import AbliterationParameters, Modelfrom .utils import get_readme_intro, load_prompts, print@@ -175,64 +175,63 @@ def run():trial_index += 1trial.set_user_attr("index", trial_index)- max_weight = trial.suggest_float("max_weight", 0, 1)- max_weight_position = trial.suggest_float(- "max_weight_position", 0, len(model.get_layers()) - 1- )- min_weight = trial.suggest_float("min_weight", 0, max_weight)- min_weight_distance = trial.suggest_float(- "min_weight_distance", 1, len(model.get_layers()) - 1- )+ parameters = {}++ for component in model.get_abliterable_components():+ # The parameter ranges are based on experiments with various models+ # and much wider ranges. They are not set in stone and might have to be+ # adjusted for future models.+ max_weight = trial.suggest_float(+ f"{component}.max_weight",+ 0.8,+ 1.2,+ )+ max_weight_position = trial.suggest_float(+ f"{component}.max_weight_position",+ 0.6 * (len(model.get_layers()) - 1),+ len(model.get_layers()) - 1,+ )+ min_weight = trial.suggest_float(+ f"{component}.min_weight",+ 0.0,+ max_weight,+ )+ min_weight_distance = trial.suggest_float(+ f"{component}.min_weight_distance",+ 1.0,+ 0.6 * (len(model.get_layers()) - 1),+ )++ parameters[component] = AbliterationParameters(+ max_weight=max_weight,+ max_weight_position=max_weight_position,+ min_weight=min_weight,+ min_weight_distance=min_weight_distance,+ )print()print(f"Running trial [bold]{trial_index}[/] of [bold]{settings.n_trials}[/]...")print("* Parameters:")- print(f" * max_weight = [bold]{max_weight:.4f}[/]")- print(f" * max_weight_position = [bold]{max_weight_position:.4f}[/]")- print(f" * min_weight = [bold]{min_weight:.4f}[/]")- print(f" * min_weight_distance = [bold]{min_weight_distance:.4f}[/]")+ for name, value in trial.params.items():+ print(f" * {name} = [bold]{value:.4f}[/]")print("* Reloading model...")model.reload_model()print("* Abliterating...")- model.abliterate(- refusal_directions,- max_weight,- max_weight_position,- min_weight,- min_weight_distance,- )+ model.abliterate(refusal_directions, parameters)print("* Evaluating...")score, kl_divergence, refusals = evaluator.get_score()trial.set_user_attr("kl_divergence", kl_divergence)trial.set_user_attr("refusals", refusals)+ trial.set_user_attr("parameters", parameters)# The optimizer searches for a minimum, so we return the negative score.return -scorestudy = optuna.create_study()- # Educated guesses for parameter values to get the optimizer started.- for max_weight, max_weight_position, min_weight, min_weight_distance in [- (0.0, 0.0, 0.0, 0.5),- (1.0, 0.5, 0.0, 0.25),- (0.8, 0.7, 0.3, 0.4),- (0.9, 0.3, 0.1, 0.1),- (1.0, 1.0, 1.0, 1.0),- ]:- study.enqueue_trial(- {- "max_weight": max_weight,- "max_weight_position": max_weight_position- * (len(model.get_layers()) - 1),- "min_weight": min_weight,- "min_weight_distance": min_weight_distance- * (len(model.get_layers()) - 1),- }- )-study.optimize(objective, n_trials=settings.n_trials)print()@@ -240,14 +239,8 @@ def run():f"[bold green]Optimization finished![/] Best was trial [bold]{study.best_trial.user_attrs['index']}[/]:")print("* Parameters:")- print(f" * max_weight = [bold]{study.best_params['max_weight']:.4f}[/]")- print(- f" * max_weight_position = [bold]{study.best_params['max_weight_position']:.4f}[/]"- )- print(f" * min_weight = [bold]{study.best_params['min_weight']:.4f}[/]")- print(- f" * min_weight_distance = [bold]{study.best_params['min_weight_distance']:.4f}[/]"- )+ for name, value in study.best_params.items():+ print(f" * {name} = [bold]{value:.4f}[/]")print("* Results:")print(f" * KL divergence: [bold]{study.best_trial.user_attrs['kl_divergence']:.4f}[/]"@@ -263,13 +256,7 @@ def run():print("* Reloading model...")model.reload_model()print("* Abliterating...")- model.abliterate(- refusal_directions,- study.best_params["max_weight"],- study.best_params["max_weight_position"],- study.best_params["min_weight"],- study.best_params["min_weight_distance"],- )+ model.abliterate(refusal_directions, study.best_trial.user_attrs["parameters"])while True:print()diff --git a/src/heretic/model.py b/src/heretic/model.pyindex 4655411..e926ac8 100644--- a/src/heretic/model.py+++ b/src/heretic/model.py@@ -2,6 +2,7 @@# Copyright (C) 2025 Philipp Emanuel Weidmann <[email protected]>from contextlib import suppress+from dataclasses import dataclassfrom typing import Anyimport torch@@ -21,6 +22,14 @@ from .config import Settingsfrom .utils import batchify, empty_cache, print+@dataclass+class AbliterationParameters:+ max_weight: float+ max_weight_position: float+ min_weight: float+ min_weight_distance: float++class Model:def __init__(self, settings: Settings):self.settings = settings@@ -61,9 +70,11 @@ class Model:raise Exception("Failed to load model with all configured dtypes.")print(f"* Transformer model with [bold]{len(self.get_layers())}[/] layers")- print(- f"* [bold]{len(self.get_layer_matrices(0))}[/] abliterable matrices per layer"- )+ print("* Abliterable components:")+ for component, matrices in self.get_layer_matrices(0).items():+ print(+ f" * [bold]{component}[/]: [bold]{len(matrices)}[/] matrices per layer"+ )def reload_model(self):dtype = self.model.dtype@@ -86,84 +97,87 @@ class Model:# Text-only models.return self.model.model.layers- def get_layer_matrices(self, layer_index: int) -> list[torch.Tensor]:+ def get_layer_matrices(self, layer_index: int) -> dict[str, list[torch.Tensor]]:layer = self.get_layers()[layer_index]- matrices = []+ matrices = {}- def try_add(matrix: Any):+ def try_add(component: str, matrix: Any):assert torch.is_tensor(matrix)- matrices.append(matrix)++ if component not in matrices:+ matrices[component] = []++ matrices[component].append(matrix)++ # Exceptions aren't suppressed here, because there is currently+ # no alternative location for the attention out-projection.+ try_add("attn.o_proj", layer.self_attn.o_proj.weight)# Most dense models.- if not matrices:- with suppress(Exception):- try_add(layer.mlp.down_proj.weight)+ with suppress(Exception):+ try_add("mlp.down_proj", layer.mlp.down_proj.weight)# Some MoE models (e.g. Qwen3).- if not matrices:- with suppress(Exception):- for expert in layer.mlp.experts:- try_add(expert.down_proj.weight)+ with suppress(Exception):+ for expert in layer.mlp.experts:+ try_add("mlp.down_proj", expert.down_proj.weight)# Phi-3.5-MoE (and possibly others).- if not matrices:- with suppress(Exception):- for expert in layer.block_sparse_moe.experts:- try_add(expert.w2.weight)+ with suppress(Exception):+ for expert in layer.block_sparse_moe.experts:+ try_add("mlp.down_proj", expert.w2.weight)# gpt-oss MoE.- if not matrices:- with suppress(Exception):- # The implementation of gpt-oss in Transformers differs from many other MoE models- # in that it stores the down-projections for all experts in a single 3D tensor,- # but thanks to PyTorch's broadcasting magic, it all just works anyway.- try_add(layer.mlp.experts.down_proj)+ with suppress(Exception):+ # The implementation of gpt-oss in Transformers differs from many other MoE models+ # in that it stores the down-projections for all experts in a single 3D tensor,+ # but thanks to PyTorch's broadcasting magic, it all just works anyway.+ try_add("mlp.down_proj", layer.mlp.experts.down_proj)# We need at least one MLP down-projection.- assert matrices-- # Exceptions aren't suppressed here, because there is currently- # no alternative location for the attention out-projection.- try_add(layer.self_attn.o_proj.weight)+ assert matrices["mlp.down_proj"]return matrices+ def get_abliterable_components(self) -> list[str]:+ return list(self.get_layer_matrices(0).keys())+def abliterate(self,refusal_directions: torch.Tensor,- max_weight: float,- max_weight_position: float,- min_weight: float,- min_weight_distance: float,+ parameters: dict[str, AbliterationParameters],):# Note that some implementations of abliteration also orthogonalize# the embedding matrix, but it's unclear if that has any benefits.for layer_index in range(len(self.get_layers())):- distance = abs(layer_index - max_weight_position)+ for component, matrices in self.get_layer_matrices(layer_index).items():+ params = parameters[component]- # Don't orthogonalize layers that are more than- # min_weight_distance away from max_weight_position.- if distance > min_weight_distance:- continue+ distance = abs(layer_index - params.max_weight_position)- # Interpolate linearly between max_weight and min_weight- # over min_weight_distance.- weight = max_weight + (distance / min_weight_distance) * (- min_weight - max_weight- )+ # Don't orthogonalize layers that are more than+ # min_weight_distance away from max_weight_position.+ if distance > params.min_weight_distance:+ continue++ # Interpolate linearly between max_weight and min_weight+ # over min_weight_distance.+ weight = params.max_weight + (distance / params.min_weight_distance) * (+ params.min_weight - params.max_weight+ )- # The index must be shifted by 1 because the first element- # of refusal_directions is the direction for the embeddings.- refusal_direction = refusal_directions[layer_index + 1]+ # The index must be shifted by 1 because the first element+ # of refusal_directions is the direction for the embeddings.+ refusal_direction = refusal_directions[layer_index + 1]- # Projects any right-multiplied vector(s) onto the subspace- # spanned by the refusal direction.- projector = torch.outer(refusal_direction, refusal_direction)+ # Projects any right-multiplied vector(s) onto the subspace+ # spanned by the refusal direction.+ projector = torch.outer(refusal_direction, refusal_direction)- for matrix in self.get_layer_matrices(layer_index):- # In-place subtraction is safe as we're not using Autograd.- matrix.sub_(weight * (projector @ matrix))+ for matrix in matrices:+ # In-place subtraction is safe as we're not using Autograd.+ matrix.sub_(weight * (projector @ matrix))def get_chat(self, prompt: str) -> list[dict[str, str]]:return [diff --git a/src/heretic/utils.py b/src/heretic/utils.pyindex 65edcc1..6f0fdfb 100644--- a/src/heretic/utils.py+++ b/src/heretic/utils.py@@ -54,28 +54,36 @@ def get_readme_intro(base_refusals: int,bad_prompts: list[str],) -> str:+ model_link = f"[{settings.model}](https://huggingface.co/{settings.model})"refusal_percentage = (study.best_trial.user_attrs["refusals"] / len(bad_prompts) * 100)base_refusal_percentage = base_refusals / len(bad_prompts) * 100- return f"""# This is a decensored version of [{settings.model}](https://huggingface.co/{settings.model}), made using [Heretic](https://github.com/p-e-w/heretic) v{version("heretic")}+ return f"""# This is a decensored version of {+ model_link+ }, made using [Heretic](https://github.com/p-e-w/heretic) v{version("heretic")}## Abliteration parameters-| Parameter | Value |-| :---------------------- | :--------------------------------------------: |-| **max_weight** | {study.best_params["max_weight"]:.4f} |-| **max_weight_position** | {study.best_params["max_weight_position"]:.4f} |-| **min_weight** | {study.best_params["min_weight"]:.4f} |-| **min_weight_distance** | {study.best_params["min_weight_distance"]:.4f} |+| Parameter | Value |+| :-------- | :---: |+{+ chr(10).join(+ [+ f"| **{name}** | {value:.4f} |"+ for name, value in study.best_params.items()+ ]+ )+ }## Performance-| Metric | This model | Original model ([{settings.model}](https://huggingface.co/{settings.model})) |-| :---------------- | :------------------------------------------------: | :--------------------------------------------------------------------------: |-| **KL divergence** | {study.best_trial.user_attrs["kl_divergence"]:.4f} | 0 *(by definition)* |-| **Refusals** | {refusal_percentage:.1f} % | {base_refusal_percentage:.1f} % |+| Metric | This model | Original model ({model_link}) |+| :----- | :--------: | :---------------------------: |+| **KL divergence** | {+ study.best_trial.user_attrs["kl_divergence"]:.4f} | 0 *(by definition)* |+| **Refusals** | {refusal_percentage:.1f} % | {base_refusal_percentage:.1f} % |-----