Fully automatic censorship removal for language models
feat: automatically reproduce model from reproduce.json (#326)
Philipp Emanuel Weidmann2fd163f5e401
diff --git a/config.default.toml b/config.default.tomlindex 9424bfb..7ce6a5a 100644--- a/config.default.toml+++ b/config.default.toml@@ -123,10 +123,6 @@ n_trials = 200# Number of trials that use random sampling for the purpose of exploration.n_startup_trials = 60-# Random seed for reproducible optimization. Set to an integer to enable.-# Applies to Python's random module, NumPy, PyTorch, and Optuna.-# seed = 75-# Directory to save and load study progress to/from.study_checkpoint_dir = "checkpoints"diff --git a/src/heretic/config.py b/src/heretic/config.pyindex ada5792..8744394 100644--- a/src/heretic/config.py+++ b/src/heretic/config.py@@ -32,6 +32,11 @@ class RowNormalization(str, Enum):FULL = "full"+class ExportStrategy(str, Enum):+ MERGE = "merge"+ ADAPTER = "adapter"++class DatasetSpecification(BaseModel):dataset: str = Field(description="Hugging Face dataset ID, or path to dataset on disk."@@ -119,6 +124,15 @@ class Settings(BaseSettings):exclude=True,)+ reproduce: str | None = Field(+ default=None,+ description=(+ "If this path or URL to a reproduce.json file is set, load reproduction information "+ "from that file, and attempt to reproduce the abliterated model it originated from."+ ),+ exclude=True,+ )+dtypes: list[str] = Field(default=[# In practice, "auto" almost always means bfloat16.@@ -167,13 +181,6 @@ class Settings(BaseSettings):),)- trust_remote_code: bool | None = Field(- default=None,- description="Whether to trust remote code when loading the model.",- # For security reasons, we don't store this setting.- exclude=True,- )-batch_size: int = Field(default=0, # autodescription="Number of input sequences to process in parallel (0 = auto).",@@ -416,6 +423,11 @@ class Settings(BaseSettings):description="Maximum size for individual safetensors files generated when exporting a model.",)+ export_strategy: ExportStrategy | None = Field(+ default=None,+ description='How to export the model: "merge", "adapter", or unset to prompt the user.',+ )+refusal_markers: list[str] = Field(default=["disclaimer",diff --git a/src/heretic/main.py b/src/heretic/main.pyindex 20895b9..d42b4d8 100644--- a/src/heretic/main.py+++ b/src/heretic/main.py@@ -47,7 +47,7 @@ import questionaryimport torchimport torch.nn.functional as Fimport transformers-from huggingface_hub import ModelCard, ModelCardData+from huggingface_hub import HfApi, ModelCard, ModelCardDatafrom lm_eval.models.huggingface import HFLMfrom optuna import Trial, TrialPrunedfrom optuna.exceptions import ExperimentalWarning@@ -55,21 +55,26 @@ from optuna.samplers import TPESamplerfrom optuna.storages import JournalStoragefrom optuna.storages.journal import JournalFileBackend, JournalFileOpenLockfrom optuna.study import StudyDirection-from optuna.trial import TrialState+from optuna.trial import TrialState, create_trialfrom pydantic import ValidationErrorfrom questionary import Choice, Stylefrom rich.table import Tablefrom rich.traceback import installfrom .analyzer import Analyzer-from .config import QuantizationMethod+from .config import ExportStrategy, QuantizationMethodfrom .evaluator import Evaluatorfrom .model import AbliterationParameters, Model, get_model_class-from .reproduce import collect_reproducibles+from .reproduce import (+ check_environment,+ collect_reproducibles,+ load_reproduction_information,+)from .system import empty_cache, get_accelerator_infofrom .utils import (format_duration,format_exception,+ get_file_sha256,get_readme_intro,get_trial_parameters,is_hf_path,@@ -85,13 +90,19 @@ from .utils import ()-def obtain_merge_strategy(settings: Settings, model: Model) -> str | None:+def obtain_export_strategy(+ settings: Settings,+ model: Model,+) -> ExportStrategy | None:"""- Prompts the user for how to proceed with saving the model.+ Gets the export strategy from settings or prompts the user.Provides info to the user if the model is quantized on memory use.- Returns "merge", "adapter", or None (if cancelled/invalid).+ Returns an export strategy, or None if cancelled."""+ if settings.export_strategy is not None:+ return settings.export_strategy+if settings.quantization == QuantizationMethod.BNB_4BIT:print()print(@@ -114,7 +125,9 @@ def obtain_merge_strategy(settings: Settings, model: Model) -> str | None:settings.model,device_map="meta",torch_dtype=torch.bfloat16,- trust_remote_code=model.trusted_models.get(settings.model),+ trust_remote_code=True+ if settings.model in model.trusted_models+ else None,**model.revision_kwargs,)footprint_bytes = meta_model.get_memory_footprint()@@ -143,11 +156,11 @@ def obtain_merge_strategy(settings: Settings, model: Model) -> str | None:if settings.quantization == QuantizationMethod.NONEelse " (requires sufficient RAM)"),- value="merge",+ value=ExportStrategy.MERGE,),Choice(title="Save LoRA adapter only (can be merged later)",- value="adapter",+ value=ExportStrategy.ADAPTER,),],)@@ -176,6 +189,7 @@ def run():len(sys.argv) > 1# Heretic is being invoked in standard (model processing) mode.and "--collect-reproducibles" not in sys.argv+ and "--reproduce" not in sys.argv# No model has been explicitly provided.and "--model" not in sys.argv# The last argument is a parameter value rather than a flag (such as "--help").@@ -186,7 +200,9 @@ def run():# Work around the "model" argument being required# when Heretic is invoked in a non-processing mode.- if "--collect-reproducibles" in sys.argv and "--model" not in sys.argv:+ if (+ "--collect-reproducibles" in sys.argv or "--reproduce" in sys.argv+ ) and "--model" not in sys.argv:sys.argv.extend(["--model", ""])try:@@ -211,6 +227,31 @@ def run():collect_reproducibles(settings.collect_reproducibles)return+ reproduction_mode = settings.reproduce is not None++ if settings.reproduce is not None:+ print(f"Loading reproduction information from [bold]{settings.reproduce}[/]...")+ # FIXME: "Reproduction"/"reproducibility" name inconsistency!+ reproduction_information = load_reproduction_information(settings.reproduce)++ if reproduction_information["version"] not in ["1", "2"]:+ print(+ (+ f"[red]Unsupported file format version: [bold]{reproduction_information['version']}[/].[/] "+ "Try loading the file with a newer version of Heretic."+ )+ )+ return++ if not check_environment(reproduction_information):+ return++ print()++ verify_hashes = reproduction_information["version"] != "1"++ settings = Settings.model_validate(reproduction_information["settings"])+if settings.seed is None:settings.seed = random.randint(0, 2**32 - 1)@@ -260,7 +301,11 @@ def run():except IndexError:existing_study = None- if existing_study is not None and settings.evaluate_model is None:+ if (+ existing_study is not None+ and settings.evaluate_model is None+ and not reproduction_mode+ ):choices = []if existing_study.user_attrs["finished"]:@@ -604,151 +649,177 @@ def run():trial.study.stop()raise TrialPruned()- study = optuna.create_study(- sampler=TPESampler(- n_startup_trials=settings.n_startup_trials,- n_ei_candidates=128,- multivariate=True,- seed=settings.seed,- ),- directions=[StudyDirection.MINIMIZE, StudyDirection.MINIMIZE],- storage=storage,- study_name="heretic",- load_if_exists=True,- )+ if not reproduction_mode:+ study = optuna.create_study(+ sampler=TPESampler(+ n_startup_trials=settings.n_startup_trials,+ n_ei_candidates=128,+ multivariate=True,+ seed=settings.seed,+ ),+ directions=[StudyDirection.MINIMIZE, StudyDirection.MINIMIZE],+ storage=storage,+ study_name="heretic",+ load_if_exists=True,+ )- study.set_user_attr("settings", settings.model_dump_json())- study.set_user_attr("finished", False)+ study.set_user_attr("settings", settings.model_dump_json())+ study.set_user_attr("finished", False)- start_index = trial_index = len(study.trials)- if start_index > 0:- print()- print("Resuming existing study.")+ start_index = trial_index = len(study.trials)+ if start_index > 0:+ print()+ print("Resuming existing study.")- try:- study.optimize(- objective_wrapper,- n_trials=settings.n_trials - len(study.trials),- )- except KeyboardInterrupt:- # This additional handler takes care of the small chance that KeyboardInterrupt- # is raised just between trials, which wouldn't be caught by the handler- # defined in objective_wrapper above.- pass+ try:+ study.optimize(+ objective_wrapper,+ n_trials=settings.n_trials - len(study.trials),+ )+ except KeyboardInterrupt:+ # This additional handler takes care of the small chance that KeyboardInterrupt+ # is raised just between trials, which wouldn't be caught by the handler+ # defined in objective_wrapper above.+ pass- if len(study.trials) == settings.n_trials:- study.set_user_attr("finished", True)+ if len(study.trials) == settings.n_trials:+ study.set_user_attr("finished", True)while True:- # If no trials at all have been evaluated, the study must have been stopped- # by pressing Ctrl+C while the first trial was running. In this case, we just- # re-raise the interrupt to invoke the standard handler defined below.- completed_trials = [t for t in study.trials if t.state == TrialState.COMPLETE]- if not completed_trials:- raise KeyboardInterrupt-- # Get the Pareto front of trials. We can't use study.best_trials directly- # as get_score() doesn't return the pure KL divergence and refusal count.- # Note: Unlike study.best_trials, this does not handle objective constraints.- sorted_trials = sorted(- completed_trials,- key=lambda trial: (- trial.user_attrs["refusals"],- trial.user_attrs["kl_divergence"],- ),- )- min_divergence = math.inf- best_trials = []- for trial in sorted_trials:- kl_divergence = trial.user_attrs["kl_divergence"]- if kl_divergence < min_divergence:- min_divergence = kl_divergence- best_trials.append(trial)-- choices = [- Choice(- title=(- f"[Trial {trial.user_attrs['index']:>3}] "- f"Refusals: {trial.user_attrs['refusals']:>2}/{len(evaluator.bad_prompts)}, "- f"KL divergence: {trial.user_attrs['kl_divergence']:.4f}"+ if not reproduction_mode:+ # If no trials at all have been evaluated, the study must have been stopped+ # by pressing Ctrl+C while the first trial was running. In this case, we just+ # re-raise the interrupt to invoke the standard handler defined below.+ completed_trials = [+ t for t in study.trials if t.state == TrialState.COMPLETE+ ]+ if not completed_trials:+ raise KeyboardInterrupt++ # Get the Pareto front of trials. We can't use study.best_trials directly+ # as get_score() doesn't return the pure KL divergence and refusal count.+ # Note: Unlike study.best_trials, this does not handle objective constraints.+ sorted_trials = sorted(+ completed_trials,+ key=lambda trial: (+ trial.user_attrs["refusals"],+ trial.user_attrs["kl_divergence"],),- value=trial,)- for trial in best_trials- ]+ min_divergence = math.inf+ best_trials = []+ for trial in sorted_trials:+ kl_divergence = trial.user_attrs["kl_divergence"]+ if kl_divergence < min_divergence:+ min_divergence = kl_divergence+ best_trials.append(trial)++ choices = [+ Choice(+ title=(+ f"[Trial {trial.user_attrs['index']:>3}] "+ f"Refusals: {trial.user_attrs['refusals']:>2}/{len(evaluator.bad_prompts)}, "+ f"KL divergence: {trial.user_attrs['kl_divergence']:.4f}"+ ),+ value=trial,+ )+ for trial in best_trials+ ]- choices.append(- Choice(- title="Run additional trials",- value="continue",+ choices.append(+ Choice(+ title="Run additional trials",+ value="continue",+ ))- )- choices.append(- Choice(- title="Exit program",- value="",+ choices.append(+ Choice(+ title="Exit program",+ value="",+ ))- )- print()- print("[bold green]Optimization finished![/]")- print()- print(- (- "The following trials resulted in Pareto optimal combinations of refusals and KL divergence. "- "After selecting a trial, you will be able to save the model, upload it to Hugging Face, "- "chat with it to test how well it works, or run standard benchmarks on it. "- "You can return to this menu later to select a different trial. "- "[yellow]Note that KL divergence values above 0.5 usually indicate significant damage to the original model's capabilities.[/]"+ print()+ print("[bold green]Optimization finished![/]")+ print()+ print(+ (+ "The following trials resulted in Pareto optimal combinations of refusals and KL divergence. "+ "After selecting a trial, you will be able to save the model, upload it to Hugging Face, "+ "chat with it to test how well it works, or run standard benchmarks on it. "+ "You can return to this menu later to select a different trial. "+ "[yellow]Note that KL divergence values above 0.5 usually indicate significant damage to the original model's capabilities.[/]"+ ))- )while True:- print()- trial = prompt_select("Which trial do you want to use?", choices)+ if reproduction_mode:+ parameters = reproduction_information["parameters"]+ metrics = reproduction_information["metrics"]++ trial = create_trial(+ values=[],+ user_attrs={+ "direction_index": parameters["direction_index"],+ "parameters": parameters["abliteration_parameters"],+ "kl_divergence": metrics["kl_divergence"],+ "refusals": metrics["refusals"],+ "base_refusals": metrics["base_refusals"],+ "n_bad_prompts": metrics["n_bad_prompts"],+ },+ )++ print()+ print("Restoring model from reproduction information...")+ else:+ print()+ trial = prompt_select("Which trial do you want to use?", choices)++ if trial is None or trial == "":+ return++ if trial == "continue":+ while True:+ try:+ n_additional_trials = prompt_text(+ "How many additional trials do you want to run?"+ )+ if n_additional_trials is None or n_additional_trials == "":+ n_additional_trials = 0+ break+ n_additional_trials = int(n_additional_trials)+ if n_additional_trials > 0:+ break+ print("[red]Please enter a number greater than 0.[/]")+ except ValueError:+ print("[red]Please enter a number.[/]")++ if n_additional_trials == 0:+ continue++ settings.n_trials += n_additional_trials+ study.set_user_attr("settings", settings.model_dump_json())+ study.set_user_attr("finished", False)- if trial == "continue":- while True:try:- n_additional_trials = prompt_text(- "How many additional trials do you want to run?"+ study.optimize(+ objective_wrapper,+ n_trials=settings.n_trials - len(study.trials),)- if n_additional_trials is None or n_additional_trials == "":- n_additional_trials = 0- break- n_additional_trials = int(n_additional_trials)- if n_additional_trials > 0:- break- print("[red]Please enter a number greater than 0.[/]")- except ValueError:- print("[red]Please enter a number.[/]")-- if n_additional_trials == 0:- continue-- settings.n_trials += n_additional_trials- study.set_user_attr("settings", settings.model_dump_json())- study.set_user_attr("finished", False)+ except KeyboardInterrupt:+ pass- try:- study.optimize(- objective_wrapper,- n_trials=settings.n_trials - len(study.trials),- )- except KeyboardInterrupt:- pass+ if len(study.trials) == settings.n_trials:+ study.set_user_attr("finished", True)- if len(study.trials) == settings.n_trials:- study.set_user_attr("finished", True)-- break+ break- elif trial is None or trial == "":- return+ print()+ print(+ f"Restoring model from trial [bold]{trial.user_attrs['index']}[/]..."+ )- print()- print(f"Restoring model from trial [bold]{trial.user_attrs['index']}[/]...")print("* Parameters:")for name, value in get_trial_parameters(trial).items():print(f" * {name} = [bold]{value}[/]")@@ -779,12 +850,20 @@ def run():"Upload the model to Hugging Face","Chat with the model","Benchmark the model",- "Return to the trial selection menu",+ Choice(+ title="Exit program"+ if reproduction_mode+ else "Return to the trial selection menu",+ value="",+ ),],)- if action is None or action == "Return to the trial selection menu":- break+ if action is None or action == "":+ if reproduction_mode:+ return+ else:+ break# All actions are wrapped in a try/except block so that if an error occurs,# another action can be tried, instead of the program crashing and losing@@ -796,11 +875,11 @@ def run():if not save_directory:continue- strategy = obtain_merge_strategy(settings, model)+ strategy = obtain_export_strategy(settings, model)if strategy is None:continue- if strategy == "adapter":+ if strategy == ExportStrategy.ADAPTER:print("Saving LoRA adapter...")model.model.save_pretrained(save_directory,@@ -822,6 +901,31 @@ def run():print(f"Model saved to [bold]{save_directory}[/].")+ if reproduction_mode and verify_hashes:+ print("Verifying hashes of weight files...")++ for (+ filename,+ original_sha256,+ ) in reproduction_information["hashes"].items():+ file_path = Path(save_directory) / filename++ if file_path.exists():+ sha256 = get_file_sha256(file_path)++ if sha256.lower() == original_sha256.lower():+ print(+ f"[bold]{filename}:[/] [green]Hash matches[/]"+ )+ else:+ print(+ f"[bold]{filename}:[/] [yellow]Hash doesn't match[/]"+ )+ else:+ print(+ f"[bold]{filename}:[/] [red]File not found[/]"+ )+case "Upload the model to Hugging Face":# We don't use huggingface_hub.login() because that stores the token on disk,# and since this program will often be run on rented or shared GPU servers,@@ -856,7 +960,7 @@ def run():continueprivate = visibility == "Private"- strategy = obtain_merge_strategy(settings, model)+ strategy = obtain_export_strategy(settings, model)if strategy is None:continue@@ -868,8 +972,10 @@ def run():settings.good_evaluation_prompts.dataset,settings.bad_evaluation_prompts.dataset,]- is_reproducible = is_hf_path(settings.model) and all(- is_hf_path(dataset) for dataset in datasets+ is_reproducible = (+ is_hf_path(settings.model)+ and all(is_hf_path(dataset) for dataset in datasets)+ and not reproduction_mode)if is_reproducible:@@ -904,7 +1010,7 @@ def run():else:reproducibility_information = "none"- if strategy == "adapter":+ if strategy == ExportStrategy.ADAPTER:print("Uploading LoRA adapter...")model.model.push_to_hub(repo_id,@@ -973,19 +1079,75 @@ def run():# Set the number of trials to the number of actual completed trials# for the reproduction configuration.settings.n_trials = len(study.trials)+ current_export_strategy = settings.export_strategy+ settings.export_strategy = strategy++ try:+ upload_reproduce_folder(+ repo_id,+ settings,+ token,+ checkpoint_path=study_checkpoint_file,+ trial=trial,+ include_system_information=(+ reproducibility_information == "full"+ ),+ )+ finally:+ settings.export_strategy = current_export_strategy++ print(f"Model uploaded to [bold]{repo_id}[/].")- upload_reproduce_folder(+ if reproduction_mode and verify_hashes:+ print("Verifying hashes of weight files...")++ api = HfApi()+ model_info = api.model_info(repo_id,- settings,- token,- checkpoint_path=study_checkpoint_file,- trial=trial,- include_system_information=(- reproducibility_information == "full"- ),+ files_metadata=True,+ token=token,)- print(f"Model uploaded to [bold]{repo_id}[/].")+ if not model_info.siblings:+ raise RuntimeError(+ "Could not fetch uploaded model hashes."+ )++ for (+ filename,+ original_sha256,+ ) in reproduction_information["hashes"].items():+ file_found = False++ for file in model_info.siblings:+ if file.rfilename == filename:+ sha256 = getattr(file, "lfs", {}).get(+ "sha256"+ )+ if not sha256:+ raise RuntimeError(+ "Could not fetch uploaded model hashes."+ )++ if (+ sha256.lower()+ == original_sha256.lower()+ ):+ print(+ f"[bold]{filename}:[/] [green]Hash matches[/]"+ )+ else:+ print(+ f"[bold]{filename}:[/] [yellow]Hash doesn't match[/]"+ )++ file_found = True+ break++ if not file_found:+ print(+ f"[bold]{filename}:[/] [red]File not found[/]"+ )case "Chat with the model":print()diff --git a/src/heretic/model.py b/src/heretic/model.pyindex 401f5b2..4aa813e 100644--- a/src/heretic/model.py+++ b/src/heretic/model.py@@ -76,7 +76,6 @@ class Model:self.tokenizer = AutoTokenizer.from_pretrained(settings.model,- trust_remote_code=settings.trust_remote_code,**self.revision_kwargs,)@@ -85,7 +84,6 @@ class Model:if get_model_class(settings.model) == AutoModelForImageTextToText:self.processor = AutoProcessor.from_pretrained(settings.model,- trust_remote_code=settings.trust_remote_code,**self.revision_kwargs,)@@ -104,10 +102,8 @@ class Model:if settings.max_memoryelse None)- self.trusted_models = {settings.model: settings.trust_remote_code}- if self.settings.evaluate_model is not None:- self.trusted_models[settings.evaluate_model] = settings.trust_remote_code+ self.trusted_models = set()for dtype in settings.dtypes:print(f"* Trying dtype [bold]{dtype}[/]...")@@ -126,16 +122,18 @@ class Model:dtype=dtype,device_map=settings.device_map,max_memory=self.max_memory,- trust_remote_code=self.trusted_models.get(settings.model),+ trust_remote_code=True+ if settings.model in self.trusted_models+ else None,**self.revision_kwargs,**extra_kwargs,)self.dtype = self.model.dtype# If we reach this point and the model requires trust_remote_code,- # either the user accepted, or settings.trust_remote_code is True.- if self.trusted_models.get(settings.model) is None:- self.trusted_models[settings.model] = True+ # the user must have agreed when prompted to execute remote code,+ # because from_pretrained raises an exception otherwise.+ self.trusted_models.add(settings.model)# A test run can reveal dtype-related problems such as the infamous# "RuntimeError: probability tensor contains either `inf`, `nan` or element < 0"@@ -283,7 +281,9 @@ class Model:self.settings.model,torch_dtype=self.model.dtype,device_map="cpu",- trust_remote_code=self.trusted_models.get(self.settings.model),+ trust_remote_code=True+ if self.settings.model in self.trusted_models+ else None,**self.revision_kwargs,)@@ -349,7 +349,9 @@ class Model:dtype=self.dtype,device_map=self.settings.device_map,max_memory=self.max_memory,- trust_remote_code=self.trusted_models.get(self.settings.model),+ trust_remote_code=True+ if self.settings.model in self.trusted_models+ else None,**self.revision_kwargs,**extra_kwargs,)@@ -574,6 +576,10 @@ class Model:W = W - W_org# Use a low-rank SVD to get an approximation of the matrix.r = self.peft_config.r+ # svd_lowrank is randomized:+ # https://github.com/pytorch/pytorch/blob/20919052303c0b5ba87f8bf7e19237dc33ab09d3/torch/_lowrank.py#L108-L109+ # Reseed immediately before the call so restoring a trial is independent of RNG history.+ torch.manual_seed(self.settings.seed)U, S, Vh = torch.svd_lowrank(W, q=2 * r + 4, niter=6)# Truncate it to the part we want to store in the LoRA adapter.# Note: svd_lowrank actually returns V, so transpose it to get Vh.diff --git a/src/heretic/reproduce.py b/src/heretic/reproduce.pyindex 52c0f87..6f82829 100644--- a/src/heretic/reproduce.py+++ b/src/heretic/reproduce.py@@ -1,13 +1,33 @@# SPDX-License-Identifier: AGPL-3.0-or-later# Copyright (C) 2025-2026 Philipp Emanuel Weidmann <[email protected]> + contributors+import json+import platform+import randomimport shutil+from dataclasses import asdict+from enum import IntEnumfrom pathlib import Path+from typing import Any, cast+from urllib.request import urlopen+import cpuinfo+import torchfrom huggingface_hub import HfApi, hf_hub_download-from huggingface_hub.utils import disable_progress_bars, enable_progress_bars+from huggingface_hub.utils import (+ GatedRepoError,+ disable_progress_bars,+ enable_progress_bars,+)+from questionary import Choice+from rich.table import Table-from .utils import print+from .system import (+ get_accelerator_info_dict,+ get_heretic_version_info,+ get_requirements_dict,+)+from .utils import print, prompt_selectdef collect_reproducibles(path: str):@@ -21,6 +41,7 @@ def collect_reproducibles(path: str):models = api.list_models(filter=["heretic", "reproducible"],sort="created_at",+ expand=["gated", "tags"],)found = 0@@ -35,6 +56,12 @@ def collect_reproducibles(path: str):if model.tags is not None and "gguf" in model.tags:continue+ if model.gated:+ try:+ api.auth_check(model.id, repo_type="model")+ except GatedRepoError:+ continue+print(f"[bold]{model.id}[/]...", end="")user, repository = model.id.split("/")@@ -81,3 +108,275 @@ def collect_reproducibles(path: str):print(f"Found: [bold]{found}[/] files")print(f"Downloaded: [bold]{downloaded}[/] files")print(f"Already stored: [bold]{found - downloaded}[/] files")+++def load_reproduction_information(path: str) -> dict[str, Any]:+ if path.lower().startswith(("http://", "https://")):+ # The path is a URL on the web.++ # Obtain raw download URL.+ path = path.replace("/blob/", "/raw/") # Hugging Face, GitHub+ path = path.replace("/src/branch/", "/raw/branch/") # Codeberg++ json_str = urlopen(path).read().decode("utf-8")+ else:+ # The path is (assumed to be) a local file system path.+ json_str = Path(path).read_text(encoding="utf-8")++ return json.loads(json_str)+++class MismatchSeverity(IntEnum):+ LOW = 1+ MEDIUM = 2+ HIGH = 3+ CRITICAL = 4++ def __rich__(self) -> str:+ match self:+ case MismatchSeverity.LOW:+ return "[green]low[/]"+ case MismatchSeverity.MEDIUM:+ return "[yellow]medium[/]"+ case MismatchSeverity.HIGH:+ return "[red]high[/]"+ case MismatchSeverity.CRITICAL:+ return "[bold red]critical[/]"+ case _:+ raise ValueError(f"unknown MismatchSeverity value: {self}")+++def get_package_mismatch_severity(package_name: str) -> MismatchSeverity:+ if package_name in [+ "heretic-llm",+ ]:+ return MismatchSeverity.CRITICAL+ elif package_name in [+ "torch",+ "transformers",+ ]:+ return MismatchSeverity.HIGH+ elif package_name in [+ "accelerate",+ "bitsandbytes",+ "kernels",+ "optuna",+ "peft",+ "tokenizers",+ "triton",+ ]:+ return MismatchSeverity.MEDIUM+ else:+ return MismatchSeverity.LOW+++def format_version_information(version_information: dict[str, Any]) -> str:+ version = version_information["version"]+ metadata = version_information["metadata"]++ if "type" in metadata:+ match metadata["type"]:+ case "pypi":+ return version+ case "git":+ return f"{version}-git+{metadata['url']}@{metadata['commit_hash']}"+ case "local":+ # Append a random number to ensure that two local installations+ # are always considered to be different versions.+ return f"{version}-local-{random.randint(2**16, 2**17)}"+ case _:+ raise ValueError(+ f"unknown metadata.type value in version information: {metadata['type']}"+ )+ else:+ return f"{version}-unknown-{random.randint(2**16, 2**17)}"+++def check_environment(reproduction_information: dict[str, Any]) -> bool:+ mismatch_severity: MismatchSeverity | None = None++ system_mismatches = []+ package_mismatches = []++ def verify(+ mismatch_list: list[tuple[str, Any, Any, MismatchSeverity]],+ name: str,+ this: Any,+ original: Any,+ severity: MismatchSeverity,+ ):+ nonlocal mismatch_severity+ if this != original:+ mismatch_list.append((name, this, original, severity))+ if mismatch_severity is None:+ mismatch_severity = severity+ else:+ mismatch_severity = max(severity, mismatch_severity)++ if "system" in reproduction_information:+ system = reproduction_information["system"]++ verify(+ system_mismatches,+ "Python version",+ platform.python_version(),+ system["python"]["version"],+ MismatchSeverity.LOW,+ )++ verify(+ system_mismatches,+ "Operating system",+ platform.platform(),+ system["os"]["platform"],+ MismatchSeverity.LOW,+ )++ verify(+ system_mismatches,+ "CPU",+ cpuinfo.get_cpu_info().get("brand_raw"),+ system["cpu"]["brand"],+ MismatchSeverity.LOW,+ )++ accelerators = get_accelerator_info_dict()++ verify(+ system_mismatches,+ "Accelerator type",+ accelerators["type"],+ system["accelerators"]["type"],+ MismatchSeverity.HIGH,+ )++ if (+ accelerators["type"]+ and accelerators["type"] == system["accelerators"]["type"]+ ):+ verify(+ system_mismatches,+ accelerators["api_name"],+ accelerators["api_version"],+ system["accelerators"]["api_version"],+ MismatchSeverity.MEDIUM,+ )+ verify(+ system_mismatches,+ "Driver version",+ accelerators["driver_version"],+ system["accelerators"]["driver_version"],+ MismatchSeverity.MEDIUM,+ )+ verify(+ system_mismatches,+ "Devices",+ "\n".join([device["name"] for device in accelerators["devices"]]),+ "\n".join(+ [device["name"] for device in system["accelerators"]["devices"]]+ ),+ MismatchSeverity.MEDIUM,+ )++ else:+ print(+ (+ "[yellow]The provided JSON file does not contain system information. "+ "Some system parameters can affect reproducibility, but due to the lack of system information, "+ "Heretic is unable to verify that those parameters match the original environment. "+ "Reproduction may or may not produce a byte-for-byte identical model.[/]"+ )+ )++ requirements = get_requirements_dict()+ requirements["heretic-llm"] = format_version_information(+ asdict(get_heretic_version_info())+ )+ requirements["torch"] = torch.__version__++ original_requirements = reproduction_information["environment"]["requirements"]+ original_requirements["heretic-llm"] = format_version_information(+ reproduction_information["environment"]["heretic"]+ )+ original_requirements["torch"] = reproduction_information["environment"][+ "pytorch_version"+ ]++ package_names = sorted(requirements.keys() | original_requirements.keys())++ for package_name in package_names:+ verify(+ package_mismatches,+ package_name,+ requirements.get(package_name),+ original_requirements.get(package_name),+ get_package_mismatch_severity(package_name),+ )++ if system_mismatches or package_mismatches:+ print()+ print(+ (+ "[yellow]Your local environment doesn't perfectly match the environment "+ "used to produce the original model. The following components differ:[/]"+ )+ )++ if system_mismatches:+ table = Table()+ table.add_column("Component")+ table.add_column("This system", overflow="fold")+ table.add_column("Original system", overflow="fold")+ table.add_column("Severity", width=8)++ for component, this, original, severity in system_mismatches:+ table.add_row(f"[bold]{component}[/]", this, original, severity)++ print()+ print("[bold]System Mismatches[/]")+ print(table)++ if package_mismatches:+ table = Table()+ table.add_column("Package")+ table.add_column("This system", overflow="fold")+ table.add_column("Original system", overflow="fold")+ table.add_column("Severity", width=8)++ for package, this, original, severity in package_mismatches:+ table.add_row(f"[bold]{package}[/]", this, original, severity)++ print()+ print("[bold]Package Mismatches[/]")+ print(table)++ if system_mismatches or package_mismatches:+ print()+ print(+ (+ f"There is a {cast(MismatchSeverity, mismatch_severity).__rich__()} chance "+ "that reproduction won't produce a byte-for-byte identical model. "+ "However, the resulting model will very likely still behave similarly "+ "to the original model."+ )+ )++ print()+ choice = prompt_select(+ "How would you like to proceed?",+ [+ Choice(+ title="Attempt to reproduce the model anyway",+ value=True,+ ),+ Choice(+ title="Exit program",+ value=False,+ ),+ ],+ )++ return choice+ else:+ # There are no mismatches at all, so there is nothing to confirm.+ return Truediff --git a/src/heretic/utils.py b/src/heretic/utils.pyindex 3d2d788..2e5924e 100644--- a/src/heretic/utils.py+++ b/src/heretic/utils.py@@ -2,6 +2,7 @@# Copyright (C) 2025-2026 Philipp Emanuel Weidmann <[email protected]> + contributorsimport getpass+import hashlibimport jsonimport osimport platform@@ -25,6 +26,7 @@ from datasets.download.download_manager import DownloadModefrom datasets.utils.info_utils import VerificationModefrom huggingface_hub.utils import validate_repo_idfrom optuna import Trial+from optuna.trial import FrozenTrialfrom psutil import Processfrom questionary import Choice, Stylefrom rich.console import Console@@ -286,7 +288,7 @@ def batchify(items: list[T], batch_size: int) -> list[list[T]]:return [items[i : i + batch_size] for i in range(0, len(items), batch_size)]-def get_trial_parameters(trial: Trial) -> dict[str, str]:+def get_trial_parameters(trial: Trial | FrozenTrial) -> dict[str, str]:params = {}direction_index = trial.user_attrs["direction_index"]@@ -303,7 +305,7 @@ def get_trial_parameters(trial: Trial) -> dict[str, str]:def get_readme_intro(settings: Settings,- trial: Trial,+ trial: Trial | FrozenTrial,contains_reproducibility_information: bool,) -> str:if is_hf_path(settings.model):@@ -395,7 +397,7 @@ def format_hf_link(def generate_reproduce_readme(settings: Settings,checkpoint_filename: str,- trial: Trial,+ trial: Trial | FrozenTrial,include_system_information: bool,) -> str:"""Generates the contents of a README.md for the reproduce/ folder."""@@ -547,13 +549,18 @@ This directory contains the necessary information and assets to reproduce the re## How to reproduce+> [!TIP]+> You can automate this process, including all verification steps, by downloading the `reproduce.json` file and running+> `heretic --reproduce reproduce.json`.+{system_instructions}1. Install the exact version of Heretic indicated in the **Environment** section above, from its original source.1. Install the packages listed in `requirements.txt`: `pip install -r requirements.txt`1. Install the correct version of PyTorch: `{pytorch_install_command}`1. Place the provided `config.toml` in your working directory.1. Run Heretic without any additional arguments: `heretic`1. Wait for the run to finish, then select trial **{trial.user_attrs["index"]}** and export the model.-1. Verify that the weight files have been exactly reproduced by comparing their SHA-256 hashes against those in `SHA256SUMS`: `sha256sum -c SHA256SUMS` (or look at the hashes online if you uploaded to Hugging Face)+1. Verify that the weight files have been exactly reproduced by comparing their SHA-256 hashes against those in `SHA256SUMS`:+ `sha256sum -c SHA256SUMS` (or look at the hashes online if you uploaded to Hugging Face)> [!TIP]> To use the included Optuna study journal `{checkpoint_filename}`, place it in the checkpoints directory (usually `checkpoints/`) before running Heretic.@@ -564,7 +571,7 @@ This directory contains the necessary information and assets to reproduce the redef generate_reproduce_json(settings: Settings,- trial: Trial,+ trial: Trial | FrozenTrial,timestamp: str,uploaded_model_hashes: dict[str, str],include_system_information: bool,@@ -574,7 +581,7 @@ def generate_reproduce_json(version_info = get_heretic_version_info()data = {- "version": "1", # Version number of the reproduce.json file format, to allow for future changes.+ "version": "2", # Version number of the reproduce.json file format, to allow for future changes."timestamp": timestamp,"system": None, # Defined here to preserve insertion order."environment": {@@ -628,11 +635,23 @@ def generate_sha256sums(hashes: dict[str, str]) -> str:return "\n".join(lines) + "\n"+# TODO: Replace this with hashlib.file_digest when we drop support for Python 3.10.+def get_file_sha256(file_path: str | Path) -> str:+ hash = hashlib.sha256()++ with open(file_path, "rb") as file:+ # Read the file in 64 kB blocks.+ for block in iter(lambda: file.read(65536), b""):+ hash.update(block)++ return hash.hexdigest()++def create_reproduce_folder(path: Path,settings: Settings,checkpoint_path: str | Path,- trial: Trial,+ trial: Trial | FrozenTrial,uploaded_model_hashes: dict[str, str],include_system_information: bool,):@@ -706,7 +725,7 @@ def upload_reproduce_folder(settings: Settings,token: str,checkpoint_path: str | Path,- trial: Trial,+ trial: Trial | FrozenTrial,include_system_information: bool,):api = huggingface_hub.HfApi()