SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024]
Fix(run_batch): Avoid result corruption with redo_existing
Kilian Liereta5fc91edbf20
diff --git a/sweagent/run/run_batch.py b/sweagent/run/run_batch.pyindex 105db09f..54514b73 100644--- a/sweagent/run/run_batch.py+++ b/sweagent/run/run_batch.py@@ -42,7 +42,7 @@ from pathlib import Pathfrom typing import Selfimport yaml-from pydantic import Field+from pydantic import Field, model_validatorfrom pydantic_settings import BaseSettingsfrom rich.live import Livefrom swerex.deployment.hooks.status import SetStatusDeploymentHook@@ -107,6 +107,19 @@ class RunBatchConfig(BaseSettings, cli_implicit_flags=False):suffix = f"__{self.suffix}" if self.suffix else ""self.output_dir = Path.cwd() / "trajectories" / user_id / f"{config_file}__{model_id}___{source_id}{suffix}"+ @model_validator(mode="after")+ def evaluate_and_redo_existing(self) -> Self:+ if not isinstance(self.instances, SWEBenchInstances):+ return self+ if self.instances.evaluate and self.redo_existing:+ msg = (+ "Cannot evaluate and redo existing at the same time. This would cause invalid results, because "+ "after the first merge_preds gives you a preds.json, this file would be submitted to SB-CLI, causing"+ "evaluation of old instances, which could then not be overwritten by the new ones."+ )+ raise ValueError(msg)+ return self+class _BreakLoop(Exception):"""Used for internal control flow"""