Skip to content
Python

优化 IFMTBench 的 scoring 和 run_eval 逻辑

sunmingrui970a42ea251f18d
diff --git a/IFMTBench/eval/scoring.py b/IFMTBench/eval/scoring.py
index 7bc32bb..f858630 100644
--- a/IFMTBench/eval/scoring.py
+++ b/IFMTBench/eval/scoring.py
@@ -167,6 +167,7 @@ def score_one_item(item: dict, model_response: str) -> dict:
entry = {
"md5": item.get("md5", ""),
+ "instruction_lang": item.get("instruction_lang", ""),
"class": cls_list,
"is_multi_constraint": is_multi,
}
@@ -197,7 +198,7 @@ def batch_score(test_data: list, responses: dict, workers: int = None) -> list:
Args:
test_data: List of test data items
- responses: {md5: response_text} mapping
+ responses: {(md5, instruction_lang): response_text} mapping
workers: Number of concurrent threads
Returns:
@@ -214,9 +215,11 @@ def batch_score(test_data: list, responses: dict, workers: int = None) -> list:
def _worker(idx):
item = test_data[idx]
md5 = item.get("md5", "")
- response = responses.get(md5, "")
+ lang = item.get("instruction_lang", "")
+ response = responses.get((md5, lang), "")
if not response:
return idx, {"md5": md5, "class": get_class_list(item),
+ "instruction_lang": lang,
"final_score": None, "error": "Model output not found"}
return idx, score_one_item(item, response)
diff --git a/IFMTBench/run_eval.py b/IFMTBench/run_eval.py
index 4101ab3..cd470c4 100644
--- a/IFMTBench/run_eval.py
+++ b/IFMTBench/run_eval.py
@@ -56,21 +56,27 @@ def load_jsonl(path: str) -> list:
def load_responses(path: str) -> dict:
"""
- Load model output file, returning a {md5: response} mapping.
+ Load model output file, returning a {(md5, instruction_lang): response} mapping.
+ Also builds a fallback {md5: response} mapping for entries without instruction_lang.
Model output file format: one JSON object per line, must contain:
- md5: unique identifier matching the md5 field in test data
- response: the model's generated translation
+ - instruction_lang (optional): the instruction language used for this response
"""
responses = {}
+ fallback = {}
data = load_jsonl(path)
for item in data:
md5 = item.get("md5", "")
response = item.get("response", "")
+ lang = item.get("instruction_lang", "")
if md5:
- responses[md5] = response
- log.info(f"Loaded {len(responses)} model responses")
- return responses
+ if lang:
+ responses[(md5, lang)] = response
+ fallback[md5] = response
+ log.info(f"Loaded {len(responses)} lang-specific responses, {len(fallback)} unique md5 entries")
+ return {"by_lang": responses, "fallback": fallback}
def main():
@@ -108,16 +114,30 @@ def main():
log.info(f"Total: {len(test_data)} test items")
# Load model responses
- responses = load_responses(args.input_response)
-
- # Check coverage
- test_md5s = {item.get("md5", "") for item in test_data if item.get("md5")}
- response_md5s = set(responses.keys())
- coverage = len(test_md5s & response_md5s) / len(test_md5s) if test_md5s else 0
- log.info(f"Response coverage: {coverage:.1%} ({len(test_md5s & response_md5s)}/{len(test_md5s)})")
+ response_data = load_responses(args.input_response)
+ by_lang = response_data["by_lang"]
+ fallback = response_data["fallback"]
+
+ # Build a flat {(md5, lang): response} lookup for batch_score
+ # For each test item, try (md5, instruction_lang) first, then fallback to md5-only
+ responses_flat = {}
+ matched = 0
+ for item in test_data:
+ md5 = item.get("md5", "")
+ lang = item.get("instruction_lang", "")
+ key = (md5, lang)
+ if key in by_lang:
+ responses_flat[key] = by_lang[key]
+ matched += 1
+ elif md5 in fallback:
+ responses_flat[key] = fallback[md5]
+ matched += 1
+
+ coverage = matched / len(test_data) if test_data else 0
+ log.info(f"Response coverage: {coverage:.1%} ({matched}/{len(test_data)})")
if coverage < 0.5:
- log.warning("Response coverage is below 50%, please check if md5 fields match")
+ log.warning("Response coverage is below 50%, please check if md5/instruction_lang fields match")
# If skipping LLM, temporarily modify config
if args.skip_llm:
@@ -127,7 +147,7 @@ def main():
config.CLASS_LLM_JUDGE = set()
# Run scoring
- results = batch_score(test_data, responses, workers=args.workers)
+ results = batch_score(test_data, responses_flat, workers=args.workers)
# Output results
os.makedirs(args.output_dir, exist_ok=True)