Python
from __future__ import annotations
import json
import os
import re
import sys
import time
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any, Optional
import pandas as pd
from openai import (
APIConnectionError,
APIStatusError,
APITimeoutError,
OpenAI,
RateLimitError,
)
from openpyxl import load_workbook
from openpyxl.styles import Alignment, Font, PatternFill
from tqdm import tqdm
# =============================================================================
# Config — edit here
# =============================================================================
current_dir = os.path.dirname(os.path.abspath(__file__))
# =============================================================================
# HOW TO USE THE INPUT/OUTPUT MODES -- READ THIS BEFORE CHANGING CONFIG
#
# This script has TWO input modes, controlled by USE_FOLDER_MODE:
#
# -------------------------------------------------------------------------
# 1) SINGLE-FILE MODE (USE_FOLDER_MODE = False)
# Best for evaluating exactly ONE model result.
#
# INPUT:
# - The script reads exactly ONE Excel file: INPUT_XLSX.
# - In this repository the default file is:
# src/Cognivia_response.xlsx
# (that file is generated by src/fine-tuned_model_generate.py).
# - To score a different single file, just change INPUT_XLSX below.
#
# OUTPUT:
# - Results are written to OUTPUT_XLSX (default):
# src/evaluation_with_existing_criteria.xlsx
#
# Example single-file setup:
# 1) Put the model output at: src/Cognivia_response.xlsx
# 2) Run: python src/evaluate_with_existing_criteria.py
#
# -------------------------------------------------------------------------
# 2) FOLDER / MULTI-MODEL MODE (USE_FOLDER_MODE = True)
# Best for comparing several models at once.
#
# INPUT:
# - The script scans EVERY *.xlsx inside INPUT_DIR.
# - The file name (without .xlsx) is used as the model name, so:
# modelA.xlsx -> model "modelA"
# modelB.xlsx -> model "modelB"
# - Default input folder:
# src/baseline/
# - Steps to use this mode:
# a) Create the folder first: src/baseline/
# b) Put one Excel file per model inside it, e.g.:
# src/baseline/modelA.xlsx
# src/baseline/modelB.xlsx
# (every file must contain the columns listed under
# "Input Excel column names" below, e.g. user1 / assistant1)
# c) Run the script.
#
# OUTPUT:
# - Each model gets its own result file:
# src/evaluation_with_existing_criteria/<model>_eval.xlsx
# - When >=2 models are present, a comparison summary is also written:
# src/evaluation_with_existing_criteria/all_models_compare.xlsx
#
# -------------------------------------------------------------------------
# IMPORTANT PITFALL ABOUT INPUT (the most common mistake):
# * Even when USE_FOLDER_MODE = True, folder mode ONLY actually runs if
# INPUT_DIR exists AND contains at least one *.xlsx file.
# * If INPUT_DIR is missing or empty, the code silently falls back to
# SINGLE-FILE mode and reads INPUT_XLSX instead.
# * Therefore:
# - To really enable FOLDER mode you MUST create INPUT_DIR first and put
# the model .xlsx files inside it.
# - To use SINGLE-FILE mode just keep USE_FOLDER_MODE = False. You do NOT
# need to create the baseline folder at all.
#
# -------------------------------------------------------------------------
# OUTPUT PATH SUMMARY:
# * USE_FOLDER_MODE = False -> results go to OUTPUT_XLSX.
# * USE_FOLDER_MODE = True -> results go to
# OUTPUT_DIR/<model_name>_eval.xlsx
# (+ all_models_compare.xlsx when >=2 models).
# * NOTE: if USE_FOLDER_MODE = True but INPUT_DIR is missing, only the INPUT
# falls back to the single file; the OUTPUT is still written under
# OUTPUT_DIR. If you want BOTH input and output to stay single-file, keep
# USE_FOLDER_MODE = False.
# =============================================================================
# --- Single file mode ---
# Input file to score (used when USE_FOLDER_MODE = False).
# Point this to any single model-result Excel file to evaluate it.
INPUT_XLSX = os.path.join(current_dir, "Cognivia_response.xlsx")
# Output file for single-file mode. Scores are written here.
OUTPUT_XLSX = os.path.join(current_dir, "evaluation_with_existing_criteria.xlsx")
# --- Multi-model folder mode ---
# Input folder for folder mode: every *.xlsx inside it is treated as one model's
# output. Example: create src/baseline/ and put modelA.xlsx, modelB.xlsx there.
INPUT_DIR = os.path.join(current_dir, "baseline")
# Output folder for folder mode: each model gets <model>_eval.xlsx here.
OUTPUT_DIR = os.path.join(current_dir, "evaluation_with_existing_criteria")
# Comparison summary file (generated when >=2 models are evaluated).
OUTPUT_COMPARE_XLSX = os.path.join(OUTPUT_DIR, "all_models_compare.xlsx")
# Master switch: False = single-file mode; True = folder mode.
USE_FOLDER_MODE = False
# Folder mode only: when INPUT_DIR contains >=2 models:
# True = relative cross-model scoring on the same Thought (recommended,
# it distinguishes strong models better);
# False = absolute scoring for each model independently.
# In single-file mode this flag has no effect (always absolute scoring).
USE_RELATIVE_MULTI_MODEL = True
API_KEY_ENV = "YOUR_SiliconFlow_API"
API_KEY = ""
BASE_URL = "https://api.siliconflow.cn/v1"
MODEL = "deepseek-ai/DeepSeek-V3"
LIMIT = None
TEMPERATURE = 0.0
MAX_RETRIES = 5
TIMEOUT = 180.0
SHEET = 0
SLEEP_BETWEEN = 0.15
# --- Input Excel column names ---
# These must match the columns of the Excel file(s) you are scoring.
# In this repository the default file comes from fine-tuned_model_generate.py:
# user1 = the user question / Thought
# assistant1 = the model-generated Rational Response
# If your input files use different column names, edit these values so the
# script can find the columns automatically.
COL_THOUGHT = "user1" # column holding the question / Thought
COL_ANSWER = "assistant1" # column holding the model answer / Rational Response
COL_CONTEXT = "context" # optional context column; if missing it is left empty
SYSTEM_PROMPT = """You are a strict expert rater for CBT counseling responses.
You evaluate using the paper's three constructs: Relevance, CBT Structure, Helpfulness.
You assign FINE integer scores 0-10 (high resolution).
Be harsh. Do not inflate. Reserve high scores for clear excellence.
Output valid JSON only. No markdown. No reasons. No extra keys."""
ABS_PROMPT = """Rate ONE Rational Response for the Thought.
## Thought
{question}
## Rational Response
{answer}
{context_block}
## Paper constructs (score each as FINE 0-10 first)
(1) Relevance: association between answer and THIS Thought/question.
(2) CBT Structure: adherence to CBT structures and principles on THIS Thought
(automatic thought, distortion, evidence, restructuring, experiment/activation,
skills, Socratic questions, collaborative stance — applied, not just named).
(3) Helpfulness: psychotherapy applicability/usefulness for THIS client thought.
## Fine scale anchors (0-10) — use the FULL scale
0-2 = poor / missing
3-4 = weak / mostly generic
5-6 = partial / mixed
7 = adequate but not strong
8 = good
9 = very good / near excellent
10 = exceptional / textbook quality for this Thought
## Strict rules
- Integers only, 0-10 inclusive.
- Default mid-range when unsure; do NOT default to 8-10.
- Fluent/polite/long text alone does not raise scores.
- Generic reassurance => low on all three.
- Naming "CBT" without applying structure => low CBT Structure.
- Score 9-10 only if clearly outstanding for THIS Thought.
## Output JSON only
{{
"relevance_10": 0,
"cbt_structure_10": 0,
"helpfulness_10": 0
}}
"""
REL_PROMPT = """Compare and rate MULTIPLE model answers for the SAME Thought.
Goal: discriminate quality among strong answers. Absolute ceiling scores are forbidden unless truly exceptional.
## Thought
{question}
## Candidate answers
{candidates_block}
## Paper constructs (fine score 0-10 each, per candidate)
(1) Relevance: association degree with THIS Thought.
(2) CBT Structure: adheres to CBT structures/principles on THIS Thought.
(3) Helpfulness: psychotherapy applicability/usefulness for THIS Thought.
## Fine scale (0-10)
0-2 poor | 3-4 weak/generic | 5-6 partial | 7 adequate | 8 good | 9 very good | 10 exceptional
## Relative discrimination rules (critical)
- Compare candidates against EACH OTHER, not against a weak imaginary baseline.
- If quality differs, scores MUST differ (spread at least 1-2 points on the best-separating metric).
- Do NOT give all candidates the same triple unless answers are near-identical in quality.
- Do NOT give everyone 8-10. Most competent answers should land around 5-8.
- 9-10 only for clearly best / textbook answers on that metric.
- 0-4 for generic, off-target, non-CBT, or empty reassurance.
- Integers only. No reasons.
## Output JSON only
{{
"scores": [
{{
"model_name": "<exact candidate model_name>",
"relevance_10": 0,
"cbt_structure_10": 0,
"helpfulness_10": 0
}}
]
}}
Include EVERY candidate exactly once. model_name must match exactly.
"""
# =============================================================================
# Helpers
# =============================================================================
@dataclass
class ScoreRow:
model_name: str
thought_id: str
thought: str
rational_response: str
relevance_10: Optional[int] = None
cbt_structure_10: Optional[int] = None
helpfulness_10: Optional[int] = None
total_10: Optional[int] = None
status: str = "ok"
error: str = ""
scoring_mode: str = "absolute"
def get_client() -> OpenAI:
api_key = (API_KEY or "").strip() or os.getenv(API_KEY_ENV) or os.getenv("OPENAI_API_KEY") or ""
if not api_key:
raise RuntimeError(
f"Missing API key. Set env {API_KEY_ENV} (or OPENAI_API_KEY), or fill API_KEY."
)
return OpenAI(api_key=api_key, base_url=BASE_URL, timeout=TIMEOUT)
def find_column(df: pd.DataFrame, preferred: str, aliases: list[str]) -> str:
lower_map = {str(c).strip().lower(): c for c in df.columns}
for name in [preferred] + aliases:
if name.strip().lower() in lower_map:
return lower_map[name.strip().lower()]
raise KeyError(f"Need column one of {[preferred] + aliases}. Got: {list(df.columns)}")
def extract_json(text: str) -> dict[str, Any]:
text = (text or "").strip()
if not text:
raise ValueError("Empty model response")
fence = re.search(r"```(?:json)?\s*([\s\S]*?)\s*```", text, re.I)
if fence:
text = fence.group(1).strip()
try:
obj = json.loads(text)
if isinstance(obj, dict):
return obj
except json.JSONDecodeError:
pass
m = re.search(r"\{[\s\S]*\}", text)
if not m:
raise ValueError(f"No JSON found: {text[:300]}")
obj = json.loads(m.group(0))
if not isinstance(obj, dict):
raise ValueError("JSON root is not object")
return obj
def clamp_10(v: Any) -> int:
if isinstance(v, bool):
raise ValueError("bool not allowed")
if isinstance(v, float) and v.is_integer():
v = int(v)
if isinstance(v, str):
v = v.strip()
v = int(float(v)) if re.fullmatch(r"-?\d+(\.0+)?", v) else int(v)
if not isinstance(v, int):
raise ValueError(f"Need int 0-10, got {v!r}")
if v < 0 or v > 10:
raise ValueError(f"Out of range 0-10: {v}")
return v
def parse_triple_10(obj: dict[str, Any]) -> tuple[int, int, int]:
def pick(keys: list[str]) -> Any:
lower = {str(k).lower(): val for k, val in obj.items()}
for k in keys:
if k in obj:
return obj[k]
if k.lower() in lower:
return lower[k.lower()]
raise KeyError(f"Missing {keys} in {list(obj.keys())}")
r = clamp_10(pick(["relevance_10", "relevance", "relevance_measure"]))
c = clamp_10(pick(["cbt_structure_10", "cbt_structure", "cbt", "cbt_structure_measure"]))
h = clamp_10(pick(["helpfulness_10", "helpfulness", "help", "helpfulness_measure"]))
return r, c, h
def chat_json(client: OpenAI, user: str) -> dict[str, Any]:
last_err: Optional[Exception] = None
for attempt in range(1, MAX_RETRIES + 1):
try:
kwargs: dict[str, Any] = dict(
model=MODEL,
temperature=TEMPERATURE,
messages=[
{"role": "system", "content": SYSTEM_PROMPT},
{"role": "user", "content": user},
],
)
try:
resp = client.chat.completions.create(
**kwargs, response_format={"type": "json_object"}
)
except Exception as e_rf:
if "response_format" in str(e_rf).lower() or "json" in str(e_rf).lower():
resp = client.chat.completions.create(**kwargs)
else:
raise
return extract_json(resp.choices[0].message.content or "")
except Exception as e:
last_err = e
time.sleep(min(2**attempt, 20))
raise RuntimeError(f"Judge failed after retries: {last_err}")
def score_absolute(client: OpenAI, thought: str, answer: str, context: str = "") -> tuple[int, int, int]:
ctx = ""
if context and str(context).strip().lower() not in {"", "nan", "none"}:
ctx = f"## Extra context\n{context}\n"
user = ABS_PROMPT.format(question=thought.strip(), answer=answer.strip(), context_block=ctx)
return parse_triple_10(chat_json(client, user))
def score_relative_group(
client: OpenAI,
thought: str,
items: list[dict[str, str]],
) -> dict[str, tuple[int, int, int]]:
"""items: [{model_name, answer}, ...] -> model_name -> (r10,c10,h10)"""
blocks = []
for i, it in enumerate(items, 1):
blocks.append(
f"### Candidate {i}\n"
f"model_name: {it['model_name']}\n"
f"Rational Response:\n{it['answer']}\n"
)
user = REL_PROMPT.format(question=thought.strip(), candidates_block="\n".join(blocks))
obj = chat_json(client, user)
scores = obj.get("scores")
if not isinstance(scores, list) or not scores:
# sometimes model returns dict keyed by model
if all(k in obj for k in ("relevance_10", "cbt_structure_10", "helpfulness_10")) and len(items) == 1:
return {items[0]["model_name"]: parse_triple_10(obj)}
raise ValueError(f"Bad relative JSON: {list(obj.keys())}")
out: dict[str, tuple[int, int, int]] = {}
for row in scores:
if not isinstance(row, dict):
continue
name = str(row.get("model_name", "")).strip()
if not name:
continue
out[name] = parse_triple_10(row)
# fuzzy match if judge slightly renames
final: dict[str, tuple[int, int, int]] = {}
for it in items:
m = it["model_name"]
if m in out:
final[m] = out[m]
continue
# case-insensitive / substring
hit = None
for k, v in out.items():
if k.lower() == m.lower() or m.lower() in k.lower() or k.lower() in m.lower():
hit = v
break
if hit is None:
raise ValueError(f"Missing scores for model_name={m}; got {list(out.keys())}")
final[m] = hit
return final
def pack_row(
model_name: str,
thought_id: str,
thought: str,
answer: str,
r10: int,
c10: int,
h10: int,
mode: str,
) -> ScoreRow:
return ScoreRow(
model_name=model_name,
thought_id=thought_id,
thought=thought,
rational_response=answer,
relevance_10=r10,
cbt_structure_10=c10,
helpfulness_10=h10,
total_10=r10 + c10 + h10,
status="ok",
scoring_mode=mode,
)
def normalize_thought(s: str) -> str:
return re.sub(r"\s+", " ", str(s).strip())
def load_model_tables(files: list[Path]) -> dict[str, pd.DataFrame]:
tables: dict[str, pd.DataFrame] = {}
for f in files:
df = pd.read_excel(f, sheet_name=SHEET)
thought_col = find_column(df, COL_THOUGHT, ["thought", "question", "问题", "来访者问题"])
answer_col = find_column(
df, COL_ANSWER, ["rational response", "rational_response", "response", "answer", "回答"]
)
ctx_col = None
try:
ctx_col = find_column(df, COL_CONTEXT, ["context", "背景"])
except KeyError:
pass
out = pd.DataFrame(
{
"thought": df[thought_col].map(lambda x: normalize_thought(x) if pd.notna(x) else ""),
"answer": df[answer_col].map(lambda x: "" if pd.isna(x) else str(x).strip()),
}
)
if ctx_col:
out["context"] = df[ctx_col]
else:
out["context"] = ""
out = out[out["thought"] != ""].reset_index(drop=True)
tables[f.stem] = out
return tables
def evaluate_absolute_one(client: OpenAI, model_name: str, df: pd.DataFrame) -> pd.DataFrame:
n = len(df) if LIMIT is None else min(len(df), int(LIMIT))
rows: list[ScoreRow] = []
for i in tqdm(range(n), desc=f"Abs {model_name}"):
thought = df.iloc[i]["thought"]
answer = df.iloc[i]["answer"]
context = df.iloc[i].get("context", "")
if not answer:
rows.append(
ScoreRow(
model_name=model_name,
thought_id=str(i),
thought=thought,
rational_response="",
status="failed",
error="empty Rational Response",
scoring_mode="absolute",
)
)
continue
try:
r, c, h = score_absolute(client, thought, answer, str(context))
rows.append(pack_row(model_name, str(i), thought, answer, r, c, h, "absolute"))
except Exception as e:
rows.append(
ScoreRow(
model_name=model_name,
thought_id=str(i),
thought=thought,
rational_response=answer,
status="failed",
error=str(e)[:500],
scoring_mode="absolute",
)
)
if SLEEP_BETWEEN:
time.sleep(SLEEP_BETWEEN)
return pd.DataFrame([asdict(r) for r in rows])
def evaluate_relative_all(client: OpenAI, tables: dict[str, pd.DataFrame]) -> pd.DataFrame:
"""Align by Thought text; score all models together for discrimination."""
model_names = sorted(tables.keys())
# thought -> model -> answer
thought_map: dict[str, dict[str, str]] = {}
for m, df in tables.items():
for _, row in df.iterrows():
t = row["thought"]
if not t:
continue
thought_map.setdefault(t, {})[m] = row["answer"]
thoughts = list(thought_map.keys())
if LIMIT is not None:
thoughts = thoughts[: int(LIMIT)]
rows: list[ScoreRow] = []
for ti, thought in enumerate(tqdm(thoughts, desc="Relative multi-model")):
items = []
for m in model_names:
ans = thought_map[thought].get(m, "")
if ans:
items.append({"model_name": m, "answer": ans})
if len(items) == 0:
continue
if len(items) == 1:
# only one model has this thought
m, ans = items[0]["model_name"], items[0]["answer"]
try:
r, c, h = score_absolute(client, thought, ans)
rows.append(pack_row(m, str(ti), thought, ans, r, c, h, "absolute_fallback"))
except Exception as e:
rows.append(
ScoreRow(
model_name=m,
thought_id=str(ti),
thought=thought,
rational_response=ans,
status="failed",
error=str(e)[:500],
scoring_mode="absolute_fallback",
)
)
if SLEEP_BETWEEN:
time.sleep(SLEEP_BETWEEN)
continue
try:
scored = score_relative_group(client, thought, items)
for it in items:
m = it["model_name"]
r, c, h = scored[m]
rows.append(pack_row(m, str(ti), thought, it["answer"], r, c, h, "relative"))
except Exception as e:
# fallback: absolute per model so run continues
for it in items:
m, ans = it["model_name"], it["answer"]
try:
r, c, h = score_absolute(client, thought, ans)
rows.append(pack_row(m, str(ti), thought, ans, r, c, h, "absolute_fallback"))
except Exception as e2:
rows.append(
ScoreRow(
model_name=m,
thought_id=str(ti),
thought=thought,
rational_response=ans,
status="failed",
error=f"relative:{e}; abs:{e2}"[:500],
scoring_mode="failed",
)
)
if SLEEP_BETWEEN:
time.sleep(SLEEP_BETWEEN)
return pd.DataFrame([asdict(r) for r in rows])
def summarize(result_df: pd.DataFrame) -> dict[str, Any]:
ok = result_df[result_df["status"] == "ok"]
s: dict[str, Any] = {
"judge_model": MODEL,
"scoring_method": "fine_0_10 (+ relative multi-model when enabled)",
"fine_metrics": "relevance_10 / cbt_structure_10 / helpfulness_10 (0-10 each; total 0-30)",
"n_total": int(len(result_df)),
"n_success": int(len(ok)),
"n_failed": int((result_df["status"] != "ok").sum()),
}
if len(ok) == 0:
return s
for col in [
"relevance_10",
"cbt_structure_10",
"helpfulness_10",
"total_10",
]:
if col not in ok.columns:
continue
vals = ok[col].dropna().astype(float)
s[f"{col}_mean"] = round(float(vals.mean()), 4)
return s
def style_excel(path: Path) -> None:
wb = load_workbook(path)
header_fill = PatternFill("solid", fgColor="1F4E79")
header_font = Font(color="FFFFFF", bold=True)
avg_fill = PatternFill("solid", fgColor="FFF2CC")
for ws in wb.worksheets:
if ws.max_row >= 1:
for cell in ws[1]:
cell.fill = header_fill
cell.font = header_font
cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)
for row in ws.iter_rows(min_row=2, max_row=ws.max_row):
first = row[0].value
if first is not None and str(first).strip().upper() == "AVERAGE":
for cell in row:
cell.fill = avg_fill
cell.font = Font(bold=True)
ws.freeze_panes = "A2"
for col in ws.columns:
letter = col[0].column_letter
maxlen = 0
for cell in col[:40]:
if cell.value is not None:
maxlen = max(maxlen, min(len(str(cell.value)), 60))
ws.column_dimensions[letter].width = max(10, maxlen + 2)
wb.save(path)
def add_average_row(df: pd.DataFrame) -> pd.DataFrame:
ok = df[df["status"] == "ok"]
if len(ok) == 0:
return df
avg: dict[str, Any] = {c: "" for c in df.columns}
avg["model_name"] = "AVERAGE"
if "status" in avg:
avg["status"] = "average"
for col in [
"relevance_10",
"cbt_structure_10",
"helpfulness_10",
"total_10",
]:
if col in ok.columns:
avg[col] = round(float(ok[col].astype(float).mean()), 4)
return pd.concat([df, pd.DataFrame([avg])], ignore_index=True)
def save_one_result(result_df: pd.DataFrame, summary: dict[str, Any], out_path: Path) -> None:
out_path.parent.mkdir(parents=True, exist_ok=True)
detailed = add_average_row(result_df)
avg_df = pd.DataFrame(
{
"metric": [
"relevance_10 (fine)",
"cbt_structure_10 (fine)",
"helpfulness_10 (fine)",
"total_10 (fine 0-30)",
],
"mean": [
summary.get("relevance_10_mean"),
summary.get("cbt_structure_10_mean"),
summary.get("helpfulness_10_mean"),
summary.get("total_10_mean"),
],
}
)
summary_df = pd.DataFrame(
[
{
"key": k,
"value": json.dumps(v, ensure_ascii=False) if isinstance(v, (dict, list)) else v,
}
for k, v in summary.items()
]
)
with pd.ExcelWriter(out_path, engine="openpyxl") as writer:
detailed.to_excel(writer, sheet_name="detailed_scores", index=False)
avg_df.to_excel(writer, sheet_name="average_scores", index=False)
summary_df.to_excel(writer, sheet_name="summary", index=False)
style_excel(out_path)
print(f"[OK] Saved -> {out_path}")
def print_summary(summary: dict[str, Any], title: str = "") -> None:
print("\n========================================")
if title:
print(f"Model: {title}")
print(f"Judge : {summary.get('judge_model')}")
print(f"Method: {summary.get('scoring_method')}")
print(f"Total/OK/Fail: {summary.get('n_total')}/{summary.get('n_success')}/{summary.get('n_failed')}")
if summary.get("n_success", 0) > 0:
print(
f"Fine 0-10 R/C/H mean: {summary.get('relevance_10_mean')} / "
f"{summary.get('cbt_structure_10_mean')} / {summary.get('helpfulness_10_mean')} "
f"total_10={summary.get('total_10_mean')}"
)
print(">> Rank models by total_10_mean (fine 0-30).")
print("========================================\n")
def list_input_files() -> list[Path]:
# Input mode is decided here:
# 1) USE_FOLDER_MODE=True AND INPUT_DIR exists -> folder mode
# (reads every *.xlsx under INPUT_DIR; create src/baseline/ and put the
# model files there to use this mode).
# 2) Otherwise -> single-file mode (reads INPUT_XLSX; change INPUT_XLSX to
# point at the single Excel file you want to score).
if USE_FOLDER_MODE and INPUT_DIR and Path(INPUT_DIR).is_dir():
files = sorted(Path(INPUT_DIR).glob("*.xlsx"))
files = [f for f in files if not f.name.startswith("~$")]
if not files:
raise FileNotFoundError(f"No .xlsx in INPUT_DIR: {INPUT_DIR}")
return files
p = Path(INPUT_XLSX)
if not p.exists():
raise FileNotFoundError(f"INPUT_XLSX not found: {p}")
return [p]
def main() -> int:
try:
files = list_input_files()
except FileNotFoundError as e:
print(f"[ERROR] {e}", file=sys.stderr)
return 1
try:
client = get_client()
except RuntimeError as e:
print(f"[ERROR] {e}", file=sys.stderr)
return 1
tables = load_model_tables(files)
Path(OUTPUT_DIR).mkdir(parents=True, exist_ok=True)
relative = bool(USE_FOLDER_MODE and USE_RELATIVE_MULTI_MODEL and len(tables) >= 2)
print(f"[INFO] Judge model : {MODEL}")
print(f"[INFO] Base URL : {BASE_URL}")
print(f"[INFO] Models : {len(tables)} -> {list(tables.keys())}")
print(f"[INFO] Mode : {'RELATIVE multi-model (same Thought)' if relative else 'ABSOLUTE per answer'}")
print(f"[INFO] Scoring : fine 0-10 only (relevance / cbt_structure / helpfulness, total 0-30)")
if relative:
all_df = evaluate_relative_all(client, tables)
# split per model
compare_rows = []
all_details = []
for m in sorted(tables.keys()):
part = all_df[all_df["model_name"] == m].copy()
summary = summarize(part)
summary["source_file"] = str(next(f for f in files if f.stem == m))
summary["model_name"] = m
out_path = Path(OUTPUT_DIR) / f"{m}_eval.xlsx"
save_one_result(part, summary, out_path)
print_summary(summary, title=m)
compare_rows.append(
{
"model_name": m,
"n_success": summary.get("n_success"),
"n_failed": summary.get("n_failed"),
"relevance_10_mean": summary.get("relevance_10_mean"),
"cbt_structure_10_mean": summary.get("cbt_structure_10_mean"),
"helpfulness_10_mean": summary.get("helpfulness_10_mean"),
"total_10_mean": summary.get("total_10_mean"),
}
)
all_details.append(part)
compare_df = pd.DataFrame(compare_rows)
compare_df = compare_df.sort_values(
by=["total_10_mean"],
ascending=False,
na_position="last",
).reset_index(drop=True)
compare_df.insert(0, "rank", range(1, len(compare_df) + 1))
compare_path = Path(OUTPUT_COMPARE_XLSX)
with pd.ExcelWriter(compare_path, engine="openpyxl") as writer:
compare_df.to_excel(writer, sheet_name="model_compare", index=False)
pd.concat(all_details, ignore_index=True).to_excel(
writer, sheet_name="all_detailed", index=False
)
style_excel(compare_path)
print(f"[OK] Multi-model compare -> {compare_path}")
print(compare_df.to_string(index=False))
return 0
# absolute path (1 model or relative disabled)
compare_rows = []
all_details = []
for fpath in files:
m = fpath.stem
print(f"\n[INFO] === {m} (absolute) ===")
result_df = evaluate_absolute_one(client, m, tables[m])
summary = summarize(result_df)
summary["source_file"] = str(fpath)
summary["model_name"] = m
out_path = Path(OUTPUT_DIR) / f"{m}_eval.xlsx"
if not USE_FOLDER_MODE or not INPUT_DIR:
out_path = Path(OUTPUT_XLSX)
save_one_result(result_df, summary, out_path)
print_summary(summary, title=m)
compare_rows.append(
{
"model_name": m,
"n_success": summary.get("n_success"),
"n_failed": summary.get("n_failed"),
"relevance_10_mean": summary.get("relevance_10_mean"),
"cbt_structure_10_mean": summary.get("cbt_structure_10_mean"),
"helpfulness_10_mean": summary.get("helpfulness_10_mean"),
"total_10_mean": summary.get("total_10_mean"),
}
)
all_details.append(result_df)
if len(compare_rows) >= 1:
compare_df = pd.DataFrame(compare_rows).sort_values(
by=["total_10_mean"], ascending=False, na_position="last"
).reset_index(drop=True)
compare_df.insert(0, "rank", range(1, len(compare_df) + 1))
compare_path = Path(OUTPUT_COMPARE_XLSX)
with pd.ExcelWriter(compare_path, engine="openpyxl") as writer:
compare_df.to_excel(writer, sheet_name="model_compare", index=False)
pd.concat(all_details, ignore_index=True).to_excel(
writer, sheet_name="all_detailed", index=False
)
style_excel(compare_path)
print(f"[OK] Compare -> {compare_path}")
print(compare_df.to_string(index=False))
return 0
if __name__ == "__main__":
raise SystemExit(main())