Skip to content
Python

Add files via upload

QI CHEN7c8f72ff43d7
diff --git a/src/evaluation_with_8_dimensions.py b/src/evaluation_with_8_dimensions.py
new file mode 100644
index 0000000..f32d4a2
--- /dev/null
+++ b/src/evaluation_with_8_dimensions.py
@@ -0,0 +1,306 @@
+import json
+import numpy as np
+from openpyxl import load_workbook
+from openpyxl.styles import PatternFill, Font, Alignment, Border, Side
+import pandas as pd
+from openai import OpenAI
+import time
+import os
+
+api_key = os.getenv("YOUR_API_KEY")
+client = OpenAI(
+ api_key=api_key,
+ base_url="https://api.siliconflow.cn/v1"
+)
+FINE_TUNED_MODEL_ID = "Qwen/Qwen2.5-7B-Instruct"
+
+
+def evaluate_text_with_dimensions(text, model="Qwen/Qwen2.5-7B-Instruct", max_retries=3):
+ prompt = f"""Please act as a text quality assessment expert and evaluate the given text according to the following dimensions (1-5 points, 1=very poor, 5=excellent), providing brief justifications.
+
+**Assessment Dimensions:**
+1. **SC Structural Clarity**: Does the text possess a clear structural hierarchy, enabling readers to quickly identify key information and logical flow?
+2. **DO Descriptive Orientation**: Does it use language understandable to the target audience to provide concrete, non-directive descriptions of their experiences and contexts?
+3. **SS Situational Safety**: Does the content completely avoid any perceived risk of inducing feelings of being judged, rushed, or emotionally pressured?
+4. **EV Empathy Validation**: Does it evoke emotional resonance and a sense of being understood and validated in the reader?
+5. **CA Conceptual Accuracy**: Does it accurately explain the underlying mechanisms, avoiding vague, outdated, or unverified statements?
+6. **IC Intervention Clarity**: Are the provided suggestions or steps clearly feasible, allowing users to implement them within a short timeframe?
+7. **CC Collaborative Curiosity**: Does it engage readers through guided exploration rather than didactic instruction, encouraging active participation?
+8. **WF Warmth & Flow**: Is the language natural and warm, reflecting genuine care for the reader, rather than being mechanical or promotional in tone?
+
+**Output Format Requirements:**
+Please output in JSON format containing the following fields:
+- "scores": Scores for each dimension (integer)
+- "reasons": Brief justification for each dimension's score (1-2 sentences)
+- "overall": Overall evaluation (1-2 sentences)
+
+**Text to Evaluate:**
+{text}
+"""
+
+ for attempt in range(max_retries):
+ try:
+ response = client.chat.completions.create(
+ model=model,
+ messages=[{"role": "user", "content": prompt}],
+ temperature=0.2,
+ timeout=30
+ )
+
+ result = response.choices[0].message.content.strip()
+
+ try:
+ start = result.find("{")
+ end = result.rfind("}") + 1
+ json_str = result[start:end]
+ evaluation = json.loads(json_str)
+ return evaluation
+ except json.JSONDecodeError:
+ print(f"JSON parsing failed, attempting to fix... (Retry {attempt + 1})")
+ time.sleep(1)
+ continue
+
+ except Exception as e:
+ print(f"API call failed: {e} (Retry {attempt + 1})")
+ time.sleep(2)
+
+ return {"error": "Evaluation failed", "scores": {}, "reasons": {}, "overall": ""}
+
+
+def batch_evaluate_excel(input_file, text_column, output_file, model="Qwen/Qwen2.5-7B-Instruct"):
+
+
+ print(f"Reading file: {input_file}")
+ df = pd.read_excel(input_file)
+
+ if text_column not in df.columns:
+ raise ValueError(f"Column '{text_column}' does not exist in the file. Available columns: {list(df.columns)}")
+
+
+ dimension_map = {
+ 'SC': 'Structural Clarity',
+ 'DO': 'Descriptive Orientation',
+ 'SS': 'Situational Safety',
+ 'EV': 'Empathy Validation',
+ 'CA': 'Conceptual Accuracy',
+ 'IC': 'Intervention Clarity',
+ 'CC': 'Collaborative Curiosity',
+ 'WF': 'Warmth & Flow'
+ }
+
+
+ all_results = []
+
+
+ print(f"Starting batch evaluation, total {len(df)} records...")
+ for idx, row in df.iterrows():
+ text = str(row[text_column])
+
+ if not text or pd.isna(text) or text.strip() == "":
+ print(f"Skipping row {idx + 1}: Empty text")
+ all_results.append({
+ 'row_index': idx,
+ 'text': text,
+ 'error': 'Empty text'
+ })
+ continue
+
+ print(f"Evaluating row {idx + 1}/{len(df)}...")
+
+
+ evaluation = evaluate_text_with_dimensions(text, model)
+
+ result = {
+ 'row_index': idx,
+ 'text': text[:100] + "..." if len(text) > 100 else text,
+ 'overall': evaluation.get('overall', ''),
+ 'error': evaluation.get('error', '')
+ }
+
+
+ scores = evaluation.get('scores', {})
+ for dim_code, dim_name in dimension_map.items():
+ result[f'{dim_code}_score'] = scores.get(dim_code, np.nan)
+ result[f'{dim_code}_reason'] = evaluation.get('reasons', {}).get(dim_code, '')
+
+ all_results.append(result)
+
+
+ time.sleep(1)
+
+
+ results_df = pd.DataFrame(all_results)
+
+
+ detailed_df = pd.DataFrame()
+
+ for col in df.columns:
+ detailed_df[col] = df[col]
+
+ for dim_code, dim_name in dimension_map.items():
+ detailed_df[f'{dim_code}_score'] = results_df[f'{dim_code}_score']
+ detailed_df[f'{dim_code}_reason'] = results_df[f'{dim_code}_reason']
+
+ detailed_df['overall_evaluation'] = results_df['overall']
+ detailed_df['evaluation_error'] = results_df['error']
+
+
+ summary_data = []
+ for idx, result in enumerate(all_results):
+ if result['error']:
+ continue
+
+ row_data = {'ID': idx + 1}
+
+ row_data['Empathy Validation'] = result.get('EV_score', np.nan)
+ row_data['Conceptual Accuracy'] = result.get('CA_score', np.nan)
+ row_data['Situational Safety'] = result.get('SS_score', np.nan)
+ row_data['Intervention Clarity'] = result.get('IC_score', np.nan)
+ row_data['Collaborative Curiosity'] = result.get('CC_score', np.nan)
+ row_data['Warmth & Flow'] = result.get('WF_score', np.nan)
+
+ row_data['Structural Clarity'] = result.get('SC_score', np.nan)
+ row_data['Descriptive Orientation'] = result.get('DO_score', np.nan)
+
+ summary_data.append(row_data)
+
+ summary_df = pd.DataFrame(summary_data)
+
+ if not summary_df.empty:
+ avg_row = {'ID': 'Average'}
+ for col in summary_df.columns:
+ if col != 'ID':
+ avg_row[col] = round(summary_df[col].mean(), 1)
+ summary_df = pd.concat([summary_df, pd.DataFrame([avg_row])], ignore_index=True)
+
+ print(f"Saving results to: {output_file}")
+ with pd.ExcelWriter(output_file, engine='openpyxl') as writer:
+ detailed_df.to_excel(writer, sheet_name='Detailed Results', index=False)
+
+ summary_df.to_excel(writer, sheet_name='Summary Scores', index=False)
+
+ dimension_explain = pd.DataFrame([
+ ['SC', 'Structural Clarity'],
+ ['DO', 'Descriptive Orientation'],
+ ['SS', 'Situational Safety'],
+ ['EV', 'Empathy Validation'],
+ ['CA', 'Conceptual Accuracy'],
+ ['IC', 'Intervention Clarity'],
+ ['CC', 'Collaborative Curiosity'],
+ ['WF', 'Warmth & Flow']
+ ], columns=['Abbreviation', 'English Name', 'Chinese Name'])
+
+ dimension_explain.to_excel(writer, sheet_name='Dimension Guide', index=False)
+
+ scoring_guide = pd.DataFrame({
+ 'Score': [1, 2, 3, 4, 5],
+ 'Description': [
+ 'Very Poor - Does not meet the criteria at all',
+ 'Poor - Meets few criteria, significant issues',
+ 'Average - Meets basic criteria, room for improvement',
+ 'Good - Meets most criteria well, minor issues',
+ 'Excellent - Exceeds criteria, outstanding performance'
+ ],
+ })
+
+ scoring_guide.to_excel(writer, sheet_name='Scoring Guide', index=False)
+
+ try:
+ workbook = load_workbook(output_file)
+
+ if 'Summary Scores' in workbook.sheetnames:
+ ws = workbook['Summary Scores']
+
+ header_fill = PatternFill(start_color="366092", end_color="366092", fill_type="solid")
+ header_font = Font(color="FFFFFF", bold=True)
+ header_alignment = Alignment(horizontal="center", vertical="center")
+
+ for cell in ws[1]:
+ cell.fill = header_fill
+ cell.font = header_font
+ cell.alignment = header_alignment
+
+ thin_border = Border(
+ left=Side(style='thin'),
+ right=Side(style='thin'),
+ top=Side(style='thin'),
+ bottom=Side(style='thin')
+ )
+
+ for row in ws.iter_rows(min_row=1, max_row=ws.max_row, min_col=1, max_col=ws.max_column):
+ for cell in row:
+ cell.border = thin_border
+
+ for column in ws.columns:
+ max_length = 0
+ column_letter = column[0].column_letter
+ for cell in column:
+ try:
+ if len(str(cell.value)) > max_length:
+ max_length = len(str(cell.value))
+ except:
+ pass
+ adjusted_width = min(max_length + 2, 30)
+ ws.column_dimensions[column_letter].width = adjusted_width
+
+ if ws.max_row > 1:
+ avg_fill = PatternFill(start_color="FFF2CC", end_color="FFF2CC", fill_type="solid")
+ for cell in ws[ws.max_row]:
+ cell.fill = avg_fill
+
+ for sheet_name in ['Detailed Results', 'Dimension Guide', 'Scoring Guide']:
+ if sheet_name in workbook.sheetnames:
+ ws = workbook[sheet_name]
+ for column in ws.columns:
+ max_length = 0
+ column_letter = column[0].column_letter
+ for cell in column:
+ try:
+ if len(str(cell.value)) > max_length:
+ max_length = len(str(cell.value))
+ except:
+ pass
+ adjusted_width = min(max_length + 2, 50)
+ ws.column_dimensions[column_letter].width = adjusted_width
+
+ workbook.save(output_file)
+ print("Excel formatting completed!")
+
+ except Exception as e:
+ print(f"Formatting failed: {e}")
+
+ print(f"Evaluation completed! Results saved to: {output_file}")
+
+ success_count = len([r for r in all_results if not r.get('error')])
+ fail_count = len([r for r in all_results if r.get('error')])
+
+ print(f"Successfully evaluated: {success_count} records")
+ print(f"Failed evaluations: {fail_count} records")
+
+ return detailed_df, summary_df
+
+
+if __name__ == "__main__":
+ INPUT_FILE = "evaluate.xlsx"
+ TEXT_COLUMN = "cbt_response"
+ OUTPUT_FILE = "evaluation_with_8_dimensions.xlsx"
+ MODEL = "Qwen/Qwen2.5-7B-Instruct"
+
+ try:
+ detailed_results, summary_results = batch_evaluate_excel(
+ input_file=INPUT_FILE,
+ text_column=TEXT_COLUMN,
+ output_file=OUTPUT_FILE,
+ model=MODEL
+ )
+
+ print("\n=== EVALUATION SUMMARY ===")
+ print(f"Total records processed: {len(detailed_results)}")
+ print(f"Successfully evaluated: {len(detailed_results[detailed_results['evaluation_error'].isna()])}")
+ print(f"Failed evaluations: {len(detailed_results[~detailed_results['evaluation_error'].isna()])}")
+
+ print("\n=== SUMMARY SCORES PREVIEW ===")
+ print(summary_results.to_string())
+
+ except Exception as e:
+ print(f"Program execution error: {e}")
\ No newline at end of file