Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(dataset): allow selecting the QA answer model (#3944)

LingYie6ecb9adcc8d
diff --git a/dataset/qa_dataset.py b/dataset/qa_dataset.py
index 393d848bf..c1baa84fd 100644
--- a/dataset/qa_dataset.py
+++ b/dataset/qa_dataset.py
@@ -15,6 +15,8 @@ python dataset/qa_dataset.py sample \
--output_dir ./dataset/samples
# 生成答案(基于采样结果)
+# 可选:追加 --model <model-name> 指定 OpenAI 兼容服务中的模型
+# 切换模型时使用新的 --output_dir;断点续跑仍会复用已有答案
python dataset/qa_dataset.py generate \
--input_dir ./dataset/samples \
--output_dir ./dataset/samples
@@ -219,14 +221,24 @@ def sample_command(args):
print("\nSampling completed successfully!")
-def generate_answers(input_dir: str, output_dir: str, max_retries: int = 3):
+def generate_answers(
+ input_dir: str,
+ output_dir: str,
+ max_retries: int = 3,
+ *,
+ model: str = DEFAULT_QA_MODEL,
+):
"""
Generate answers for sampled queries with resume support
+ Saved answers are reused even when the model changes. Use a new output
+ directory to regenerate all answers with a different model.
+
Args:
input_dir: Directory containing sampled queries/corpus/qrels
output_dir: Directory to save answer files
max_retries: Maximum retry attempts for failed queries
+ model: OpenAI-compatible model to use for answer generation
"""
print("\nLoading sampled data...")
queries = read_parquet(f"{input_dir}/queries.parquet")
@@ -261,7 +273,7 @@ def generate_answers(input_dir: str, output_dir: str, max_retries: int = 3):
retry_count = 0
while retry_count <= max_retries:
try:
- answer_text = qa_system.answer_question(qid)
+ answer_text = qa_system.answer_question(qid, model=model)
aid = answer_id_counter
answers.append({"id": aid, "text": answer_text})
qa_pairs.append({"qid": qid, "aid": aid})
@@ -332,6 +344,12 @@ def show_results(input_dir: str, n: int = 5):
print("=" * 50 + "\n")
+def _model_argument(value: str) -> str:
+ if not value.strip():
+ raise argparse.ArgumentTypeError("model must not be empty or whitespace-only")
+ return value
+
+
def main():
# Set up command line arguments
parser = argparse.ArgumentParser(description="QA Dataset Tool")
@@ -364,8 +382,19 @@ def main():
generate_parser.add_argument(
"--output_dir", type=str, default="./save", help="Output directory"
)
+ generate_parser.add_argument(
+ "--model",
+ type=_model_argument,
+ default=DEFAULT_QA_MODEL,
+ help=(
+ f"Model for answer generation (default: {DEFAULT_QA_MODEL}). "
+ "Resume reuses saved answers; use a new --output_dir when changing models."
+ ),
+ )
generate_parser.set_defaults(
- func=lambda args: generate_answers(args.input_dir, args.output_dir)
+ func=lambda args: generate_answers(
+ args.input_dir, args.output_dir, model=args.model
+ )
)
# Show command
diff --git a/dataset/tests/test_qa_dataset_model.py b/dataset/tests/test_qa_dataset_model.py
new file mode 100644
index 000000000..3ee548071
--- /dev/null
+++ b/dataset/tests/test_qa_dataset_model.py
@@ -0,0 +1,175 @@
+"""Offline regression tests for the actual WeKnora QA-generation call chain.
+
+Uses real pandas/Parquet input and output, but never a real OpenAI client.
+"""
+
+import argparse
+from contextlib import redirect_stderr, redirect_stdout
+import importlib.util
+import io
+from pathlib import Path
+import sys
+import tempfile
+from types import SimpleNamespace
+import unittest
+from unittest.mock import Mock, patch
+
+import pandas as pd
+
+
+SOURCE = Path(__file__).resolve().parents[1] / "qa_dataset.py"
+DEFAULT_MODEL = "gpt-5.6-sol"
+EXPLICIT_MODEL = "synthetic-compatible-model"
+
+
+class QADatasetModelTests(unittest.TestCase):
+ def enter_context(self, manager):
+ value = manager.__enter__()
+ self.addCleanup(manager.__exit__, None, None, None)
+ return value
+
+ def setUp(self):
+ spec = importlib.util.spec_from_file_location("qa_dataset_under_test", SOURCE)
+ self.module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(self.module)
+ self.directory = tempfile.TemporaryDirectory()
+ self.addCleanup(self.directory.cleanup)
+ self.input_dir = Path(self.directory.name) / "input"
+ self.output_dir = Path(self.directory.name) / "output"
+ self.input_dir.mkdir()
+ self.queries = pd.DataFrame([
+ {"id": "q1", "text": "What color is the synthetic box?"},
+ {"id": "q2", "text": "What shape is the synthetic tile?"},
+ ])
+ self.corpus = pd.DataFrame([
+ {"id": "p1", "text": "The synthetic box is blue."},
+ {"id": "p2", "text": "The synthetic tile is square."},
+ ])
+ self.qrels = pd.DataFrame([
+ {"qid": "q1", "pid": "p1"},
+ {"qid": "q2", "pid": "p2"},
+ ])
+ for name, frame in (("queries", self.queries), ("corpus", self.corpus),
+ ("qrels", self.qrels)):
+ frame.to_parquet(self.input_dir / f"{name}.parquet")
+ self.client = Mock()
+ self.client.chat.completions.create.return_value = SimpleNamespace(
+ choices=[SimpleNamespace(message=SimpleNamespace(content="synthetic answer"))]
+ )
+ self.client_factory = self.enter_context(
+ patch.object(self.module.openai, "Client", return_value=self.client)
+ )
+ self.enter_context(patch.dict("os.environ", {
+ "OPENAI_API_KEY": "local-test-placeholder",
+ "OPENAI_BASE_URL": "https://example.invalid/v1",
+ }))
+ self.enter_context(patch("socket.create_connection", side_effect=AssertionError("network forbidden")))
+ self.enter_context(patch("socket.socket.connect", side_effect=AssertionError("network forbidden")))
+ self.enter_context(redirect_stdout(io.StringIO()))
+
+ def generated_models(self):
+ return [call.kwargs["model"] for call in self.client.chat.completions.create.call_args_list]
+
+ def assert_real_output(self, count=2):
+ answers = pd.read_parquet(self.output_dir / "answers.parquet")
+ pairs = pd.read_parquet(self.output_dir / "qas.parquet")
+ self.assertEqual(len(answers), count)
+ self.assertEqual(len(pairs), count)
+ self.assertEqual(answers["text"].tolist(), ["synthetic answer"] * count)
+
+ def run_cli(self, *extra):
+ argv = [str(SOURCE), "generate", "--input_dir", str(self.input_dir),
+ "--output_dir", str(self.output_dir), *extra]
+ with patch.object(sys, "argv", argv):
+ self.module.main()
+
+ def test_direct_answer_default_is_unchanged(self):
+ system = self.module.QAAnsweringSystem(self.queries, self.corpus, self.qrels)
+ self.assertEqual(system.answer_question("q1"), "synthetic answer")
+ self.assertEqual(self.generated_models(), [DEFAULT_MODEL])
+
+ def test_existing_direct_answer_override_remains_supported(self):
+ system = self.module.QAAnsweringSystem(self.queries, self.corpus, self.qrels)
+ system.answer_question("q1", model=EXPLICIT_MODEL)
+ self.assertEqual(self.generated_models(), [EXPLICIT_MODEL])
+
+ def test_generate_default_retains_legacy_model(self):
+ self.module.generate_answers(str(self.input_dir), str(self.output_dir))
+ self.assertEqual(self.generated_models(), [DEFAULT_MODEL, DEFAULT_MODEL])
+ self.assert_real_output()
+
+ def test_generate_explicit_model_reaches_actual_client(self):
+ self.module.generate_answers(str(self.input_dir), str(self.output_dir), model=EXPLICIT_MODEL)
+ self.assertEqual(self.generated_models(), [EXPLICIT_MODEL, EXPLICIT_MODEL])
+ self.assert_real_output()
+ sent = self.client.chat.completions.create.call_args_list[0].kwargs
+ self.assertIn("The synthetic box is blue.", sent["messages"][0]["content"])
+ self.assertIn("What color is the synthetic box?", sent["messages"][0]["content"])
+ self.assertEqual(sent["temperature"], 0.3)
+
+ def test_cli_without_model_retains_default(self):
+ self.run_cli()
+ self.assertEqual(self.generated_models(), [DEFAULT_MODEL, DEFAULT_MODEL])
+ self.assert_real_output()
+
+ def test_cli_model_reaches_real_generation_chain(self):
+ self.run_cli("--model", EXPLICIT_MODEL)
+ self.assertEqual(self.generated_models(), [EXPLICIT_MODEL, EXPLICIT_MODEL])
+ self.assert_real_output()
+ self.client_factory.assert_called_once_with(
+ api_key="local-test-placeholder", base_url="https://example.invalid/v1"
+ )
+
+ def test_cli_help_exposes_model_option_without_calling_client(self):
+ captured = io.StringIO()
+ with patch.object(sys, "argv", [str(SOURCE), "generate", "--help"]), redirect_stdout(captured):
+ with self.assertRaises(SystemExit) as stopped:
+ self.module.main()
+ self.assertEqual(stopped.exception.code, 0)
+ self.assertIn("--model", captured.getvalue())
+ help_text = " ".join(captured.getvalue().split())
+ self.assertIn("saved answers", help_text)
+ self.assertIn("new --output_dir", help_text)
+ self.client_factory.assert_not_called()
+
+ def assert_invalid_cli_model(self, value):
+ errors = io.StringIO()
+ with patch.object(self.module, "read_parquet") as read_input:
+ with redirect_stderr(errors), self.assertRaises(SystemExit) as stopped:
+ self.run_cli("--model", value)
+ self.assertEqual(stopped.exception.code, 2)
+ self.assertIn("model must not be empty or whitespace-only", errors.getvalue())
+ read_input.assert_not_called()
+ self.client_factory.assert_not_called()
+ self.client.chat.completions.create.assert_not_called()
+ self.assertFalse(self.output_dir.exists())
+
+ def test_cli_rejects_empty_model_before_loading_or_provider(self):
+ self.assert_invalid_cli_model("")
+
+ def test_cli_rejects_whitespace_model_before_loading_or_provider(self):
+ for value in (" ", "\t", "\n"):
+ with self.subTest(value=repr(value)):
+ self.assert_invalid_cli_model(value)
+
+ def test_existing_third_positional_retry_argument_remains_valid(self):
+ self.module.generate_answers(str(self.input_dir), str(self.output_dir), 0)
+ self.assertEqual(self.generated_models(), [DEFAULT_MODEL, DEFAULT_MODEL])
+ self.assert_real_output()
+
+ def test_explicit_model_does_not_change_resume_behavior(self):
+ self.output_dir.mkdir()
+ pd.DataFrame([{"id": 1, "text": "existing answer"}]).to_parquet(self.output_dir / "answers.parquet")
+ pd.DataFrame([{"qid": "q1", "aid": 1}]).to_parquet(self.output_dir / "qas.parquet")
+ self.run_cli("--model", EXPLICIT_MODEL)
+ self.assertEqual(self.generated_models(), [EXPLICIT_MODEL])
+ saved = pd.read_parquet(self.output_dir / "answers.parquet")
+ self.assertEqual(saved["text"].tolist(), ["existing answer", "synthetic answer"])
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(add_help=False)
+ parser.add_argument("--source", type=Path, default=SOURCE)
+ options, remaining = parser.parse_known_args()
+ SOURCE = options.source.resolve()
+ unittest.main(argv=[sys.argv[0], *remaining])