Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

support multiple evals at once

Abi Raja298cf99ea4d1
diff --git a/backend/routes/evals.py b/backend/routes/evals.py
index c432cd77..da902fce 100644
--- a/backend/routes/evals.py
+++ b/backend/routes/evals.py
@@ -137,11 +137,21 @@ async def get_pairwise_evals(
)
+class RunEvalsRequest(BaseModel):
+ models: List[str]
+ stack: Stack
+
+
@router.post("/run_evals", response_model=List[str])
-async def run_evals(model: str, stack: Stack):
- """Run evaluations on all images in the inputs directory"""
- output_files = await run_image_evals(model=model, stack=stack)
- return output_files
+async def run_evals(request: RunEvalsRequest) -> List[str]:
+ """Run evaluations on all images in the inputs directory for multiple models"""
+ all_output_files: List[str] = []
+
+ for model in request.models:
+ output_files = await run_image_evals(model=model, stack=request.stack)
+ all_output_files.extend(output_files)
+
+ return all_output_files
@router.get("/models", response_model=Dict[str, List[str]])
diff --git a/frontend/src/components/evals/RunEvalsPage.tsx b/frontend/src/components/evals/RunEvalsPage.tsx
index 5c86acf2..bd3c1eff 100644
--- a/frontend/src/components/evals/RunEvalsPage.tsx
+++ b/frontend/src/components/evals/RunEvalsPage.tsx
@@ -13,7 +13,7 @@ function RunEvalsPage() {
const navigate = useNavigate();
const [models, setModels] = useState<string[]>([]);
const [stacks, setStacks] = useState<string[]>([]);
- const [selectedModel, setSelectedModel] = useState<string>("");
+ const [selectedModels, setSelectedModels] = useState<string[]>([]);
const [selectedStack, setSelectedStack] = useState<string>("");
useEffect(() => {
@@ -23,7 +23,7 @@ function RunEvalsPage() {
setModels(data.models);
setStacks(data.stacks);
if (data.models.length > 0) {
- setSelectedModel(data.models[0]);
+ setSelectedModels([data.models[0]]);
}
if (data.stacks.length > 0) {
setSelectedStack(data.stacks[0]);
@@ -36,9 +36,16 @@ function RunEvalsPage() {
try {
setIsRunning(true);
const response = await fetch(
- `${HTTP_BACKEND_URL}/run_evals?model=${selectedModel}&stack=${selectedStack}`,
+ `${HTTP_BACKEND_URL}/run_evals`,
{
method: "POST",
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify({
+ models: selectedModels,
+ stack: selectedStack
+ })
}
);
@@ -57,6 +64,11 @@ function RunEvalsPage() {
}
};
+ const handleModelSelection = (event: React.ChangeEvent<HTMLSelectElement>) => {
+ const selectedOptions = Array.from(event.target.selectedOptions).map(option => option.value);
+ setSelectedModels(selectedOptions);
+ };
+
return (
<div className="flex flex-col items-center justify-center min-h-screen">
<h1 className="text-2xl font-bold mb-8">Run Evaluations</h1>
@@ -79,13 +91,15 @@ function RunEvalsPage() {
<div className="space-y-4 w-full max-w-md">
<div>
<label className="block text-gray-700 text-sm font-bold mb-2">
- Select Model
+ Select Models
</label>
<select
- value={selectedModel}
- onChange={(e) => setSelectedModel(e.target.value)}
+ multiple
+ value={selectedModels}
+ onChange={handleModelSelection}
className="shadow border rounded w-full py-2 px-3 text-gray-700"
required
+ size={4}
>
{models.map((model) => (
<option key={model} value={model}>
@@ -93,6 +107,9 @@ function RunEvalsPage() {
</option>
))}
</select>
+ <p className="text-sm text-gray-500 mt-1">
+ Hold Ctrl/Cmd to select multiple models
+ </p>
</div>
<div>
@@ -114,7 +131,11 @@ function RunEvalsPage() {
</div>
</div>
- <Button onClick={runEvals} disabled={isRunning} className="w-48 mt-6">
+ <Button
+ onClick={runEvals}
+ disabled={isRunning || selectedModels.length === 0}
+ className="w-48 mt-6"
+ >
{isRunning ? "Running Evals..." : "Run Evals"}
</Button>
</div>