Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-s1): load training examples on demand (#4022)

Iflaq khurshid197e28cfcb4e
diff --git a/libs/cua-s1/python/tests/test_training.py b/libs/cua-s1/python/tests/test_training.py
new file mode 100644
index 0000000000..a43bc31c51
--- /dev/null
+++ b/libs/cua-s1/python/tests/test_training.py
@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import pytest
+from torch.utils.data import DataLoader
+
+TRAINING_ROOT = Path(__file__).resolve().parents[2] / "training"
+sys.path.insert(0, str(TRAINING_ROOT))
+
+from train import MetaDataset # noqa: E402
+
+
+def _write_jsonl(path: Path, rows: list[dict | None]) -> None:
+ path.write_text(
+ "\n".join("" if row is None else json.dumps(row) for row in rows),
+ encoding="utf-8",
+ )
+
+
+def test_meta_dataset_retains_blob_and_parses_rows_on_demand(tmp_path: Path):
+ path = tmp_path / "choices.jsonl"
+ rows = [
+ {"context": "first" * 100, "options": ["click", "skip"], "label": 0},
+ None,
+ {
+ "context": "middle" * 100,
+ "options": ["custom value", "skip", "click"],
+ "label": 0,
+ "meta": {"action": "custom"},
+ },
+ {"context": "last" * 100, "options": ["scroll", "skip"], "label": 1},
+ ]
+ _write_jsonl(path, rows)
+
+ dataset = MetaDataset(path)
+
+ assert len(dataset) == 3
+ assert not hasattr(dataset, "examples")
+ assert dataset.blob == path.read_bytes()
+ assert dataset.actions == ["click", "custom", "skip"]
+ assert dataset[0].context == rows[0]["context"]
+ assert dataset[1].context == rows[2]["context"]
+ assert dataset[-1].context == rows[3]["context"]
+
+
+def test_meta_dataset_supports_shuffled_loader_access(tmp_path: Path):
+ path = tmp_path / "choices.jsonl"
+ _write_jsonl(
+ path,
+ [
+ {"context": f"row-{index}", "options": ["click", "skip"], "label": index % 2}
+ for index in range(12)
+ ],
+ )
+ dataset = MetaDataset(path)
+
+ contexts = [
+ batch[0].context
+ for batch in DataLoader(dataset, batch_size=1, shuffle=True, collate_fn=lambda rows: rows)
+ ]
+
+ assert set(contexts) == {f"row-{index}" for index in range(12)}
+
+
+def test_meta_dataset_reports_original_line_number(tmp_path: Path):
+ path = tmp_path / "invalid.jsonl"
+ _write_jsonl(
+ path,
+ [
+ {"context": "valid", "options": ["click", "skip"], "label": 0},
+ None,
+ {"context": "invalid", "options": ["only one"], "label": 0},
+ ],
+ )
+
+ with pytest.raises(ValueError, match=r"invalid\.jsonl:3$"):
+ MetaDataset(path)
diff --git a/libs/cua-s1/training/train.py b/libs/cua-s1/training/train.py
index aa8e1c0ded..93e2af3d0a 100644
--- a/libs/cua-s1/training/train.py
+++ b/libs/cua-s1/training/train.py
@@ -39,27 +39,40 @@ class MetaDataset(Dataset):
def __init__(self, path: str | Path) -> None:
self.path = Path(path)
- self.examples: list[ChoiceExample] = []
+ self.blob = self.path.read_bytes()
+ self.spans: list[tuple[int, int, int]] = []
self.actions: list[str] = []
- with self.path.open(encoding="utf-8") as handle:
- for line_number, line in enumerate(handle, start=1):
- if not line.strip():
- continue
- try:
- payload = json.loads(line)
- example = validate_example(payload)
- except (AttributeError, json.JSONDecodeError, TypeError, ValueError) as error:
- raise ValueError(f"invalid example at {self.path}:{line_number}") from error
- self.examples.append(example)
+ start = 0
+ line_number = 1
+ while start < len(self.blob):
+ newline = self.blob.find(b"\n", start)
+ end = len(self.blob) if newline == -1 else newline
+ if self.blob[start:end].strip():
+ payload, example = self._parse(start, end, line_number)
+ self.spans.append((start, end, line_number))
metadata = payload.get("meta")
action = metadata.get("action") if isinstance(metadata, Mapping) else None
self.actions.append(action or action_name(example))
+ if newline == -1:
+ break
+ start = newline + 1
+ line_number += 1
+
+ def _parse(self, start: int, end: int, line_number: int) -> tuple[Mapping, ChoiceExample]:
+ try:
+ payload = json.loads(self.blob[start:end])
+ example = validate_example(payload)
+ except (AttributeError, json.JSONDecodeError, TypeError, ValueError) as error:
+ raise ValueError(f"invalid example at {self.path}:{line_number}") from error
+ return payload, example
def __len__(self) -> int:
- return len(self.examples)
+ return len(self.spans)
def __getitem__(self, index: int) -> ChoiceExample:
- return self.examples[index]
+ start, end, line_number = self.spans[index]
+ _, example = self._parse(start, end, line_number)
+ return example
def action_name(example: ChoiceExample) -> str: