Skip to content

Official Codebase of 3DCodeVerse

Python

toolkits: a full-scan audit for the pair datasets, and the rule it exists to enforce

gaoypengf7c702c1e952
diff --git a/toolkits/llamafactory/README.md b/toolkits/llamafactory/README.md
index 01b77350..9710236f 100644
--- a/toolkits/llamafactory/README.md
+++ b/toolkits/llamafactory/README.md
@@ -57,3 +57,15 @@ as system binaries. Paths to the binaries are constants at the top of each scrip
and earlier 949 vs 318). Deleting the apparent duplicate would have silently discarded 1,239 samples. The
safe order is always: compare, promote the newer copy, then delete — in a single commit, so the repository is
never seen with the data in two places or in neither.
+| `audit_pairs.py` | checks every pair dataset for the invariants that fail only once training has started: absolute paths, wrong extension, image files that do not exist, `<image>` count vs `len(images)`, turn shape, fenced code, empty system, row count vs `qc.json`. Exits non-zero on any finding, so it can gate a publish |
+
+### A fifth rule
+
+5. **Sampling proves a problem exists; it cannot prove one does not.** Checking four datasets said "every path is
+ relative". Scanning all 411 found **36,964 absolute paths** in three of them — introduced by a later rebuild
+ that reused already-extracted renders and took a code path which skipped the relativisation. For a global
+ invariant, scan globally: `audit_pairs.py` exists for exactly this.
+
+ The same audit script then reported all 1,948,765 rows as malformed, which was a defect in the checker
+ (parquet returns numpy arrays, and `isinstance(x, list)` is False for them) — rule 1 applies to your own
+ checkers too.
diff --git a/toolkits/llamafactory/audit_pairs.py b/toolkits/llamafactory/audit_pairs.py
new file mode 100644
index 00000000..3297f383
--- /dev/null
+++ b/toolkits/llamafactory/audit_pairs.py
@@ -0,0 +1,82 @@
+"""Audit every pair dataset for the invariants that fail silently at train time.
+
+Sampling proves a problem exists; it cannot prove one does not. Checking four datasets said "paths are relative";
+a full scan of all 411 found 36,964 absolute paths in three of them. This scans everything.
+
+usage: python scripts/audit_pairs.py [--media_root DIR] [--fix]
+"""
+import argparse
+import collections
+import glob
+import json
+import os
+
+import pandas as pd
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--media_root", default="/wekafs/ict/hx_624/data/pair_media_root")
+ ap.add_argument("--roots", default="/wekafs/ict/hx_624/hf_pairs*")
+ a = ap.parse_args()
+ dirs = sorted(glob.glob(f"{a.roots}/*_llamafactory_*"))
+ bad = collections.Counter()
+ tot = rows = 0
+ examples = {}
+ for d in dirs:
+ kind = os.path.basename(d).rsplit("_", 1)[-1]
+ try:
+ df = pd.read_parquet(f"{d}/train.parquet")
+ except Exception as e:
+ bad["unreadable_parquet"] += 1
+ examples.setdefault("unreadable_parquet", f"{d}: {type(e).__name__}")
+ continue
+ q = json.load(open(f"{d}/qc.json"))
+ if len(df) != q["rows"]:
+ bad["row_count_vs_qc"] += 1
+ for r in df.to_dict("records"):
+ rows += 1
+ # parquet hands back numpy arrays, not lists: an isinstance check against (list, tuple) alone
+ # marks every single row as malformed, which is a defect in the checker, not the data
+ conv = r.get("conversations")
+ try:
+ conv = list(conv)
+ except Exception:
+ conv = None
+ if conv is None or len(conv) != 2:
+ bad["bad_turn_shape"] += 1
+ continue
+ code = conv[1]["value"]
+ if not code.startswith("```") or not code.rstrip().endswith("```"):
+ bad["unfenced_code"] += 1
+ if not str(r.get("system") or "").strip():
+ bad["empty_system"] += 1
+ if kind == "text":
+ if "<image>" in conv[0]["value"]:
+ bad["placeholder_in_text_pair"] += 1
+ continue
+ # `or []` on a numpy array raises rather than defaulting; the same numpy-vs-list confusion as above
+ raw = r.get("images")
+ imgs = [str(x) for x in raw] if raw is not None else []
+ if conv[0]["value"].count("<image>") != len(imgs):
+ bad["placeholder_mismatch"] += 1
+ for x in imgs:
+ tot += 1
+ if x.startswith("/"):
+ bad["absolute_path"] += 1
+ examples.setdefault("absolute_path", x)
+ elif not x.endswith(".jpg"):
+ bad["not_jpg"] += 1
+ examples.setdefault("not_jpg", x)
+ elif not os.path.exists(os.path.join(a.media_root, x)):
+ bad["unresolvable"] += 1
+ examples.setdefault("unresolvable", x)
+ print(f" {len(dirs)} datasets | {rows:,} rows | {tot:,} image references")
+ print(f" problems: {dict(bad) if bad else 'NONE'}")
+ for k, v in examples.items():
+ print(f" example {k}: {v[:100]}")
+ raise SystemExit(1 if bad else 0)
+
+
+if __name__ == "__main__":
+ main()