Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
Fix NameError: quote README.md string in f-string (cell 16) (#104)
Varghese Thomas535710ba0cc6
diff --git a/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb b/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynbindex 913799cc..21740f3a 100644--- a/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb+++ b/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb@@ -346,6 +346,25 @@" if rank == 0:\n"," print(f\"=== Epoch {epoch+1}/{EPOCHS} Completed in {time.time()-t0:.1f}s | Avg Loss: {epoch_loss/max(1, n_batches):.4f} ===\")\n","\n",+ " dist.barrier()\n",+ "\n",+ " # Overwrite a single rolling checkpoint after each epoch so a crash,\n",+ " # OOM, or Kaggle session timeout doesn't lose all prior training.\n",+ " if rank == 0:\n",+ " ckpt_dir = os.path.join(output_dir, \"checkpoint_latest\")\n",+ " os.makedirs(ckpt_dir, exist_ok=True)\n",+ " ckpt_sd = {k: v.half().contiguous().cpu() for k, v in model.state_dict().items()}\n",+ " save_file(ckpt_sd, os.path.join(ckpt_dir, \"model.safetensors\"))\n",+ " model.encoder.config.save_pretrained(os.path.join(ckpt_dir, \"encoder\"))\n",+ " tok.save_pretrained(os.path.join(ckpt_dir, \"tokenizer\"))\n",+ " with open(os.path.join(ckpt_dir, \"checkpoint_meta.json\"), \"w\") as f:\n",+ " json.dump({\n",+ " \"epoch\": epoch + 1,\n",+ " \"total_epochs\": EPOCHS,\n",+ " \"avg_loss\": epoch_loss / max(1, n_batches)\n",+ " }, f, indent=2)\n",+ " print(f\" Saved rolling checkpoint (epoch {epoch+1}/{EPOCHS}) to {ckpt_dir}\")\n",+ "\n"," dist.barrier()\n"," \n"," # Post-training temperature calibration on rank 0 (micro-batched in chunks of 16 to prevent OOM)\n",@@ -663,10 +682,36 @@" from kaggle_secrets import UserSecretsClient\n"," token = UserSecretsClient().get_secret(\"HF_TOKEN\")\n","except Exception:\n",- " token = \"YOUR_HF_WRITE_TOKEN\"\n",+ " token = None\n",+ "\n",+ "if not token:\n",+ " raise RuntimeError(\n",+ " \"No HF_TOKEN found. Add it via Kaggle's Add-ons -> Secrets -> HF_TOKEN \"\n",+ " \"before running this cell, or set token = \\\"<your-write-token>\\\" manually above.\"\n",+ " )\n","\n","NEW_REPO = \"convaiinnovations/laya-typed-decisions\"\n",- "print(f\"Publishing 0.766 accuracy fine-tuned model to {NEW_REPO}...\")\n",+ "print(f\"Publishing {laya_acc:.3f} accuracy fine-tuned model to {NEW_REPO}...\")\n",+ "\n",+ "# Reference baselines pulled from the comparison table above, so the model\n",+ "# card's narrative always matches the numbers actually computed this run.\n",+ "JEV_ACC = next(r[\"Accuracy\"] for r in comparison_data if r[\"Model\"] == \"TypeSafe Jev 1.13.0\")\n",+ "CEILING_ACC = next(r[\"Accuracy\"] for r in comparison_data if r[\"Model\"] == \"Teacher Self-Agreement\")\n",+ "\n",+ "def describe_vs_baseline(acc, jev_acc=JEV_ACC, ceiling_acc=CEILING_ACC):\n",+ " \"\"\"Builds an accuracy comparison sentence that reflects this run's actual result,\n",+ " instead of assuming the fine-tuned model always wins.\"\"\"\n",+ " if acc > ceiling_acc:\n",+ " return (f\"outperforming **TypeSafe Jev 1.13.0 ({jev_acc:.3f})** and surpassing the \"\n",+ " f\"benchmark's **Teacher Self-Agreement ceiling ({ceiling_acc:.3f})**\")\n",+ " elif acc > jev_acc:\n",+ " return (f\"outperforming **TypeSafe Jev 1.13.0 ({jev_acc:.3f})**, though still below the \"\n",+ " f\"benchmark's Teacher Self-Agreement ceiling ({ceiling_acc:.3f})\")\n",+ " else:\n",+ " return (f\"trailing **TypeSafe Jev 1.13.0 ({jev_acc:.3f})** and the benchmark's \"\n",+ " f\"Teacher Self-Agreement ceiling ({ceiling_acc:.3f})\")\n",+ "\n",+ "comparison_sentence = describe_vs_baseline(laya_acc)\n","\n","# 1. Write comprehensive model card README.md into OUTPUT_DIR before upload\n","readme_content = f\"\"\"---\n",@@ -703,7 +748,7 @@"\n","This is **Laya** fine-tuned on the 1,200 training cases (6,000 decisions) of the independent [LocalLLaMA/typed-decisions](https://huggingface.co/datasets/LocalLLaMA/typed-decisions) benchmark.\n","\n",- "On the official 400-case test set (2,000 decisions across Agent Trace Observability, Customer Service, Invoice Processing, and Security Incidents), it achieves **{laya_acc:.3f} Accuracy**, outperforming **TypeSafe Jev 1.13.0 (0.727)** and surpassing the benchmark's **Teacher Self-Agreement ceiling (0.735)**.\n",+ "On the official 400-case test set (2,000 decisions across Agent Trace Observability, Customer Service, Invoice Processing, and Security Incidents), it achieves **{laya_acc:.3f} Accuracy**, {comparison_sentence}.\n","\n","## Head-to-Head Benchmark Results\n","\n",@@ -738,7 +783,7 @@"with open(os.path.join(OUTPUT_DIR, \"README.md\"), \"w\") as f:\n"," f.write(readme_content)\n","\n",- "print(f\"Generated model card at {os.path.join(OUTPUT_DIR, README.md)}\")\n",+ "print(f\"Generated model card at {os.path.join(OUTPUT_DIR, 'README.md')}\")\n","\n","api = HfApi(token=token)\n","api.create_repo(NEW_REPO, repo_type=\"model\", private=False, exist_ok=True)\n",@@ -748,7 +793,7 @@" folder_path=OUTPUT_DIR,\n"," repo_id=NEW_REPO,\n"," repo_type=\"model\",\n",- " commit_message=f\"Laya fine-tuned on typed-decisions: Accuracy {laya_acc:.3f} (beats Jev 0.727)\"\n",+ " commit_message=f\"Laya fine-tuned on typed-decisions: Accuracy {laya_acc:.3f} vs Jev {JEV_ACC:.3f} / Ceiling {CEILING_ACC:.3f}\"\n",")\n","\n","print(f\"\\nModel successfully published to: https://huggingface.co/{NEW_REPO}\")\n"