Skip to content

A programming framework for agentic AI

notebook

Anonymous-submission-repoff995d08d838
diff --git a/notebook/tune_lexicographic.ipynb b/notebook/tune_lexicographic.ipynb
new file mode 100644
index 000000000..0490f89a0
--- /dev/null
+++ b/notebook/tune_lexicographic.ipynb
@@ -0,0 +1,285 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Tune neural networks with leicographic preference across objectives\n",
+ "This example is to tune neural networks model with two objectives \"error_rate\", \"flops\" on FashionMnist dataset. \n",
+ "\n",
+ "**Requirements.** This notebook requires:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# %pip install torch torchvision flaml[blendsearch,ray] thop"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Data"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import torch\n",
+ "import thop\n",
+ "import torch.nn as nn\n",
+ "from flaml import tune\n",
+ "import torch.nn.functional as F\n",
+ "import torchvision\n",
+ "import numpy as np\n",
+ "import os\n",
+ "\n",
+ "DEVICE = torch.device(\"cpu\")\n",
+ "BATCHSIZE = 128\n",
+ "N_TRAIN_EXAMPLES = BATCHSIZE * 30\n",
+ "N_VALID_EXAMPLES = BATCHSIZE * 10\n",
+ "data_dir = os.path.abspath(\"data\")\n",
+ "\n",
+ "train_dataset = torchvision.datasets.FashionMNIST(\n",
+ " data_dir,\n",
+ " train=True,\n",
+ " download=True,\n",
+ " transform=torchvision.transforms.ToTensor(),\n",
+ ")\n",
+ "\n",
+ "train_loader = torch.utils.data.DataLoader(\n",
+ " torch.utils.data.Subset(train_dataset, list(range(N_TRAIN_EXAMPLES))),\n",
+ " batch_size=BATCHSIZE,\n",
+ " shuffle=True,\n",
+ ")\n",
+ "\n",
+ "val_dataset = torchvision.datasets.FashionMNIST(\n",
+ " data_dir, train=False, transform=torchvision.transforms.ToTensor()\n",
+ ")\n",
+ "\n",
+ "val_loader = torch.utils.data.DataLoader(\n",
+ " torch.utils.data.Subset(val_dataset, list(range(N_VALID_EXAMPLES))),\n",
+ " batch_size=BATCHSIZE,\n",
+ " shuffle=True,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Specify the model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def define_model(configuration):\n",
+ " n_layers = configuration[\"n_layers\"]\n",
+ " layers = []\n",
+ " in_features = 28 * 28\n",
+ " for i in range(n_layers):\n",
+ " out_features = configuration[\"n_units_l{}\".format(i)]\n",
+ " layers.append(nn.Linear(in_features, out_features))\n",
+ " layers.append(nn.ReLU())\n",
+ " p = configuration[\"dropout_{}\".format(i)]\n",
+ " layers.append(nn.Dropout(p))\n",
+ " in_features = out_features\n",
+ " layers.append(nn.Linear(in_features, 10))\n",
+ " layers.append(nn.LogSoftmax(dim=1))\n",
+ " return nn.Sequential(*layers)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Train"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def train_model(model, optimizer, train_loader):\n",
+ " model.train()\n",
+ " for batch_idx, (data, target) in enumerate(train_loader):\n",
+ " data, target = data.view(-1, 28 * 28).to(DEVICE), target.to(DEVICE)\n",
+ " optimizer.zero_grad()\n",
+ " F.nll_loss(model(data), target).backward()\n",
+ " optimizer.step()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Metrics "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def eval_model(model, valid_loader):\n",
+ " model.eval()\n",
+ " correct = 0\n",
+ " with torch.no_grad():\n",
+ " for batch_idx, (data, target) in enumerate(valid_loader):\n",
+ " data, target = data.view(-1, 28 * 28).to(DEVICE), target.to(DEVICE)\n",
+ " pred = model(data).argmax(dim=1, keepdim=True)\n",
+ " correct += pred.eq(target.view_as(pred)).sum().item()\n",
+ "\n",
+ " accuracy = correct / N_VALID_EXAMPLES\n",
+ " flops, params = thop.profile(\n",
+ " model, inputs=(torch.randn(1, 28 * 28).to(DEVICE),), verbose=False\n",
+ " )\n",
+ " return np.log2(flops), 1 - accuracy, params"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Evaluate function"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def evaluate_function(configuration):\n",
+ " model = define_model(configuration).to(DEVICE)\n",
+ " optimizer = torch.optim.Adam(model.parameters(), configuration[\"lr\"])\n",
+ " n_epoch = configuration[\"n_epoch\"]\n",
+ " for epoch in range(n_epoch):\n",
+ " train_model(model, optimizer, train_loader)\n",
+ " flops, error_rate, params = eval_model(model, val_loader)\n",
+ " return {\"error_rate\": error_rate, \"flops\": flops, \"params\": params}"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Lexicographic information across objectives"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "lexico_objectives = {}\n",
+ "lexico_objectives[\"metrics\"] = [\"error_rate\", \"flops\"]\n",
+ "lexico_objectives[\"tolerances\"] = {\"error_rate\": 0.02, \"flops\": 0.0}\n",
+ "lexico_objectives[\"targets\"] = {\"error_rate\": 0.0, \"flops\": 0.0}\n",
+ "lexico_objectives[\"modes\"] = [\"min\", \"min\"]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Search space"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "search_space = {\n",
+ " \"n_layers\": tune.randint(lower=1, upper=3),\n",
+ " \"n_units_l0\": tune.randint(lower=4, upper=128),\n",
+ " \"n_units_l1\": tune.randint(lower=4, upper=128),\n",
+ " \"n_units_l2\": tune.randint(lower=4, upper=128),\n",
+ " \"dropout_0\": tune.uniform(lower=0.2, upper=0.5),\n",
+ " \"dropout_1\": tune.uniform(lower=0.2, upper=0.5),\n",
+ " \"dropout_2\": tune.uniform(lower=0.2, upper=0.5),\n",
+ " \"lr\": tune.loguniform(lower=1e-5, upper=1e-1),\n",
+ " \"n_epoch\": tune.randint(lower=1, upper=20),\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Launch the tuning"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "low_cost_partial_config = {\n",
+ " \"n_layers\": 1,\n",
+ " \"n_units_l0\": 4,\n",
+ " \"n_units_l1\": 4,\n",
+ " \"n_units_l2\": 4,\n",
+ " \"n_epoch\": 1,\n",
+ "}\n",
+ "\n",
+ "analysis = tune.run(\n",
+ " evaluate_function,\n",
+ " num_samples=100000000,\n",
+ " time_budget_s=100,\n",
+ " config=search_space,\n",
+ " use_ray=False,\n",
+ " lexico_objectives=lexico_objectives,\n",
+ " low_cost_partial_config=low_cost_partial_config,\n",
+ ")\n",
+ "result = analysis.best_result\n",
+ "print(result)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3.9.14 64-bit",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.14"
+ },
+ "orig_nbformat": 4,
+ "vscode": {
+ "interpreter": {
+ "hash": "949777d72b0d2535278d3dc13498b2535136f6dfe0678499012e853ee9abcab1"
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}