Skip to content

A programming framework for agentic AI

nni example + bugfix in blendsearch (#44)

Iman Hosseinibf95d7c45535
diff --git a/flaml/searcher/blendsearch.py b/flaml/searcher/blendsearch.py
index 61a760d8e..788f98030 100644
--- a/flaml/searcher/blendsearch.py
+++ b/flaml/searcher/blendsearch.py
@@ -455,7 +455,7 @@ try:
value: final metrics of the trial, including default metric
'''
result = {}
- for key, value in parameters:
+ for key, value in parameters.items():
result['config/'+key] = value
reward = extract_scalar_reward(value)
result[self._metric] = reward
diff --git a/flaml/tune/README.md b/flaml/tune/README.md
index 8114c8b6a..6b7e0b93a 100644
--- a/flaml/tune/README.md
+++ b/flaml/tune/README.md
@@ -159,6 +159,12 @@ Recommended scenario: cost-related hyperparameters exist, a low-cost
initial point is known, and the search space is complex such that local search
is prone to be stuck at local optima.
+An example of using BlendSearch with NNI can be seen in [test](https://github.com/microsoft/FLAML/tree/main/test/nni), CFO can be used with NNI as well in a similar manner. To run the example, first make sure you have [NNI](https://nni.readthedocs.io/en/stable/) installed, then run:
+
+```shell
+$nnictl create --config ./config.yml
+```
+
For more technical details, please check our papers.
* [Frugal Optimization for Cost-related Hyperparameters](https://arxiv.org/abs/2005.01571). Qingyun Wu, Chi Wang, Silu Huang. AAAI 2021.
diff --git a/test/nni/config.yml b/test/nni/config.yml
new file mode 100644
index 000000000..1544fff08
--- /dev/null
+++ b/test/nni/config.yml
@@ -0,0 +1,19 @@
+# usage: nnictl create --config ./config.yml
+authorName: default
+experimentName: example_mnist
+trialConcurrency: 1
+maxExecDuration: 1h
+maxTrialNum: 10
+trainingServicePlatform: local
+# The path to Search Space
+searchSpacePath: search_space.json
+useAnnotation: false
+tuner:
+ codeDir: ./
+ classFileName: flaml_nni_wrap.py
+ className: BlendSearchTuner
+# The path and the running command of trial
+trial:
+ command: python3 mnist.py
+ codeDir: .
+ gpuNum: 0
diff --git a/test/nni/flaml_nni_wrap.py b/test/nni/flaml_nni_wrap.py
new file mode 100644
index 000000000..60265f87d
--- /dev/null
+++ b/test/nni/flaml_nni_wrap.py
@@ -0,0 +1,5 @@
+from flaml.searcher.blendsearch import BlendSearchTuner as BST, BlendSearch
+class BlendSearchTuner(BST):
+ # for best performance pass low cost initial parameters here
+ def __init__(self, points_to_evaluate=[{"hidden_size":128}]):
+ super.__init__(self,points_to_evaluate=points_to_evaluate)
diff --git a/test/nni/mnist.py b/test/nni/mnist.py
new file mode 100644
index 000000000..faeb950f4
--- /dev/null
+++ b/test/nni/mnist.py
@@ -0,0 +1,169 @@
+# This file is copied from NNI project
+# https://github.com/microsoft/nni/blob/master/examples/trials/mnist-tfv1/mnist.py
+
+"""
+A deep MNIST classifier using convolutional layers.
+
+This file is a modification of the official pytorch mnist example:
+https://github.com/pytorch/examples/blob/master/mnist/main.py
+"""
+
+import os
+import argparse
+import logging
+import nni
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+from nni.utils import merge_parameter
+from torchvision import datasets, transforms
+
+logger = logging.getLogger('mnist_AutoML')
+
+
+class Net(nn.Module):
+ def __init__(self, hidden_size):
+ super(Net, self).__init__()
+ self.conv1 = nn.Conv2d(1, 20, 5, 1)
+ self.conv2 = nn.Conv2d(20, 50, 5, 1)
+ self.fc1 = nn.Linear(4*4*50, hidden_size)
+ self.fc2 = nn.Linear(hidden_size, 10)
+
+ def forward(self, x):
+ x = F.relu(self.conv1(x))
+ x = F.max_pool2d(x, 2, 2)
+ x = F.relu(self.conv2(x))
+ x = F.max_pool2d(x, 2, 2)
+ x = x.view(-1, 4*4*50)
+ x = F.relu(self.fc1(x))
+ x = self.fc2(x)
+ return F.log_softmax(x, dim=1)
+
+
+def train(args, model, device, train_loader, optimizer, epoch):
+ model.train()
+ for batch_idx, (data, target) in enumerate(train_loader):
+ if (args['batch_num'] is not None) and batch_idx >= args['batch_num']:
+ break
+ data, target = data.to(device), target.to(device)
+ optimizer.zero_grad()
+ output = model(data)
+ loss = F.nll_loss(output, target)
+ loss.backward()
+ optimizer.step()
+ if batch_idx % args['log_interval'] == 0:
+ logger.info('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
+ epoch, batch_idx * len(data), len(train_loader.dataset),
+ 100. * batch_idx / len(train_loader), loss.item()))
+
+
+def test(args, model, device, test_loader):
+ model.eval()
+ test_loss = 0
+ correct = 0
+ with torch.no_grad():
+ for data, target in test_loader:
+ data, target = data.to(device), target.to(device)
+ output = model(data)
+ # sum up batch loss
+ test_loss += F.nll_loss(output, target, reduction='sum').item()
+ # get the index of the max log-probability
+ pred = output.argmax(dim=1, keepdim=True)
+ correct += pred.eq(target.view_as(pred)).sum().item()
+
+ test_loss /= len(test_loader.dataset)
+
+ accuracy = 100. * correct / len(test_loader.dataset)
+
+ logger.info('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
+ test_loss, correct, len(test_loader.dataset), accuracy))
+
+ return accuracy
+
+
+def main(args):
+ use_cuda = not args['no_cuda'] and torch.cuda.is_available()
+
+ torch.manual_seed(args['seed'])
+
+ device = torch.device("cuda" if use_cuda else "cpu")
+
+ kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
+
+ data_dir = args['data_dir']
+
+ train_loader = torch.utils.data.DataLoader(
+ datasets.MNIST(data_dir, train=True, download=True,
+ transform=transforms.Compose([
+ transforms.ToTensor(),
+ transforms.Normalize((0.1307,), (0.3081,))
+ ])),
+ batch_size=args['batch_size'], shuffle=True, **kwargs)
+ test_loader = torch.utils.data.DataLoader(
+ datasets.MNIST(data_dir, train=False, transform=transforms.Compose([
+ transforms.ToTensor(),
+ transforms.Normalize((0.1307,), (0.3081,))
+ ])),
+ batch_size=1000, shuffle=True, **kwargs)
+
+ hidden_size = args['hidden_size']
+
+ model = Net(hidden_size=hidden_size).to(device)
+ optimizer = optim.SGD(model.parameters(), lr=args['lr'],
+ momentum=args['momentum'])
+
+ for epoch in range(1, args['epochs'] + 1):
+ train(args, model, device, train_loader, optimizer, epoch)
+ test_acc = test(args, model, device, test_loader)
+
+ # report intermediate result
+ nni.report_intermediate_result(test_acc)
+ logger.debug('test accuracy %g', test_acc)
+ logger.debug('Pipe send intermediate result done.')
+
+ # report final result
+ nni.report_final_result(test_acc)
+ logger.debug('Final result is %g', test_acc)
+ logger.debug('Send final result done.')
+
+
+def get_params():
+ # Training settings
+ parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
+ parser.add_argument("--data_dir", type=str,
+ default='./data', help="data directory")
+ parser.add_argument('--batch_size', type=int, default=64, metavar='N',
+ help='input batch size for training (default: 64)')
+ parser.add_argument("--batch_num", type=int, default=None)
+ parser.add_argument("--hidden_size", type=int, default=512, metavar='N',
+ help='hidden layer size (default: 512)')
+ parser.add_argument('--lr', type=float, default=0.01, metavar='LR',
+ help='learning rate (default: 0.01)')
+ parser.add_argument('--momentum', type=float, default=0.5, metavar='M',
+ help='SGD momentum (default: 0.5)')
+ parser.add_argument('--epochs', type=int, default=10, metavar='N',
+ help='number of epochs to train (default: 10)')
+ parser.add_argument('--seed', type=int, default=1, metavar='S',
+ help='random seed (default: 1)')
+ parser.add_argument('--no_cuda', action='store_true', default=False,
+ help='disables CUDA training')
+ parser.add_argument('--log_interval', type=int, default=1000, metavar='N',
+ help='how many batches to wait before logging training status')
+
+
+ args, _ = parser.parse_known_args()
+ return args
+
+
+if __name__ == '__main__':
+ try:
+ # get parameters form tuner
+ tuner_params = nni.get_next_parameter()
+ logger.debug(tuner_params)
+ params = vars(merge_parameter(get_params(), tuner_params))
+ print(params)
+ main(params)
+ except Exception as exception:
+ logger.exception(exception)
+ raise
diff --git a/test/nni/search_space.json b/test/nni/search_space.json
new file mode 100644
index 000000000..c26cdce36
--- /dev/null
+++ b/test/nni/search_space.json
@@ -0,0 +1,6 @@
+{
+ "batch_size": {"_type":"choice", "_value": [16, 32, 64, 128]},
+ "hidden_size":{"_type":"choice","_value":[128, 256, 512, 1024]},
+ "lr":{"_type":"choice","_value":[0.0001, 0.001, 0.01, 0.1]},
+ "momentum":{"_type":"uniform","_value":[0, 1]}
+}