A programming framework for agentic AI
nni example + bugfix in blendsearch (#44)
Iman Hosseinibf95d7c45535
diff --git a/flaml/searcher/blendsearch.py b/flaml/searcher/blendsearch.pyindex 61a760d8e..788f98030 100644--- a/flaml/searcher/blendsearch.py+++ b/flaml/searcher/blendsearch.py@@ -455,7 +455,7 @@ try:value: final metrics of the trial, including default metric'''result = {}- for key, value in parameters:+ for key, value in parameters.items():result['config/'+key] = valuereward = extract_scalar_reward(value)result[self._metric] = rewarddiff --git a/flaml/tune/README.md b/flaml/tune/README.mdindex 8114c8b6a..6b7e0b93a 100644--- a/flaml/tune/README.md+++ b/flaml/tune/README.md@@ -159,6 +159,12 @@ Recommended scenario: cost-related hyperparameters exist, a low-costinitial point is known, and the search space is complex such that local searchis prone to be stuck at local optima.+An example of using BlendSearch with NNI can be seen in [test](https://github.com/microsoft/FLAML/tree/main/test/nni), CFO can be used with NNI as well in a similar manner. To run the example, first make sure you have [NNI](https://nni.readthedocs.io/en/stable/) installed, then run:++```shell+$nnictl create --config ./config.yml+```+For more technical details, please check our papers.* [Frugal Optimization for Cost-related Hyperparameters](https://arxiv.org/abs/2005.01571). Qingyun Wu, Chi Wang, Silu Huang. AAAI 2021.diff --git a/test/nni/config.yml b/test/nni/config.ymlnew file mode 100644index 000000000..1544fff08--- /dev/null+++ b/test/nni/config.yml@@ -0,0 +1,19 @@+# usage: nnictl create --config ./config.yml+authorName: default+experimentName: example_mnist+trialConcurrency: 1+maxExecDuration: 1h+maxTrialNum: 10+trainingServicePlatform: local+# The path to Search Space+searchSpacePath: search_space.json+useAnnotation: false+tuner:+ codeDir: ./+ classFileName: flaml_nni_wrap.py+ className: BlendSearchTuner+# The path and the running command of trial+trial:+ command: python3 mnist.py+ codeDir: .+ gpuNum: 0diff --git a/test/nni/flaml_nni_wrap.py b/test/nni/flaml_nni_wrap.pynew file mode 100644index 000000000..60265f87d--- /dev/null+++ b/test/nni/flaml_nni_wrap.py@@ -0,0 +1,5 @@+from flaml.searcher.blendsearch import BlendSearchTuner as BST, BlendSearch+class BlendSearchTuner(BST):+ # for best performance pass low cost initial parameters here+ def __init__(self, points_to_evaluate=[{"hidden_size":128}]):+ super.__init__(self,points_to_evaluate=points_to_evaluate)diff --git a/test/nni/mnist.py b/test/nni/mnist.pynew file mode 100644index 000000000..faeb950f4--- /dev/null+++ b/test/nni/mnist.py@@ -0,0 +1,169 @@+# This file is copied from NNI project+# https://github.com/microsoft/nni/blob/master/examples/trials/mnist-tfv1/mnist.py++"""+A deep MNIST classifier using convolutional layers.++This file is a modification of the official pytorch mnist example:+https://github.com/pytorch/examples/blob/master/mnist/main.py+"""++import os+import argparse+import logging+import nni+import torch+import torch.nn as nn+import torch.nn.functional as F+import torch.optim as optim+from nni.utils import merge_parameter+from torchvision import datasets, transforms++logger = logging.getLogger('mnist_AutoML')+++class Net(nn.Module):+ def __init__(self, hidden_size):+ super(Net, self).__init__()+ self.conv1 = nn.Conv2d(1, 20, 5, 1)+ self.conv2 = nn.Conv2d(20, 50, 5, 1)+ self.fc1 = nn.Linear(4*4*50, hidden_size)+ self.fc2 = nn.Linear(hidden_size, 10)++ def forward(self, x):+ x = F.relu(self.conv1(x))+ x = F.max_pool2d(x, 2, 2)+ x = F.relu(self.conv2(x))+ x = F.max_pool2d(x, 2, 2)+ x = x.view(-1, 4*4*50)+ x = F.relu(self.fc1(x))+ x = self.fc2(x)+ return F.log_softmax(x, dim=1)+++def train(args, model, device, train_loader, optimizer, epoch):+ model.train()+ for batch_idx, (data, target) in enumerate(train_loader):+ if (args['batch_num'] is not None) and batch_idx >= args['batch_num']:+ break+ data, target = data.to(device), target.to(device)+ optimizer.zero_grad()+ output = model(data)+ loss = F.nll_loss(output, target)+ loss.backward()+ optimizer.step()+ if batch_idx % args['log_interval'] == 0:+ logger.info('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(+ epoch, batch_idx * len(data), len(train_loader.dataset),+ 100. * batch_idx / len(train_loader), loss.item()))+++def test(args, model, device, test_loader):+ model.eval()+ test_loss = 0+ correct = 0+ with torch.no_grad():+ for data, target in test_loader:+ data, target = data.to(device), target.to(device)+ output = model(data)+ # sum up batch loss+ test_loss += F.nll_loss(output, target, reduction='sum').item()+ # get the index of the max log-probability+ pred = output.argmax(dim=1, keepdim=True)+ correct += pred.eq(target.view_as(pred)).sum().item()++ test_loss /= len(test_loader.dataset)++ accuracy = 100. * correct / len(test_loader.dataset)++ logger.info('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(+ test_loss, correct, len(test_loader.dataset), accuracy))++ return accuracy+++def main(args):+ use_cuda = not args['no_cuda'] and torch.cuda.is_available()++ torch.manual_seed(args['seed'])++ device = torch.device("cuda" if use_cuda else "cpu")++ kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}++ data_dir = args['data_dir']++ train_loader = torch.utils.data.DataLoader(+ datasets.MNIST(data_dir, train=True, download=True,+ transform=transforms.Compose([+ transforms.ToTensor(),+ transforms.Normalize((0.1307,), (0.3081,))+ ])),+ batch_size=args['batch_size'], shuffle=True, **kwargs)+ test_loader = torch.utils.data.DataLoader(+ datasets.MNIST(data_dir, train=False, transform=transforms.Compose([+ transforms.ToTensor(),+ transforms.Normalize((0.1307,), (0.3081,))+ ])),+ batch_size=1000, shuffle=True, **kwargs)++ hidden_size = args['hidden_size']++ model = Net(hidden_size=hidden_size).to(device)+ optimizer = optim.SGD(model.parameters(), lr=args['lr'],+ momentum=args['momentum'])++ for epoch in range(1, args['epochs'] + 1):+ train(args, model, device, train_loader, optimizer, epoch)+ test_acc = test(args, model, device, test_loader)++ # report intermediate result+ nni.report_intermediate_result(test_acc)+ logger.debug('test accuracy %g', test_acc)+ logger.debug('Pipe send intermediate result done.')++ # report final result+ nni.report_final_result(test_acc)+ logger.debug('Final result is %g', test_acc)+ logger.debug('Send final result done.')+++def get_params():+ # Training settings+ parser = argparse.ArgumentParser(description='PyTorch MNIST Example')+ parser.add_argument("--data_dir", type=str,+ default='./data', help="data directory")+ parser.add_argument('--batch_size', type=int, default=64, metavar='N',+ help='input batch size for training (default: 64)')+ parser.add_argument("--batch_num", type=int, default=None)+ parser.add_argument("--hidden_size", type=int, default=512, metavar='N',+ help='hidden layer size (default: 512)')+ parser.add_argument('--lr', type=float, default=0.01, metavar='LR',+ help='learning rate (default: 0.01)')+ parser.add_argument('--momentum', type=float, default=0.5, metavar='M',+ help='SGD momentum (default: 0.5)')+ parser.add_argument('--epochs', type=int, default=10, metavar='N',+ help='number of epochs to train (default: 10)')+ parser.add_argument('--seed', type=int, default=1, metavar='S',+ help='random seed (default: 1)')+ parser.add_argument('--no_cuda', action='store_true', default=False,+ help='disables CUDA training')+ parser.add_argument('--log_interval', type=int, default=1000, metavar='N',+ help='how many batches to wait before logging training status')+++ args, _ = parser.parse_known_args()+ return args+++if __name__ == '__main__':+ try:+ # get parameters form tuner+ tuner_params = nni.get_next_parameter()+ logger.debug(tuner_params)+ params = vars(merge_parameter(get_params(), tuner_params))+ print(params)+ main(params)+ except Exception as exception:+ logger.exception(exception)+ raisediff --git a/test/nni/search_space.json b/test/nni/search_space.jsonnew file mode 100644index 000000000..c26cdce36--- /dev/null+++ b/test/nni/search_space.json@@ -0,0 +1,6 @@+{+ "batch_size": {"_type":"choice", "_value": [16, 32, 64, 128]},+ "hidden_size":{"_type":"choice","_value":[128, 256, 512, 1024]},+ "lr":{"_type":"choice","_value":[0.0001, 0.001, 0.01, 0.1]},+ "momentum":{"_type":"uniform","_value":[0, 1]}+}