from hyperopt import Trials, STATUS_OK, tpe
from hyperas import optim
from hyperas.distributions import choice, uniform
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
from keras.optimizers import RMSprop
from keras.datasets import mnist
from keras.utils import np_utils
/Users/jannes/anaconda/lib/python3.5/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`. from ._conv import register_converters as _register_converters Using TensorFlow backend. Couldn't import dot_parser, loading of dot files will not be possible.
def data():
'''
Data providing function:
This function is separated from model() so that hyperopt
won't reload data for each evaluation run.
'''
(X_train, y_train), (X_test, y_test) = mnist.load_data()
X_train = X_train.reshape(60000, 784)
X_test = X_test.reshape(10000, 784)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255
nb_classes = 10
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
return X_train, Y_train, X_test, Y_test
def model(X_train, Y_train, X_test, Y_test):
'''
Model providing function:
Create Keras model with double curly brackets dropped-in as needed.
Return value has to be a valid python dictionary with two customary keys:
- loss: Specify a numeric evaluation metric to be minimized
- status: Just use STATUS_OK and see hyperopt documentation if not feasible
The last one is optional, though recommended, namely:
- model: specify the model just created so that we can later use it again.
'''
model = Sequential()
model.add(Dense(512, input_shape=(784,)))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 0.5)}}))
model.add(Dense({{choice([256, 512, 1024])}}))
model.add(Activation({{choice(['relu','tanh'])}}))
model.add(Dropout({{uniform(0, 0.5)}}))
model.add(Dense(10))
model.add(Activation('softmax'))
rms = RMSprop()
model.compile(loss='categorical_crossentropy',
optimizer=rms,
metrics=['accuracy'])
model.fit(X_train, y_train,
batch_size={{choice([64, 128])}},
epochs=1,
verbose=2,
validation_data=(X_test, Y_test))
score, acc = model.evaluate(X_test, Y_test, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
best_run, best_model = optim.minimize(model=model,
data=data,
algo=tpe.suggest,
max_evals=5,
trials=Trials(),
notebook_name='simple_notebook')
>>> Imports: #coding=utf-8 try: from hyperopt import Trials, STATUS_OK, tpe except: pass try: from hyperas import optim except: pass try: from hyperas.distributions import choice, uniform except: pass try: from keras.models import Sequential except: pass try: from keras.layers import Dense, Dropout, Activation except: pass try: from keras.optimizers import RMSprop except: pass try: from keras.datasets import mnist except: pass try: from keras.utils import np_utils except: pass >>> Hyperas search space: def get_space(): return { 'Dropout': hp.uniform('Dropout', 0, 0.5), 'Dense': hp.choice('Dense', [256, 512, 1024]), 'Activation': hp.choice('Activation', ['relu','tanh']), 'Dropout_1': hp.uniform('Dropout_1', 0, 0.5), 'batch_size': hp.choice('batch_size', [64, 128]), } >>> Data 1: 2: ''' 3: Data providing function: 4: 5: This function is separated from model() so that hyperopt 6: won't reload data for each evaluation run. 7: ''' 8: (X_train, y_train), (X_test, y_test) = mnist.load_data() 9: X_train = X_train.reshape(60000, 784) 10: X_test = X_test.reshape(10000, 784) 11: X_train = X_train.astype('float32') 12: X_test = X_test.astype('float32') 13: X_train /= 255 14: X_test /= 255 15: nb_classes = 10 16: Y_train = np_utils.to_categorical(y_train, nb_classes) 17: Y_test = np_utils.to_categorical(y_test, nb_classes) 18: 19: 20: >>> Resulting replaced keras model: 1: def keras_fmin_fnct(space): 2: 3: ''' 4: Model providing function: 5: 6: Create Keras model with double curly brackets dropped-in as needed. 7: Return value has to be a valid python dictionary with two customary keys: 8: - loss: Specify a numeric evaluation metric to be minimized 9: - status: Just use STATUS_OK and see hyperopt documentation if not feasible 10: The last one is optional, though recommended, namely: 11: - model: specify the model just created so that we can later use it again. 12: ''' 13: model = Sequential() 14: model.add(Dense(512, input_shape=(784,))) 15: 16: model.add(Activation('relu')) 17: 18: model.add(Dropout(space['Dropout'])) 19: 20: model.add(Dense(space['Dense'])) 21: 22: model.add(Activation(space['Activation'])) 23: 24: model.add(Dropout(space['Dropout_1'])) 25: 26: model.add(Dense(10)) 27: model.add(Activation('softmax')) 28: 29: rms = RMSprop() 30: model.compile(loss='categorical_crossentropy', 31: optimizer=rms, 32: metrics=['accuracy']) 33: 34: model.fit(X_train, Y_train, 35: batch_size=space['batch_size'], 36: epochs=1, 37: verbose=2, 38: validation_data=(X_test, Y_test)) 39: score, acc = model.evaluate(X_test, Y_test, verbose=0) 40: print('Test accuracy:', acc) 41: return {'loss': -acc, 'status': STATUS_OK, 'model': model} 42: Train on 60000 samples, validate on 10000 samples Epoch 1/1 - 10s - loss: 0.2395 - acc: 0.9268 - val_loss: 0.1246 - val_acc: 0.9648 Test accuracy: 0.9648 Train on 60000 samples, validate on 10000 samples Epoch 1/1 - 10s - loss: 0.2225 - acc: 0.9323 - val_loss: 0.1066 - val_acc: 0.9670 Test accuracy: 0.967 Train on 60000 samples, validate on 10000 samples Epoch 1/1 - 7s - loss: 0.2672 - acc: 0.9174 - val_loss: 0.1273 - val_acc: 0.9589 Test accuracy: 0.9589 Train on 60000 samples, validate on 10000 samples Epoch 1/1 - 10s - loss: 0.2476 - acc: 0.9235 - val_loss: 0.1337 - val_acc: 0.9615 Test accuracy: 0.9615 Train on 60000 samples, validate on 10000 samples Epoch 1/1 - 10s - loss: 0.2328 - acc: 0.9292 - val_loss: 0.0982 - val_acc: 0.9707 Test accuracy: 0.9707
print("Evalutation of best performing model:")
print(best_model.evaluate(X_test, Y_test))
Evalutation of best performing model: 10000/10000 [==============================] - 1s 95us/step [0.12623950149156155, 0.9624]
best_run
{'Activation': 1, 'Dense': 2, 'Dropout': 0.11654484411565585, 'Dropout_1': 0.4566135465496464, 'batch_size': 1}
best_model.summary()
_________________________________________________________________ Layer (type) Output Shape Param # ================================================================= dense_79 (Dense) (None, 512) 401920 _________________________________________________________________ activation_79 (Activation) (None, 512) 0 _________________________________________________________________ dropout_53 (Dropout) (None, 512) 0 _________________________________________________________________ dense_80 (Dense) (None, 512) 262656 _________________________________________________________________ activation_80 (Activation) (None, 512) 0 _________________________________________________________________ dropout_54 (Dropout) (None, 512) 0 _________________________________________________________________ dense_81 (Dense) (None, 10) 5130 _________________________________________________________________ activation_81 (Activation) (None, 10) 0 ================================================================= Total params: 669,706 Trainable params: 669,706 Non-trainable params: 0 _________________________________________________________________
??choice