Classification of Iris

Package imports

In [1]:
# For building neural networks.
import keras as kr

# For interacting with data sets.
import pandas as pd

# For encoding categorical variables.
import sklearn.preprocessing as pre

# For splitting into training and test sets.
import sklearn.model_selection as mod
Using TensorFlow backend.

Load data

In [2]:
# Load the iris data set from a URL.
df = pd.read_csv("https://raw.githubusercontent.com/ianmcloughlin/datasets/master/iris.csv")
In [3]:
df
Out[3]:
sepal_length sepal_width petal_length petal_width class
0 5.1 3.5 1.4 0.2 setosa
1 4.9 3.0 1.4 0.2 setosa
2 4.7 3.2 1.3 0.2 setosa
3 4.6 3.1 1.5 0.2 setosa
4 5.0 3.6 1.4 0.2 setosa
... ... ... ... ... ...
145 6.7 3.0 5.2 2.3 virginica
146 6.3 2.5 5.0 1.9 virginica
147 6.5 3.0 5.2 2.0 virginica
148 6.2 3.4 5.4 2.3 virginica
149 5.9 3.0 5.1 1.8 virginica

150 rows × 5 columns

Inputs

In [4]:
# Separate the inputs from the rest of the variables.
inputs = df[['petal_length', 'petal_width', 'sepal_length', 'sepal_width']]
In [5]:
inputs
Out[5]:
petal_length petal_width sepal_length sepal_width
0 1.4 0.2 5.1 3.5
1 1.4 0.2 4.9 3.0
2 1.3 0.2 4.7 3.2
3 1.5 0.2 4.6 3.1
4 1.4 0.2 5.0 3.6
... ... ... ... ...
145 5.2 2.3 6.7 3.0
146 5.0 1.9 6.3 2.5
147 5.2 2.0 6.5 3.0
148 5.4 2.3 6.2 3.4
149 5.1 1.8 5.9 3.0

150 rows × 4 columns

Encoded outputs

$$ \begin{align*} setosa & \rightarrow [1,0,0] \\ versicolor & \rightarrow [0,1,0] \\ virginica & \rightarrow [0,0,1] \end{align*} $$
In [6]:
# Encode the classes as above.
encoder = pre.LabelBinarizer()
encoder.fit(df['class'])
outputs = encoder.transform(df['class'])

outputs
Out[6]:
array([[1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [1, 0, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 1, 0],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1],
       [0, 0, 1]])

Idea

The neural network will turn four floating point inputs into three "floating point" outputs.

$$ [5.1, 3.5, 1.4, 0.2] \rightarrow [0.8, 0.19, 0.01] $$$$ [5.1, 3.5, 1.4, 0.2] \rightarrow [1, 0, 0] $$

Build model

In [7]:
# Start a neural network, building it by layers.
model = kr.models.Sequential()

# Add a hidden layer with x neurons and an input layer with 4.
model.add(kr.layers.Dense(units=30, activation='relu', input_dim=4))
# Add a three neuron output layer.
model.add(kr.layers.Dense(units=3, activation='softmax'))

# Build the graph.
model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy'])
WARNING:tensorflow:From C:\Users\mclou\Anaconda3\lib\site-packages\keras\backend\tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.

WARNING:tensorflow:From C:\Users\mclou\Anaconda3\lib\site-packages\keras\backend\tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.

WARNING:tensorflow:From C:\Users\mclou\Anaconda3\lib\site-packages\keras\backend\tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.

WARNING:tensorflow:From C:\Users\mclou\Anaconda3\lib\site-packages\keras\optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.

WARNING:tensorflow:From C:\Users\mclou\Anaconda3\lib\site-packages\keras\backend\tensorflow_backend.py:3295: The name tf.log is deprecated. Please use tf.math.log instead.

Split

In [8]:
# Split the inputs and outputs into training and test sets.
inputs_train, inputs_test, outputs_train, outputs_test = mod.train_test_split(inputs, outputs, test_size=0.2)
In [9]:
inputs_test.iloc[0]
Out[9]:
petal_length    4.8
petal_width     1.4
sepal_length    6.8
sepal_width     2.8
Name: 76, dtype: float64
In [10]:
model.predict(inputs_test.as_matrix()[0:1])
WARNING:tensorflow:From C:\Users\mclou\Anaconda3\lib\site-packages\keras\backend\tensorflow_backend.py:2741: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.

WARNING:tensorflow:From C:\Users\mclou\Anaconda3\lib\site-packages\keras\backend\tensorflow_backend.py:174: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.

WARNING:tensorflow:From C:\Users\mclou\Anaconda3\lib\site-packages\keras\backend\tensorflow_backend.py:181: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.

WARNING:tensorflow:From C:\Users\mclou\Anaconda3\lib\site-packages\keras\backend\tensorflow_backend.py:190: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.

WARNING:tensorflow:From C:\Users\mclou\Anaconda3\lib\site-packages\keras\backend\tensorflow_backend.py:199: The name tf.is_variable_initialized is deprecated. Please use tf.compat.v1.is_variable_initialized instead.

C:\Users\mclou\Anaconda3\lib\site-packages\ipykernel_launcher.py:1: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.
  """Entry point for launching an IPython kernel.
WARNING:tensorflow:From C:\Users\mclou\Anaconda3\lib\site-packages\keras\backend\tensorflow_backend.py:206: The name tf.variables_initializer is deprecated. Please use tf.compat.v1.variables_initializer instead.

Out[10]:
array([[0.02864362, 0.96718645, 0.00416994]], dtype=float32)

Train

In [11]:
# Train the neural network.
model.fit(inputs_train, outputs_train, epochs=15, batch_size=10)
WARNING:tensorflow:From C:\Users\mclou\Anaconda3\lib\site-packages\tensorflow_core\python\ops\math_grad.py:1424: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.where in 2.0, which has the same broadcast rule as np.where
WARNING:tensorflow:From C:\Users\mclou\Anaconda3\lib\site-packages\keras\backend\tensorflow_backend.py:986: The name tf.assign_add is deprecated. Please use tf.compat.v1.assign_add instead.

WARNING:tensorflow:From C:\Users\mclou\Anaconda3\lib\site-packages\keras\backend\tensorflow_backend.py:973: The name tf.assign is deprecated. Please use tf.compat.v1.assign instead.

Epoch 1/15
120/120 [==============================] - 1s 10ms/step - loss: 1.6282 - acc: 0.3417
Epoch 2/15
120/120 [==============================] - 0s 671us/step - loss: 1.0749 - acc: 0.4250
Epoch 3/15
120/120 [==============================] - 0s 767us/step - loss: 0.8976 - acc: 0.5167
Epoch 4/15
120/120 [==============================] - 0s 650us/step - loss: 0.8089 - acc: 0.7500
Epoch 5/15
120/120 [==============================] - 0s 850us/step - loss: 0.7383 - acc: 0.7750
Epoch 6/15
120/120 [==============================] - 0s 813us/step - loss: 0.6916 - acc: 0.8500
Epoch 7/15
120/120 [==============================] - 0s 700us/step - loss: 0.6547 - acc: 0.8583
Epoch 8/15
120/120 [==============================] - 0s 333us/step - loss: 0.6256 - acc: 0.8167
Epoch 9/15
120/120 [==============================] - 0s 342us/step - loss: 0.5870 - acc: 0.7667
Epoch 10/15
120/120 [==============================] - 0s 550us/step - loss: 0.5789 - acc: 0.8667
Epoch 11/15
120/120 [==============================] - 0s 817us/step - loss: 0.5514 - acc: 0.8500
Epoch 12/15
120/120 [==============================] - 0s 900us/step - loss: 0.5314 - acc: 0.8917
Epoch 13/15
120/120 [==============================] - 0s 746us/step - loss: 0.5162 - acc: 0.8417
Epoch 14/15
120/120 [==============================] - 0s 392us/step - loss: 0.5029 - acc: 0.8583
Epoch 15/15
120/120 [==============================] - 0s 292us/step - loss: 0.4830 - acc: 0.8417
Out[11]:
<keras.callbacks.History at 0x1c4c4343248>

Predict

In [12]:
model.predict(inputs_test.as_matrix()[0:1])
C:\Users\mclou\Anaconda3\lib\site-packages\ipykernel_launcher.py:1: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.
  """Entry point for launching an IPython kernel.
Out[12]:
array([[0.07707678, 0.5562058 , 0.36671743]], dtype=float32)
In [13]:
# Have the network predict the classes of the test inputs.
predictions = model.predict(inputs_test)
predictions_labels = encoder.inverse_transform(predictions)
predictions_labels
Out[13]:
array(['versicolor', 'virginica', 'setosa', 'setosa', 'setosa',
       'virginica', 'setosa', 'virginica', 'versicolor', 'versicolor',
       'virginica', 'versicolor', 'virginica', 'setosa', 'setosa',
       'versicolor', 'setosa', 'setosa', 'versicolor', 'virginica',
       'setosa', 'versicolor', 'versicolor', 'setosa', 'setosa',
       'virginica', 'virginica', 'versicolor', 'virginica', 'virginica'],
      dtype='<U10')

Evaluate

In [14]:
# Compare the predictions to the actual classes.
predictions_labels == encoder.inverse_transform(outputs_test)
Out[14]:
array([ True,  True,  True,  True,  True, False,  True,  True,  True,
        True,  True,  True,  True,  True,  True,  True,  True,  True,
        True,  True,  True,  True,  True,  True,  True,  True,  True,
        True,  True,  True])
In [15]:
(predictions_labels == encoder.inverse_transform(outputs_test)).sum()
Out[15]:
29

Whitening

In [16]:
import sklearn.decomposition as dec
In [17]:
pca = dec.PCA(n_components=4, whiten=True)
pca.fit(inputs_train)
inputs_train_white = pd.DataFrame(pca.transform(inputs_train), columns=inputs_train.columns)
inputs_train_white
Out[17]:
petal_length petal_width sepal_length sepal_width
0 0.387221 0.484604 1.353335 0.761078
1 0.394136 -0.687376 -2.114471 -0.958366
2 0.925573 0.351229 2.526484 -0.225238
3 0.675388 -0.325813 -0.203693 -0.997615
4 0.221818 -1.285400 -0.049826 1.718668
... ... ... ... ...
115 0.015610 -1.118584 -0.788006 0.266962
116 -1.387309 0.580464 0.304557 1.042104
117 1.109262 0.787130 0.996607 0.446290
118 0.611141 -0.274729 0.178736 -0.899852
119 1.388267 0.856468 -1.735097 0.208429

120 rows × 4 columns

In [18]:
# Start a neural network, building it by layers.
model = kr.models.Sequential()

# Add a hidden layer with x neurons and an input layer with 4.
model.add(kr.layers.Dense(units=30, activation='relu', input_dim=4))
# Add a three neuron output layer.
model.add(kr.layers.Dense(units=3, activation='softmax'))

# Build the graph.
model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy'])
In [19]:
# Train the neural network.
model.fit(inputs_train_white, outputs_train, epochs=15, batch_size=10)
Epoch 1/15
120/120 [==============================] - 1s 8ms/step - loss: 1.0737 - acc: 0.4500
Epoch 2/15
120/120 [==============================] - 0s 283us/step - loss: 1.0342 - acc: 0.5667
Epoch 3/15
120/120 [==============================] - 0s 392us/step - loss: 0.9963 - acc: 0.6083
Epoch 4/15
120/120 [==============================] - 0s 291us/step - loss: 0.9618 - acc: 0.6333
Epoch 5/15
120/120 [==============================] - 0s 417us/step - loss: 0.9288 - acc: 0.6500
Epoch 6/15
120/120 [==============================] - 0s 325us/step - loss: 0.8977 - acc: 0.6667
Epoch 7/15
120/120 [==============================] - 0s 496us/step - loss: 0.8687 - acc: 0.6833
Epoch 8/15
120/120 [==============================] - 0s 881us/step - loss: 0.8403 - acc: 0.6917
Epoch 9/15
120/120 [==============================] - 0s 792us/step - loss: 0.8140 - acc: 0.7333
Epoch 10/15
120/120 [==============================] - 0s 933us/step - loss: 0.7887 - acc: 0.7333
Epoch 11/15
120/120 [==============================] - 0s 325us/step - loss: 0.7646 - acc: 0.7583
Epoch 12/15
120/120 [==============================] - 0s 342us/step - loss: 0.7417 - acc: 0.7500
Epoch 13/15
120/120 [==============================] - 0s 267us/step - loss: 0.7200 - acc: 0.7917
Epoch 14/15
120/120 [==============================] - 0s 375us/step - loss: 0.6999 - acc: 0.8000
Epoch 15/15
120/120 [==============================] - 0s 292us/step - loss: 0.6800 - acc: 0.8000
Out[19]:
<keras.callbacks.History at 0x1c4c573b888>
In [20]:
# Have the network predict the classes of the test inputs.
predictions = model.predict(pca.transform(inputs_test))
predictions_labels = encoder.inverse_transform(predictions)
predictions_labels
Out[20]:
array(['versicolor', 'virginica', 'setosa', 'setosa', 'setosa',
       'virginica', 'setosa', 'virginica', 'versicolor', 'versicolor',
       'virginica', 'virginica', 'virginica', 'setosa', 'setosa',
       'versicolor', 'setosa', 'setosa', 'virginica', 'virginica',
       'setosa', 'virginica', 'versicolor', 'setosa', 'setosa',
       'virginica', 'virginica', 'versicolor', 'virginica', 'virginica'],
      dtype='<U10')
In [21]:
(predictions_labels == encoder.inverse_transform(outputs_test)).sum()
Out[21]:
26

End